mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 01:13:23 -04:00
# Backport This will backport the following commits from `main` to `8.8`: - [Removed unused method searchForOutdatedDocuments (#157470)](https://github.com/elastic/kibana/pull/157470) <!--- Backport version: 8.9.7 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Gerard Soldevila","email":"gerard.soldevila@elastic.co"},"sourceCommit":{"committedDate":"2023-05-15T12:14:32Z","message":"Removed unused method searchForOutdatedDocuments (#157470)\n\nThis method is no longer used (replaced by `readWithPit`).\n\n---------\n\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"4951e29973b4125d1b8d6ecb0b1afd9e9095c048","branchLabelMapping":{"^v8.9.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Core","technical debt","release_note:skip","Feature:Migrations","backport:prev-minor","v8.8.0","v8.9.0","v8.8.1"],"number":157470,"url":"https://github.com/elastic/kibana/pull/157470","mergeCommit":{"message":"Removed unused method searchForOutdatedDocuments (#157470)\n\nThis method is no longer used (replaced by `readWithPit`).\n\n---------\n\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"4951e29973b4125d1b8d6ecb0b1afd9e9095c048"}},"sourceBranch":"main","suggestedTargetBranches":["8.8"],"targetPullRequestStates":[{"branch":"8.8","label":"v8.8.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/157470","number":157470,"mergeCommit":{"message":"Removed unused method searchForOutdatedDocuments (#157470)\n\nThis method is no longer used (replaced by `readWithPit`).\n\n---------\n\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"4951e29973b4125d1b8d6ecb0b1afd9e9095c048"}}]}] BACKPORT--> Co-authored-by: Gerard Soldevila <gerard.soldevila@elastic.co>
This commit is contained in:
Kibana Machine
GitHub
parent
bf7e0f9528
commit
661dee06eb
6 changed files with 38 additions and 234 deletions
|
|
@ -32,7 +32,6 @@ export {
|
|||
updateAliases,
|
||||
transformDocs,
|
||||
setWriteBlock,
|
||||
searchForOutdatedDocuments,
|
||||
removeWriteBlock,
|
||||
reindex,
|
||||
readWithPit,
|
||||
|
|
@ -45,7 +44,6 @@ export {
|
|||
export type {
|
||||
OpenPitResponse,
|
||||
ReadWithPit,
|
||||
SearchResponse,
|
||||
ReindexResponse,
|
||||
UpdateByQueryResponse,
|
||||
UpdateAndPickupMappingsResponse,
|
||||
|
|
|
|||
|
|
@ -426,7 +426,7 @@ We can’t use the temporary index as our target index because one instance can
|
|||
|
||||
## OUTDATED_DOCUMENTS_SEARCH
|
||||
### Next action
|
||||
`searchForOutdatedDocuments`
|
||||
`readWithPit(outdatedDocumentsQuery)`
|
||||
|
||||
Search for outdated saved object documents. Will return one batch of
|
||||
documents.
|
||||
|
|
|
|||
|
|
@ -118,12 +118,6 @@ export { checkForUnknownDocs } from './check_for_unknown_docs';
|
|||
|
||||
export { waitForPickupUpdatedMappingsTask } from './wait_for_pickup_updated_mappings_task';
|
||||
|
||||
export type {
|
||||
SearchResponse,
|
||||
SearchForOutdatedDocumentsOptions,
|
||||
} from './search_for_outdated_documents';
|
||||
export { searchForOutdatedDocuments } from './search_for_outdated_documents';
|
||||
|
||||
export type { BulkOverwriteTransformedDocumentsParams } from './bulk_overwrite_transformed_documents';
|
||||
export { bulkOverwriteTransformedDocuments } from './bulk_overwrite_transformed_documents';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { catchRetryableEsClientErrors } from './catch_retryable_es_client_errors';
|
||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
|
||||
import { searchForOutdatedDocuments } from './search_for_outdated_documents';
|
||||
|
||||
jest.mock('./catch_retryable_es_client_errors');
|
||||
|
||||
describe('searchForOutdatedDocuments', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
// Create a mock client that rejects all methods with a 503 status code
|
||||
// response.
|
||||
const retryableError = new EsErrors.ResponseError(
|
||||
elasticsearchClientMock.createApiResponse({
|
||||
statusCode: 503,
|
||||
body: { error: { type: 'es_type', reason: 'es_reason' } },
|
||||
})
|
||||
);
|
||||
const client = elasticsearchClientMock.createInternalClient(
|
||||
elasticsearchClientMock.createErrorTransportRequestPromise(retryableError)
|
||||
);
|
||||
it('calls catchRetryableEsClientErrors when the promise rejects', async () => {
|
||||
const task = searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'new_index',
|
||||
outdatedDocumentsQuery: {},
|
||||
});
|
||||
|
||||
try {
|
||||
await task();
|
||||
} catch (e) {
|
||||
/** ignore */
|
||||
}
|
||||
|
||||
expect(catchRetryableEsClientErrors).toHaveBeenCalledWith(retryableError);
|
||||
});
|
||||
|
||||
it('configures request according to given parameters', async () => {
|
||||
const esClient = elasticsearchClientMock.createInternalClient();
|
||||
const query = {};
|
||||
const targetIndex = 'new_index';
|
||||
const batchSize = 1000;
|
||||
const task = searchForOutdatedDocuments(esClient, {
|
||||
batchSize,
|
||||
targetIndex,
|
||||
outdatedDocumentsQuery: query,
|
||||
});
|
||||
|
||||
await task();
|
||||
|
||||
expect(esClient.search).toHaveBeenCalledTimes(1);
|
||||
expect(esClient.search).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
index: targetIndex,
|
||||
size: batchSize,
|
||||
body: expect.objectContaining({ query }),
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import * as Either from 'fp-ts/lib/Either';
|
||||
import * as TaskEither from 'fp-ts/lib/TaskEither';
|
||||
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
catchRetryableEsClientErrors,
|
||||
type RetryableEsClientError,
|
||||
} from './catch_retryable_es_client_errors';
|
||||
|
||||
/** @internal */
|
||||
export interface SearchResponse {
|
||||
outdatedDocuments: SavedObjectsRawDoc[];
|
||||
}
|
||||
|
||||
export interface SearchForOutdatedDocumentsOptions {
|
||||
batchSize: number;
|
||||
targetIndex: string;
|
||||
outdatedDocumentsQuery?: estypes.QueryDslQueryContainer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for outdated saved object documents with the provided query. Will
|
||||
* return one batch of documents. Searching should be repeated until no more
|
||||
* outdated documents can be found.
|
||||
*
|
||||
* Used for testing only
|
||||
*/
|
||||
export const searchForOutdatedDocuments =
|
||||
(
|
||||
client: ElasticsearchClient,
|
||||
options: SearchForOutdatedDocumentsOptions
|
||||
): TaskEither.TaskEither<RetryableEsClientError, SearchResponse> =>
|
||||
() => {
|
||||
return client
|
||||
.search<SavedObjectsRawDocSource>({
|
||||
index: options.targetIndex,
|
||||
// Return the _seq_no and _primary_term so we can use optimistic
|
||||
// concurrency control for updates
|
||||
seq_no_primary_term: true,
|
||||
size: options.batchSize,
|
||||
body: {
|
||||
query: options.outdatedDocumentsQuery,
|
||||
// Optimize search performance by sorting by the "natural" index order
|
||||
sort: ['_doc'],
|
||||
},
|
||||
// Return an error when targeting missing or closed indices
|
||||
allow_no_indices: false,
|
||||
// Don't return partial results if timeouts or shard failures are
|
||||
// encountered. This is important because 0 search hits is interpreted as
|
||||
// there being no more outdated documents left that require
|
||||
// transformation. Although the default is `false`, we set this
|
||||
// explicitly to avoid users overriding the
|
||||
// search.default_allow_partial_results cluster setting to true.
|
||||
allow_partial_search_results: false,
|
||||
// Improve performance by not calculating the total number of hits
|
||||
// matching the query.
|
||||
track_total_hits: false,
|
||||
// Reduce the response payload size by only returning the data we care about
|
||||
filter_path: [
|
||||
'hits.hits._id',
|
||||
'hits.hits._source',
|
||||
'hits.hits._seq_no',
|
||||
'hits.hits._primary_term',
|
||||
],
|
||||
})
|
||||
.then((res) =>
|
||||
Either.right({ outdatedDocuments: (res.hits?.hits as SavedObjectsRawDoc[]) ?? [] })
|
||||
)
|
||||
.catch(catchRetryableEsClientErrors);
|
||||
};
|
||||
|
|
@ -23,8 +23,6 @@ import {
|
|||
reindex,
|
||||
readWithPit,
|
||||
type ReadWithPit,
|
||||
searchForOutdatedDocuments,
|
||||
type SearchResponse,
|
||||
setWriteBlock,
|
||||
updateAliases,
|
||||
waitForReindexTask,
|
||||
|
|
@ -724,14 +722,9 @@ describe('migration actions', () => {
|
|||
}
|
||||
`);
|
||||
|
||||
const results = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'reindex_target',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(results.map((doc) => doc._source.title).sort()).toMatchInlineSnapshot(`
|
||||
const results = await client.search({ index: 'reindex_target', size: 1000 });
|
||||
expect((results.hits?.hits as SavedObjectsRawDoc[]).map((doc) => doc._source.title).sort())
|
||||
.toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"doc 1",
|
||||
"doc 2",
|
||||
|
|
@ -764,14 +757,9 @@ describe('migration actions', () => {
|
|||
}
|
||||
`);
|
||||
|
||||
const results = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'reindex_target_excluded_docs',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(results.map((doc) => doc._source.title).sort()).toMatchInlineSnapshot(`
|
||||
const results = await client.search({ index: 'reindex_target_excluded_docs', size: 1000 });
|
||||
expect((results.hits?.hits as SavedObjectsRawDoc[]).map((doc) => doc._source.title).sort())
|
||||
.toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"doc 1",
|
||||
"doc 2",
|
||||
|
|
@ -796,14 +784,10 @@ describe('migration actions', () => {
|
|||
"right": "reindex_succeeded",
|
||||
}
|
||||
`);
|
||||
const results = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'reindex_target_2',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(results.map((doc) => doc._source.title).sort()).toMatchInlineSnapshot(`
|
||||
|
||||
const results = await client.search({ index: 'reindex_target_2', size: 1000 });
|
||||
expect((results.hits?.hits as SavedObjectsRawDoc[]).map((doc) => doc._source.title).sort())
|
||||
.toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"doc 1_updated",
|
||||
"doc 2_updated",
|
||||
|
|
@ -850,14 +834,9 @@ describe('migration actions', () => {
|
|||
`);
|
||||
|
||||
// Assert that documents weren't overridden by the second, unscripted reindex
|
||||
const results = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'reindex_target_3',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(results.map((doc) => doc._source.title).sort()).toMatchInlineSnapshot(`
|
||||
const results = await client.search({ index: 'reindex_target_3', size: 1000 });
|
||||
expect((results.hits?.hits as SavedObjectsRawDoc[]).map((doc) => doc._source.title).sort())
|
||||
.toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"doc 1_updated",
|
||||
"doc 2_updated",
|
||||
|
|
@ -872,13 +851,8 @@ describe('migration actions', () => {
|
|||
// Simulate a reindex that only adds some of the documents from the
|
||||
// source index into the target index
|
||||
await createIndex({ client, indexName: 'reindex_target_4', mappings: { properties: {} } })();
|
||||
const sourceDocs = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'existing_index_with_docs',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments
|
||||
const response = await client.search({ index: 'existing_index_with_docs', size: 1000 });
|
||||
const sourceDocs = (response.hits?.hits as SavedObjectsRawDoc[])
|
||||
.slice(0, 2)
|
||||
.map(({ _id, _source }) => ({
|
||||
_id,
|
||||
|
|
@ -909,14 +883,9 @@ describe('migration actions', () => {
|
|||
`);
|
||||
// Assert that existing documents weren't overridden, but that missing
|
||||
// documents were added by the reindex
|
||||
const results = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'reindex_target_4',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(results.map((doc) => doc._source.title).sort()).toMatchInlineSnapshot(`
|
||||
const results = await client.search({ index: 'reindex_target_4', size: 1000 });
|
||||
expect((results.hits?.hits as SavedObjectsRawDoc[]).map((doc) => doc._source.title).sort())
|
||||
.toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"doc 1",
|
||||
"doc 2",
|
||||
|
|
@ -1450,16 +1419,15 @@ describe('migration actions', () => {
|
|||
})();
|
||||
|
||||
// Assert that we can't search over the unmapped fields of the document
|
||||
const originalSearchResults = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'existing_index_without_mappings',
|
||||
outdatedDocumentsQuery: {
|
||||
match: { title: { query: 'doc' } },
|
||||
},
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(originalSearchResults.length).toBe(0);
|
||||
|
||||
const originalSearchResults = await client.search({
|
||||
index: 'existing_index_without_mappings',
|
||||
size: 1000,
|
||||
query: {
|
||||
match: { title: { query: 'doc' } },
|
||||
},
|
||||
});
|
||||
expect(originalSearchResults.hits?.hits.length).toBe(0);
|
||||
|
||||
// Update and pickup mappings so that the title field is searchable
|
||||
const res = await updateAndPickupMappings({
|
||||
|
|
@ -1476,16 +1444,14 @@ describe('migration actions', () => {
|
|||
await waitForPickupUpdatedMappingsTask({ client, taskId, timeout: '60s' })();
|
||||
|
||||
// Repeat the search expecting to be able to find the existing documents
|
||||
const pickedUpSearchResults = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'existing_index_without_mappings',
|
||||
outdatedDocumentsQuery: {
|
||||
match: { title: { query: 'doc' } },
|
||||
},
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
expect(pickedUpSearchResults.length).toBe(4);
|
||||
const pickedUpSearchResults = await client.search({
|
||||
index: 'existing_index_without_mappings',
|
||||
size: 1000,
|
||||
query: {
|
||||
match: { title: { query: 'doc' } },
|
||||
},
|
||||
});
|
||||
expect(pickedUpSearchResults.hits?.hits.length).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1907,13 +1873,8 @@ describe('migration actions', () => {
|
|||
`);
|
||||
});
|
||||
it('resolves right even if there were some version_conflict_engine_exception', async () => {
|
||||
const existingDocs = (
|
||||
(await searchForOutdatedDocuments(client, {
|
||||
batchSize: 1000,
|
||||
targetIndex: 'existing_index_with_docs',
|
||||
outdatedDocumentsQuery: undefined,
|
||||
})()) as Either.Right<SearchResponse>
|
||||
).right.outdatedDocuments;
|
||||
const response = await client.search({ index: 'existing_index_with_docs', size: 1000 });
|
||||
const existingDocs = response.hits?.hits as SavedObjectsRawDoc[];
|
||||
|
||||
const task = bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue