github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add migrations.discardCorruptObjects flag (#132984)

Browse source 
* Add migrations.ignoreUnknownObjects flag

* Misc enhancements following PR comments, fix UTs

* Fix validation

* Make check_for_unknown_docs flag-agnostic. Misc enhancements

* Minor fixes

* Fix UTs

* misc doc updates

* Simplify excludeOnUpgradeQuery helper methods

* Improve unknown type detection, improve e2e tests

* Fix tests

* Simplify queries for unknown types, use "missing:" in agg

* Unify logic from unknown_object_types and check_for_unknown_docs

* Small code enhancements

* Update FTs datasets mappings, ensure saved object type is a "keyword"

* Undo changes in non-related type

* Fix UTs

* Add migrations.discardCorruptObjects flag

* Add migrations.discardCorruptObjects flag

* Fix imports that change with new packages

* Fix imports that change with new packages

* Fix tests

* Update Jest snapshot

* Fix snapshots

* Update ascii docs, revert unwanted mapping update

* Revert to default log output type

* Fix typo [Prebot => Preboot]

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Gerard Soldevila 2022-06-14 09:28:23 +02:00 • committed by GitHub
parent 1fa326b095
commit df5d2c913b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 496 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

16
docs/setup/upgrade/resolving-migration-failures.asciidoc
View file

@ -85,6 +85,16 @@ DELETE .kibana_7.12.0_001/_doc/marketing_space:dashboard:e3c5fc71-ac71-4805-bcab
+
The dashboard with the `e3c5fc71-ac71-4805-bcab-2bcc9cc93275` ID that belongs to the `marketing_space` space **is no longer available**.
You can configure {kib} to automatically discard all corrupt objects and transform errors that occur during a migration. When setting the configuration option `migrations.discardCorruptObjects`, {kib} will delete the conflicting objects and proceed with the migration.
For instance, for an upgrade to 8.4.0, you can define the following setting in kibana.yml:
[source,yaml]
--------------------------------------------
migrations.discardCorruptObjects: "8.4.0"
--------------------------------------------
**WARNING:** Enabling the flag above will cause the system to discard all corrupt objects, as well as those causing transform errors. Thus, it is HIGHLY recommended that you carefully review the list of conflicting objects in the logs.
[float]
[[unknown-saved-object-types]]
==== Documents for unknown saved objects
@ -113,12 +123,12 @@ Unable to complete saved object migrations for the [.kibana] index: Migration fa
To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.
--------------------------------------------
To proceed with the migration, re-enable any plugins that previously created these saved objects. Alternatively, carefully review the list of unknown saved objects in the Kibana log entry. If the corresponding disabled plugins and their associated saved objects will no longer be used, they can be deleted by setting the configuration option `migrations.discardUnknownObjects` to the version you are upgrading to.
For instance, for an upgrade to 8.3.0, you can define the following setting in kibana.yml:
To proceed with the migration, re-enable any plugins that previously created these saved objects. Alternatively, carefully review the list of unknown saved objects in the {kib} log entry. If the corresponding disabled plugins and their associated saved objects will no longer be used, they can be deleted by setting the configuration option `migrations.discardUnknownObjects` to the version you are upgrading to.
For instance, for an upgrade to 8.4.0, you can define the following setting in kibana.yml:
[source,yaml]
--------------------------------------------
migrations.discardUnknownObjects: "8.3.0"
migrations.discardUnknownObjects: "8.4.0"
--------------------------------------------
[float]

6
src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap generated
View file

@ -20,6 +20,7 @@ Object {
"batchSize": 1000,
"controlState": "LEGACY_REINDEX",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -189,6 +190,7 @@ Object {
"batchSize": 1000,
"controlState": "LEGACY_DELETE",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -362,6 +364,7 @@ Object {
"batchSize": 1000,
"controlState": "LEGACY_DELETE",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -539,6 +542,7 @@ Object {
"batchSize": 1000,
"controlState": "DONE",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -758,6 +762,7 @@ Object {
"batchSize": 1000,
"controlState": "LEGACY_DELETE",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -938,6 +943,7 @@ Object {
"batchSize": 1000,
"controlState": "FATAL",
"currentAlias": ".my-so-index",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {

8
src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts
View file

@ -192,9 +192,15 @@ describe('migrateRawDocsSafely', () => {
const result = (await task()) as Either.Left<DocumentsTransformFailed>;
expect(transform).toHaveBeenCalledTimes(1);
expect(result._tag).toEqual('Left');
expect(Object.keys(result.left)).toEqual(['type', 'corruptDocumentIds', 'transformErrors']);
expect(Object.keys(result.left)).toEqual([
'type',
'corruptDocumentIds',
'transformErrors',
'processedDocs',
]);
expect(result.left.corruptDocumentIds.length).toEqual(1);
expect(result.left.transformErrors.length).toEqual(0);
expect(result.left.processedDocs.length).toEqual(1);
});
test('handles when one document is transformed into multiple documents', async () => {

2
src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts
View file

@ -24,6 +24,7 @@ export interface DocumentsTransformFailed {
readonly type: string;
readonly corruptDocumentIds: string[];
readonly transformErrors: TransformErrorObjects[];
readonly processedDocs: SavedObjectsRawDoc[];
}
export interface DocumentsTransformSuccess {
@ -139,6 +140,7 @@ export function migrateRawDocsSafely({
type: 'documents_transform_failed',
corruptDocumentIds: [...corruptSavedObjectIds],
transformErrors,
processedDocs,
});
}
return Either.right({ processedDocs });

53
src/core/server/saved_objects/migrations/initial_state.test.ts
View file

@ -26,11 +26,14 @@ describe('createInitialState', () => {
docLinks = docLinksServiceMock.createSetupContract();
});
afterEach(() => jest.clearAllMocks());
const migrationsConfig = {
retryAttempts: 15,
batchSize: 1000,
maxBatchSizeBytes: ByteSizeValue.parse('100mb'),
} as unknown as SavedObjectsMigrationConfigType;
it('creates the initial state for the model based on the passed in parameters', () => {
expect(
createInitialState({
@ -51,6 +54,7 @@ describe('createInitialState', () => {
"batchSize": 1000,
"controlState": "INIT",
"currentAlias": ".kibana_task_manager",
"discardCorruptObjects": false,
"discardUnknownObjects": false,
"excludeFromUpgradeFilterHooks": Object {},
"excludeOnUpgradeQuery": Object {
@ -400,7 +404,7 @@ describe('createInitialState', () => {
expect(initialState.discardUnknownObjects).toEqual(false);
expect(logger.warn).toBeCalledTimes(1);
expect(logger.warn).toBeCalledWith(
'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be ignored.'
'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be discarded.'
);
});
@ -424,4 +428,51 @@ describe('createInitialState', () => {
expect(initialState.discardUnknownObjects).toEqual(true);
});
it('initializes the `discardCorruptObjects` flag to false if the value provided in the config does not match the current kibana version', () => {
const logger = mockLogger.get();
const initialState = createInitialState({
kibanaVersion: '8.1.0',
targetMappings: {
dynamic: 'strict',
properties: { my_type: { properties: { title: { type: 'text' } } } },
},
migrationVersionPerType: {},
indexPrefix: '.kibana_task_manager',
migrationsConfig: {
...migrationsConfig,
discardCorruptObjects: '8.0.0',
},
typeRegistry,
docLinks,
logger,
});
expect(initialState.discardCorruptObjects).toEqual(false);
expect(logger.warn).toBeCalledTimes(1);
expect(logger.warn).toBeCalledWith(
'The flag `migrations.discardCorruptObjects` is defined but does not match the current kibana version; corrupt objects will NOT be discarded.'
);
});
it('initializes the `discardCorruptObjects` flag to true if the value provided in the config matches the current kibana version', () => {
const initialState = createInitialState({
kibanaVersion: '8.1.0',
targetMappings: {
dynamic: 'strict',
properties: { my_type: { properties: { title: { type: 'text' } } } },
},
migrationVersionPerType: {},
indexPrefix: '.kibana_task_manager',
migrationsConfig: {
...migrationsConfig,
discardCorruptObjects: '8.1.0',
},
typeRegistry,
docLinks,
logger: mockLogger.get(),
});
expect(initialState.discardCorruptObjects).toEqual(true);
});
});

12
src/core/server/saved_objects/migrations/initial_state.ts
View file

@ -78,7 +78,16 @@ export const createInitialState = ({
migrationsConfig.discardUnknownObjects !== kibanaVersion
) {
logger.warn(
'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be ignored.'
'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be discarded.'
);
}
if (
migrationsConfig.discardCorruptObjects &&
migrationsConfig.discardCorruptObjects !== kibanaVersion
) {
logger.warn(
'The flag `migrations.discardCorruptObjects` is defined but does not match the current kibana version; corrupt objects will NOT be discarded.'
);
}
@ -101,6 +110,7 @@ export const createInitialState = ({
batchSize: migrationsConfig.batchSize,
maxBatchSizeBytes: migrationsConfig.maxBatchSizeBytes.getValueInBytes(),
discardUnknownObjects: migrationsConfig.discardUnknownObjects === kibanaVersion,
discardCorruptObjects: migrationsConfig.discardCorruptObjects === kibanaVersion,
logs: [],
excludeOnUpgradeQuery: excludeUnusedTypesQuery,
knownTypes,

279
src/core/server/saved_objects/migrations/integration_tests/7_13_0_transform_failures.test.ts
View file

@ -9,8 +9,13 @@
import Path from 'path';
import Fs from 'fs';
import Util from 'util';
import { Env } from '@kbn/config';
import { REPO_ROOT } from '@kbn/utils';
import { getEnvOptions } from '@kbn/config-mocks';
import type { ElasticsearchClient } from '../../../elasticsearch';
import * as kbnTestServer from '../../../../test_helpers/kbn_server';
import { Root } from '../../../root';
import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types';
const logFilePath = Path.join(__dirname, '7_13_corrupt_transform_failures.log');
@ -29,133 +34,162 @@ describe('migration v2', () => {
await removeLogFile();
});
afterAll(async () => {
afterEach(async () => {
if (root) {
await root.shutdown();
}
if (esServer) {
await esServer.stop();
}
await new Promise((resolve) => setTimeout(resolve, 10000));
});
it('migrates the documents to the highest version', async () => {
const { startES } = kbnTestServer.createTestServers({
adjustTimeout: (t: number) => jest.setTimeout(t),
settings: {
es: {
license: 'basic',
// example of original 'foo' SO with corrupt id:
// _id: one
// {
// foo: {
// name: 'one',
// },
// type: 'foo',
// references: [],
// migrationVersion: {
// foo: '7.13.0',
// },
// "coreMigrationVersion": "7.13.0",
// "updated_at": "2021-05-16T18:16:45.450Z"
// },
describe('when `migrations.discardCorruptObjects` does not match current kibana version', () => {
it('fails to migrate when corrupt objects and transform errors are encountered', async () => {
const { startES } = createTestServers();
root = createRoot();
esServer = await startES();
await rootPrebootAndSetup(root);
// SO that will fail transformation:
// {
// type: 'space',
// space: {},
// },
//
//
dataArchive: Path.join(
__dirname,
'archives',
'7_13_corrupt_and_transform_failures_docs.zip'
),
},
},
});
try {
await root.start();
} catch (err) {
const errorMessage = err.message;
const errorLines = errorMessage.split('\n');
const errorMessageWithoutStack = errorLines
.filter((line: string) => !line.includes(' at '))
.join('\n');
root = createRoot();
expect(errorMessageWithoutStack).toMatchInlineSnapshot(`
"Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 7 corrupt saved object documents were found: P2SQfHkBs3dBRGh--No5, QGSZfHkBs3dBRGh-ANoD, QWSZfHkBs3dBRGh-hNob, QmSZfHkBs3dBRGh-w9qH, one, two, Q2SZfHkBs3dBRGh-9dp2
7 transformation errors were encountered:
- space:default: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:first: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:forth: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:second: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:fifth: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:third: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
- space:sixth: Error: Migration function for version 7.14.0 threw an error
Caused by:
TypeError: Cannot set properties of undefined (setting 'bar')
esServer = await startES();
await root.preboot();
const coreSetup = await root.setup();
coreSetup.savedObjects.registerType({
name: 'foo',
hidden: false,
mappings: {
properties: {},
},
namespaceType: 'agnostic',
migrations: {
'7.14.0': (doc) => doc,
},
});
// registering the `space` type with a throwing migration fn to avoid the migration failing for unknown types
coreSetup.savedObjects.registerType({
name: 'space',
hidden: false,
mappings: {
properties: {},
},
namespaceType: 'single',
migrations: {
'7.14.0': (doc) => {
doc.attributes.foo.bar = 12;
return doc;
},
},
});
try {
await root.start();
} catch (err) {
const errorMessage = err.message;
expect(
errorMessage.startsWith(
'Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 7 corrupt saved object documents were found: '
)
).toBeTruthy();
expect(
errorMessage.endsWith(
'To allow migrations to proceed, please delete or fix these documents.'
)
).toBeTruthy();
const expectedCorruptDocIds = [
'P2SQfHkBs3dBRGh--No5',
'QGSZfHkBs3dBRGh-ANoD',
'QWSZfHkBs3dBRGh-hNob',
'QmSZfHkBs3dBRGh-w9qH',
'one',
'two',
'Q2SZfHkBs3dBRGh-9dp2',
];
for (const corruptDocId of expectedCorruptDocIds) {
expect(errorMessage.includes(corruptDocId)).toBeTruthy();
To allow migrations to proceed, please delete or fix these documents.
Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
Please refer to https://www.elastic.co/guide/en/kibana/master/resolve-migrations-failures.html for more information."
`);
return;
}
// Fail test if above expression doesn't throw anything.
expect('to throw').toBe('but did not');
});
});
expect(errorMessage.includes('7 transformation errors were encountered:')).toBeTruthy();
expect(errorMessage).toEqual(
expect.stringContaining(
'space:sixth: Error: Migration function for version 7.14.0 threw an error'
)
);
}
describe('when `migrations.discardCorruptObjects` matches current kibana version', () => {
it('proceeds with the migration, ignoring corrupt objects and transform errors', async () => {
const { startES } = createTestServers();
const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
root = createRoot(currentVersion);
esServer = await startES();
await rootPrebootAndSetup(root);
await expect(root.start()).resolves.not.toThrowError();
// TODO check that the destination indices contain data, but NOT the conflicting objects
const esClient: ElasticsearchClient = esServer.es.getClient();
const docs = await esClient.search({
index: '.kibana',
_source: false,
fields: ['_id'],
size: 50,
});
// 23 saved objects + 14 corrupt (discarded) = 37 total in the old index
expect((docs.hits.total as SearchTotalHits).value).toEqual(23);
expect(docs.hits.hits.map(({ _id }) => _id)).toEqual([
'config:7.13.0',
'index-pattern:logs-*',
'index-pattern:metrics-*',
'usage-counters:uiCounter:21052021:click:global_search_bar:user_navigated_to_application',
'usage-counters:uiCounter:21052021:count:console:GET_cat.aliases',
'usage-counters:uiCounter:21052021:loaded:console:opened_app',
'usage-counters:uiCounter:21052021:count:console:GET_cat.indices',
'usage-counters:uiCounter:21052021:count:global_search_bar:search_focus',
'usage-counters:uiCounter:21052021:click:global_search_bar:user_navigated_to_application_unknown',
'usage-counters:uiCounter:21052021:count:global_search_bar:search_request',
'usage-counters:uiCounter:21052021:count:global_search_bar:shortcut_used',
'ui-metric:console:POST_delete_by_query',
'usage-counters:uiCounter:21052021:count:console:PUT_indices.put_mapping',
'usage-counters:uiCounter:21052021:count:console:POST_delete_by_query',
'usage-counters:uiCounter:21052021:count:console:GET_search',
'ui-metric:console:PUT_indices.put_mapping',
'ui-metric:console:GET_search',
'usage-counters:uiCounter:21052021:count:console:DELETE_delete',
'ui-metric:console:DELETE_delete',
'usage-counters:uiCounter:21052021:count:console:GET_get',
'ui-metric:console:GET_get',
'usage-counters:uiCounter:21052021:count:console:POST_index',
'ui-metric:console:POST_index',
]);
});
});
});
function createRoot() {
function createTestServers() {
return kbnTestServer.createTestServers({
adjustTimeout: (t: number) => jest.setTimeout(t),
settings: {
es: {
license: 'basic',
// example of original 'foo' SO with corrupt id:
// _id: one
// {
// foo: {
// name: 'one',
// },
// type: 'foo',
// references: [],
// migrationVersion: {
// foo: '7.13.0',
// },
// "coreMigrationVersion": "7.13.0",
// "updated_at": "2021-05-16T18:16:45.450Z"
// },
// SO that will fail transformation:
// {
// type: 'space',
// space: {},
// },
//
//
dataArchive: Path.join(
__dirname,
'archives',
'7_13_corrupt_and_transform_failures_docs.zip'
),
},
},
});
}
function createRoot(discardCorruptObjects?: string) {
return kbnTestServer.createRootWithCorePlugins(
{
migrations: {
skip: false,
batchSize: 5,
discardCorruptObjects,
},
logging: {
appenders: {
@ -181,3 +215,36 @@ function createRoot() {
}
);
}
async function rootPrebootAndSetup(root: Root) {
await root.preboot();
const coreSetup = await root.setup();
coreSetup.savedObjects.registerType({
name: 'foo',
hidden: false,
mappings: {
properties: {},
},
namespaceType: 'agnostic',
migrations: {
'7.14.0': (doc) => doc,
},
});
// registering the `space` type with a throwing migration fn to avoid the migration failing for unknown types
coreSetup.savedObjects.registerType({
name: 'space',
hidden: false,
mappings: {
properties: {},
},
namespaceType: 'single',
migrations: {
'7.14.0': (doc) => {
doc.attributes.foo.bar = 12;
return doc;
},
},
});
}

9
src/core/server/saved_objects/migrations/integration_tests/cleanup.test.ts
View file

@ -114,9 +114,12 @@ describe('migration v2', () => {
});
await expect(root.start()).rejects.toThrowErrorMatchingInlineSnapshot(`
"Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 1 corrupt saved object documents were found: index-pattern:test_index*
To allow migrations to proceed, please delete or fix these documents."
`);
"Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 1 corrupt saved object documents were found: index-pattern:test_index*
To allow migrations to proceed, please delete or fix these documents.
Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
Please refer to https://www.elastic.co/guide/en/kibana/master/resolve-migrations-failures.html for more information."
`);
const logFileContent = await asyncReadFile(logFilePath, 'utf-8');
const records = logFileContent

22
src/core/server/saved_objects/migrations/integration_tests/collects_corrupt_docs.test.ts
View file

@ -90,13 +90,23 @@ describe('migration v2 with corrupt saved object documents', () => {
} catch (err) {
const errorMessage = err.message;
const errorLines = errorMessage.split('\n');
const errorMessageWithoutStack = errorLines
.filter((line: string) => !line.includes(' at '))
.join('\n');
expect(errorLines[0]).toEqual(
`Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 2 transformation errors were encountered:`
);
expect(errorLines[errorLines.length - 1]).toEqual(
`To allow migrations to proceed, please delete or fix these documents.`
);
expect(errorMessageWithoutStack).toMatchInlineSnapshot(`
"Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 2 transformation errors were encountered:
- foo:3: Error: Migration function for version 7.14.0 threw an error
Caused by:
Error: \\"number\\" attribute should be present
- foo:4: Error: Migration function for version 7.14.0 threw an error
Caused by:
Error: \\"number\\" attribute should be present
To allow migrations to proceed, please delete or fix these documents.
Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
Please refer to https://www.elastic.co/guide/en/kibana/master/resolve-migrations-failures.html for more information."
`);
expectMatchOrder(errorLines, [
{

52
src/core/server/saved_objects/migrations/model/extract_errors.test.ts
View file

@ -10,6 +10,8 @@ import {
extractUnknownDocFailureReason,
extractDiscardedUnknownDocs,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
extractDiscardedCorruptDocs,
extractTransformFailuresReason,
} from './extract_errors';
describe('extractUnknownDocFailureReason', () => {
@ -59,6 +61,56 @@ describe('extractDiscardedUnknownDocs', () => {
});
});
describe('extractTransformFailuresReason', () => {
it('generates the correct error message', () => {
const errorMessage = extractTransformFailuresReason(
'https://someurl.co',
['corrupt_1', 'corrupt_2'],
[
{ rawId: 'transform_1', err: new Error('Oops!') },
{ rawId: 'transform_2', err: new Error('I did it again!') },
]
);
const errorLines = errorMessage.split('\n');
const errorMessageWithoutStack = errorLines
.filter((line: string) => !line.includes(' at '))
.join('\n');
expect(errorMessageWithoutStack).toMatchInlineSnapshot(`
"Migrations failed. Reason: 2 corrupt saved object documents were found: corrupt_1, corrupt_2
2 transformation errors were encountered:
- transform_1: Error: Oops!
- transform_2: Error: I did it again!
To allow migrations to proceed, please delete or fix these documents.
Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
Please refer to https://someurl.co for more information."
`);
});
});
describe('extractDiscardedCorruptDocs', () => {
it('generates the correct error message', () => {
expect(
extractDiscardedCorruptDocs(
['corrupt_1', 'corrupt_2'],
[
{ rawId: 'transform_1', err: new Error('Oops!') },
{ rawId: 'transform_2', err: new Error('I did it again!') },
]
)
).toMatchInlineSnapshot(`
"Kibana has been configured to discard corrupt documents and documents that cause transform errors for this migration.
Therefore, the following documents will not be taken into account and they will not be available after the migration:
- \\"corrupt_1\\" (corrupt)
- \\"corrupt_2\\" (corrupt)
- \\"transform_1\\" (Oops!)
- \\"transform_2\\" (I did it again!)
"
`);
});
});
describe('fatalReasonDocumentExceedsMaxBatchSizeBytes', () => {
it('generate the correct error message', () => {
expect(

21
src/core/server/saved_objects/migrations/model/extract_errors.ts
View file

@ -13,6 +13,7 @@ import type { DocumentIdAndType } from '../actions';
* Constructs migration failure message strings from corrupt document ids and document transformation errors
*/
export function extractTransformFailuresReason(
resolveMigrationFailuresUrl: string,
corruptDocumentIds: string[],
transformErrors: TransformErrorObjects[]
): string {
@ -20,19 +21,21 @@ export function extractTransformFailuresReason(
corruptDocumentIds.length > 0
? ` ${
corruptDocumentIds.length
} corrupt saved object documents were found: ${corruptDocumentIds.join(',')}`
} corrupt saved object documents were found: ${corruptDocumentIds.join(', ')}\n`
: '';
// we have both the saved object Id and the stack trace in each `transformErrors` item.
const transformErrorsReason =
transformErrors.length > 0
? ` ${transformErrors.length} transformation errors were encountered:\n ` +
? ` ${transformErrors.length} transformation errors were encountered:\n` +
transformErrors
.map((errObj) => `- ${errObj.rawId}: ${errObj.err.stack ?? errObj.err.message}\n`)
.join('')
: '';
return (
`Migrations failed. Reason:${corruptDocumentIdReason}${transformErrorsReason}\n` +
`To allow migrations to proceed, please delete or fix these documents.`
`To allow migrations to proceed, please delete or fix these documents.\n` +
`Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.\n` +
`Please refer to ${resolveMigrationFailuresUrl} for more information.`
);
}
@ -56,6 +59,18 @@ export function extractUnknownDocFailureReason(
);
}
export function extractDiscardedCorruptDocs(
corruptDocumentIds: string[],
transformErrors: TransformErrorObjects[]
): string {
return (
`Kibana has been configured to discard corrupt documents and documents that cause transform errors for this migration.\n` +
`Therefore, the following documents will not be taken into account and they will not be available after the migration:\n` +
corruptDocumentIds.map((id) => `- "${id}" (corrupt)\n`).join('') +
transformErrors.map((error) => `- "${error.rawId}" (${error.err.message})\n`).join('')
);
}
/**
* Constructs migration failure message string for doc exceeds max batch size in bytes
*/

74
src/core/server/saved_objects/migrations/model/model.test.ts
View file

@ -60,6 +60,7 @@ describe('migrations v2 model', () => {
batchSize: 1000,
maxBatchSizeBytes: 1e8,
discardUnknownObjects: false,
discardCorruptObjects: false,
indexPrefix: '.kibana',
outdatedDocumentsQuery: {},
targetIndexMappings: {
@ -96,7 +97,7 @@ describe('migrations v2 model', () => {
knownTypes: ['dashboard', 'config'],
excludeFromUpgradeFilterHooks: {},
migrationDocLinks: {
resolveMigrationFailures: 'resolveMigrationFailures',
resolveMigrationFailures: 'https://someurl.co/',
repeatedTimeoutRequests: 'repeatedTimeoutRequests',
routingAllocationDisabled: 'routingAllocationDisabled',
clusterShardLimitExceeded: 'clusterShardLimitExceeded',
@ -1138,24 +1139,56 @@ describe('migrations v2 model', () => {
expect(newState.logs).toStrictEqual([]); // No logs because no hits
});
it('REINDEX_SOURCE_TO_TEMP_READ -> FATAL if no outdated documents to reindex and transform failures seen with previous outdated documents', () => {
const testState: ReindexSourceToTempRead = {
...state,
corruptDocumentIds: ['a:b'],
transformErrors: [],
};
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({
outdatedDocuments: [],
lastHitSortValue: undefined,
totalHits: undefined,
describe('when transform failures or corrupt documents are found', () => {
it('REINDEX_SOURCE_TO_TEMP_READ -> FATAL if no outdated documents to reindex and transform failures seen with previous outdated documents', () => {
const testState: ReindexSourceToTempRead = {
...state,
corruptDocumentIds: ['a:b'],
transformErrors: [],
};
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({
outdatedDocuments: [],
lastHitSortValue: undefined,
totalHits: undefined,
});
const newState = model(testState, res) as FatalState;
expect(newState.controlState).toBe('FATAL');
expect(newState.reason).toMatchInlineSnapshot(`
"Migrations failed. Reason: 1 corrupt saved object documents were found: a:b
To allow migrations to proceed, please delete or fix these documents.
Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
Please refer to https://someurl.co/ for more information."
`);
expect(newState.logs).toStrictEqual([]); // No logs because no hits
});
it('REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT if discardCorruptObjects=true', () => {
const testState: ReindexSourceToTempRead = {
...state,
discardCorruptObjects: true,
corruptDocumentIds: ['a:b'],
transformErrors: [{ rawId: 'c:d', err: new Error('Oops!') }],
};
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({
outdatedDocuments: [],
lastHitSortValue: undefined,
totalHits: undefined,
});
const newState = model(testState, res) as ReindexSourceToTempClosePit;
expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT');
expect(newState.logs.length).toEqual(1);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "warning",
"message": "Kibana has been configured to discard corrupt documents and documents that cause transform errors for this migration.
Therefore, the following documents will not be taken into account and they will not be available after the migration:
- \\"a:b\\" (corrupt)
- \\"c:d\\" (Oops!)
",
}
`);
});
const newState = model(testState, res) as FatalState;
expect(newState.controlState).toBe('FATAL');
expect(newState.reason).toMatchInlineSnapshot(`
"Migrations failed. Reason: 1 corrupt saved object documents were found: a:b
To allow migrations to proceed, please delete or fix these documents."
`);
expect(newState.logs).toStrictEqual([]); // No logs because no hits
});
});
@ -1250,6 +1283,7 @@ describe('migrations v2 model', () => {
type: 'documents_transform_failed',
corruptDocumentIds: ['a:b'],
transformErrors: [],
processedDocs: [],
});
const newState = model(state, res) as ReindexSourceToTempRead;
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_READ');
@ -1279,6 +1313,8 @@ describe('migrations v2 model', () => {
sourceIndexPitId: 'pit_id',
targetIndex: '.kibana_7.11.0_001',
lastHitSortValue: undefined,
transformErrors: [],
corruptDocumentIds: [],
progress: createInitialProgress(),
};
test('REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ if action succeeded', () => {
@ -1671,6 +1707,7 @@ describe('migrations v2 model', () => {
type: 'documents_transform_failed',
corruptDocumentIds,
transformErrors: [],
processedDocs: [],
});
const newState = model(
outdatedDocumentsTransformState,
@ -1691,6 +1728,7 @@ describe('migrations v2 model', () => {
type: 'documents_transform_failed',
corruptDocumentIds: newFailedTransformDocumentIds,
transformErrors: transformationErrors,
processedDocs: [],
});
const newState = model(
outdatedDocumentsTransformStateWithFailedDocuments,

97
src/core/server/saved_objects/migrations/model/model.ts
View file

@ -25,6 +25,7 @@ import {
extractUnknownDocFailureReason,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
extractDiscardedUnknownDocs,
extractDiscardedCorruptDocs,
} from './extract_errors';
import type { ExcludeRetryableEsError } from './types';
import {
@ -53,6 +54,8 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// `const res = resW as ResponseType<typeof stateP.controlState>;`
let stateP: State = currentState;
let logs: MigrationLog[] = stateP.logs;
let excludeOnUpgradeQuery = stateP.excludeOnUpgradeQuery;
// Handle retryable_es_client_errors. Other left values need to be handled
// by the control state specific code below.
@ -372,9 +375,6 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
} else if (stateP.controlState === 'CHECK_UNKNOWN_DOCUMENTS') {
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
let logs: MigrationLog[] = stateP.logs;
let excludeOnUpgradeQuery = stateP.excludeOnUpgradeQuery;
if (isTypeof(res.right, 'unknown_docs_found')) {
if (!stateP.discardUnknownObjects) {
return {
@ -447,7 +447,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
if (Either.isRight(res)) {
const excludeOnUpgradeQuery = addMustNotClausesToBoolQuery(
excludeOnUpgradeQuery = addMustNotClausesToBoolQuery(
res.right.mustNotClauses,
stateP.excludeOnUpgradeQuery?.bool
);
@ -518,7 +518,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
if (Either.isRight(res)) {
const progress = setProgressTotal(stateP.progress, res.right.totalHits);
const logs = logProgress(stateP.logs, progress);
logs = logProgress(stateP.logs, progress);
if (res.right.outdatedDocuments.length > 0) {
return {
...stateP,
@ -531,24 +531,42 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
} else {
// we don't have any more outdated documents and need to either fail or move on to updating the target mappings.
if (stateP.corruptDocumentIds.length > 0 || stateP.transformErrors.length > 0) {
const transformFailureReason = extractTransformFailuresReason(
stateP.corruptDocumentIds,
stateP.transformErrors
);
return {
...stateP,
controlState: 'FATAL',
reason: transformFailureReason,
};
} else {
// we don't have any more outdated documents and we haven't encountered any document transformation issues.
// Close the PIT search and carry on with the happy path.
return {
...stateP,
controlState: 'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT',
logs,
};
if (!stateP.discardCorruptObjects) {
const transformFailureReason = extractTransformFailuresReason(
stateP.migrationDocLinks.resolveMigrationFailures,
stateP.corruptDocumentIds,
stateP.transformErrors
);
return {
...stateP,
controlState: 'FATAL',
reason: transformFailureReason,
};
}
// at this point, users have configured kibana to discard corrupt objects
// thus, we can ignore corrupt documents and transform errors and proceed with the migration
logs = [
...stateP.logs,
{
level: 'warning',
message: extractDiscardedCorruptDocs(
stateP.corruptDocumentIds,
stateP.transformErrors
),
},
];
}
// we don't have any more outdated documents and either
// we haven't encountered any document transformation issues.
// or the user chose to ignore them
// Close the PIT search and carry on with the happy path.
return {
...stateP,
controlState: 'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT',
logs,
};
}
} else {
throwBadResponse(stateP, res);
@ -576,16 +594,31 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// Otherwise the progress might look off when there are errors.
const progress = incrementProcessedProgress(stateP.progress, stateP.outdatedDocuments.length);
if (Either.isRight(res)) {
if (stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) {
const batches = createBatches(
res.right.processedDocs,
stateP.tempIndex,
stateP.maxBatchSizeBytes
);
if (
Either.isRight(res) ||
(isTypeof(res.left, 'documents_transform_failed') && stateP.discardCorruptObjects)
) {
if (
(stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) ||
stateP.discardCorruptObjects
) {
const processedDocs = Either.isRight(res)
? res.right.processedDocs
: res.left.processedDocs;
const batches = createBatches(processedDocs, stateP.tempIndex, stateP.maxBatchSizeBytes);
if (Either.isRight(batches)) {
let corruptDocumentIds = stateP.corruptDocumentIds;
let transformErrors = stateP.transformErrors;
if (Either.isLeft(res)) {
corruptDocumentIds = [...stateP.corruptDocumentIds, ...res.left.corruptDocumentIds];
transformErrors = [...stateP.transformErrors, ...res.left.transformErrors];
}
return {
...stateP,
corruptDocumentIds,
transformErrors,
controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX_BULK', // handles the actual bulk indexing into temp index
transformedDocBatches: batches.right,
currentBatch: 0,
@ -642,9 +675,6 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
return {
...stateP,
controlState: 'REINDEX_SOURCE_TO_TEMP_READ',
// we're still on the happy path with no transformation failures seen.
corruptDocumentIds: [],
transformErrors: [],
};
}
} else {
@ -764,7 +794,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
if (Either.isRight(res)) {
if (res.right.outdatedDocuments.length > 0) {
const progress = setProgressTotal(stateP.progress, res.right.totalHits);
const logs = logProgress(stateP.logs, progress);
logs = logProgress(stateP.logs, progress);
return {
...stateP,
@ -778,6 +808,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// we don't have any more outdated documents and need to either fail or move on to updating the target mappings.
if (stateP.corruptDocumentIds.length > 0 || stateP.transformErrors.length > 0) {
const transformFailureReason = extractTransformFailuresReason(
stateP.migrationDocLinks.resolveMigrationFailures,
stateP.corruptDocumentIds,
stateP.transformErrors
);

26
src/core/server/saved_objects/migrations/state.ts
View file

@ -95,6 +95,13 @@ export interface BaseState extends ControlState {
* saved objects is disabled.
*/
readonly discardUnknownObjects: boolean;
/**
* If saved objects exist which are corrupt or they can't be migrated due to
* transform errors, they will cause the migration to fail. If this flag is set
* to `true`, kibana will discard the objects that cause these errors
* and proceed with the migration.
*/
readonly discardCorruptObjects: boolean;
/**
* The current alias e.g. `.kibana` which always points to the latest
* version index
@ -221,8 +228,7 @@ export interface ReindexSourceToTempOpenPit extends PostInitState {
readonly sourceIndex: Option.Some<string>;
}
export interface ReindexSourceToTempRead extends PostInitState {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_READ';
interface ReindexSourceToTempBatch extends PostInitState {
readonly sourceIndexPitId: string;
readonly lastHitSortValue: number[] | undefined;
readonly corruptDocumentIds: string[];
@ -230,28 +236,24 @@ export interface ReindexSourceToTempRead extends PostInitState {
readonly progress: Progress;
}
export interface ReindexSourceToTempRead extends ReindexSourceToTempBatch {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_READ';
}
export interface ReindexSourceToTempClosePit extends PostInitState {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT';
readonly sourceIndexPitId: string;
}
export interface ReindexSourceToTempTransform extends PostInitState {
export interface ReindexSourceToTempTransform extends ReindexSourceToTempBatch {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_TRANSFORM';
readonly outdatedDocuments: SavedObjectsRawDoc[];
readonly sourceIndexPitId: string;
readonly lastHitSortValue: number[] | undefined;
readonly corruptDocumentIds: string[];
readonly transformErrors: TransformErrorObjects[];
readonly progress: Progress;
}
export interface ReindexSourceToTempIndexBulk extends PostInitState {
export interface ReindexSourceToTempIndexBulk extends ReindexSourceToTempBatch {
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX_BULK';
readonly transformedDocBatches: [SavedObjectsRawDoc[]];
readonly currentBatch: number;
readonly sourceIndexPitId: string;
readonly lastHitSortValue: number[] | undefined;
readonly progress: Progress;
}
export type SetTempWriteBlock = PostInitState & {

6
src/core/server/saved_objects/saved_objects_config.ts
View file

@ -19,6 +19,12 @@ const migrationSchema = schema.object({
valid(value) ? undefined : 'The value is not a valid semantic version',
})
),
discardCorruptObjects: schema.maybe(
schema.string({
validate: (value: string) =>
valid(value) ? undefined : 'The value is not a valid semantic version',
})
),
scrollDuration: schema.string({ defaultValue: '15m' }),
pollInterval: schema.number({ defaultValue: 1_500 }),
skip: schema.boolean({ defaultValue: false }),