github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

Improve saved objects migrations failure errors and logs (#131359)

Browse source
This commit is contained in:
Christiane (Tina) Heiligers 2022-05-03 18:03:48 -07:00 • committed by GitHub
parent 01e12ec1d6
commit 4b19c72dde
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 258 additions and 165 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/setup/upgrade/resolving-migration-failures.asciidoc
View file

@ -171,7 +171,7 @@ Upgrade migrations fail because routing allocation is disabled or restricted (`c
[source,sh]
--------------------------------------------
Unable to complete saved object migrations for the [.kibana] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
Unable to complete saved object migrations for the [.kibana] index: [unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
--------------------------------------------
To get around the issue, remove the transient and persisted routing allocation settings:

2
packages/kbn-doc-links/src/get_doc_links.ts
View file

@ -648,6 +648,8 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => {
},
kibanaUpgradeSavedObjects: {
resolveMigrationFailures: `${KIBANA_DOCS}resolve-migrations-failures.html`,
repeatedTimeoutRequests: `${KIBANA_DOCS}resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail`,
routingAllocationDisabled: `${KIBANA_DOCS}resolve-migrations-failures.html#routing-allocation-disabled`,
},
});
};

2
packages/kbn-doc-links/src/types.ts
View file

@ -403,5 +403,7 @@ export interface DocLinks {
};
readonly kibanaUpgradeSavedObjects: {
readonly resolveMigrationFailures: string;
readonly repeatedTimeoutRequests: string;
readonly routingAllocationDisabled: string;
};
}

12
src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap generated
View file

@ -33,7 +33,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
@ -197,7 +199,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
@ -365,7 +369,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
@ -537,7 +543,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [],
"outdatedDocumentsQuery": Object {
@ -735,7 +743,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [
Object {
@ -910,7 +920,9 @@ Object {
],
"maxBatchSizeBytes": 100000000,
"migrationDocLinks": Object {
"repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail",
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
},
"outdatedDocuments": Array [
Object {

7
src/core/server/saved_objects/migrations/actions/initialize_action.ts
View file

@ -29,6 +29,7 @@ export interface InitActionParams {
export interface UnsupportedClusterRoutingAllocation {
type: 'unsupported_cluster_routing_allocation';
message: string;
}
export const checkClusterRoutingAllocationEnabledTask =
@ -53,7 +54,11 @@ export const checkClusterRoutingAllocationEnabledTask =
[...clusterRoutingAllocations].every((s: string) => s === 'all'); // if set, only allow 'all'
if (!clusterRoutingAllocationEnabled) {
return Either.left({ type: 'unsupported_cluster_routing_allocation' as const });
return Either.left({
type: 'unsupported_cluster_routing_allocation' as const,
message:
'[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.',
});
} else {
return Either.right({});
}

279
src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts
View file

@ -167,6 +167,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
},
}
@ -187,6 +188,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
},
}
@ -207,6 +209,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
},
}
@ -395,14 +398,14 @@ describe('migration actions', () => {
timeout: '1s',
});
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"message": "[index_not_yellow_timeout] Timeout waiting for the status of the [red_index] index to become 'yellow'",
"type": "index_not_yellow_timeout",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"message": "[index_not_yellow_timeout] Timeout waiting for the status of the [red_index] index to become 'yellow'",
"type": "index_not_yellow_timeout",
},
}
`);
});
});
@ -422,14 +425,14 @@ describe('migration actions', () => {
});
expect.assertions(1);
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": Object {
"acknowledged": true,
"shardsAcknowledged": true,
},
}
`);
Object {
"_tag": "Right",
"right": Object {
"acknowledged": true,
"shardsAcknowledged": true,
},
}
`);
});
it('resolves right after waiting for index status to be yellow if clone target already existed', async () => {
expect.assertions(2);
@ -488,14 +491,14 @@ describe('migration actions', () => {
expect.assertions(1);
const task = cloneIndex({ client, source: 'no_such_index', target: 'clone_target_3' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"index": "no_such_index",
"type": "index_not_found_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"index": "no_such_index",
"type": "index_not_found_exception",
},
}
`);
});
it('resolves left with a index_not_yellow_timeout if clone target already exists but takes longer than the specified timeout before turning yellow', async () => {
// Create a red index
@ -524,14 +527,14 @@ describe('migration actions', () => {
})();
await expect(cloneIndexPromise).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"message": "[index_not_yellow_timeout] Timeout waiting for the status of the [clone_red_index] index to become 'yellow'",
"type": "index_not_yellow_timeout",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"message": "[index_not_yellow_timeout] Timeout waiting for the status of the [clone_red_index] index to become 'yellow'",
"type": "index_not_yellow_timeout",
},
}
`);
// Now that we know timeouts work, make the index yellow again and call cloneIndex a second time to verify that it completes
@ -552,14 +555,14 @@ describe('migration actions', () => {
})();
await expect(cloneIndexPromise2).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": Object {
"acknowledged": true,
"shardsAcknowledged": true,
},
}
`);
Object {
"_tag": "Right",
"right": Object {
"acknowledged": true,
"shardsAcknowledged": true,
},
}
`);
});
});
@ -577,11 +580,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
const results = (
(await searchForOutdatedDocuments(client, {
@ -617,11 +620,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
const results = (
(await searchForOutdatedDocuments(client, {
@ -650,11 +653,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
const results = (
(await searchForOutdatedDocuments(client, {
batchSize: 1000,
@ -685,11 +688,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
let task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
// reindex without a script
res = (await reindex({
@ -702,11 +705,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
// Assert that documents weren't overridden by the second, unscripted reindex
const results = (
@ -761,11 +764,11 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "reindex_succeeded",
}
`);
// Assert that existing documents weren't overridden, but that missing
// documents were added by the reindex
const results = (
@ -818,13 +821,13 @@ describe('migration actions', () => {
const task = waitForReindexTask({ client, taskId: reindexTaskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "incompatible_mapping_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "incompatible_mapping_exception",
},
}
`);
});
it('resolves left incompatible_mapping_exception if all reindex failures are due to a mapper_parsing_exception', async () => {
expect.assertions(1);
@ -857,13 +860,13 @@ describe('migration actions', () => {
const task = waitForReindexTask({ client, taskId: reindexTaskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "incompatible_mapping_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "incompatible_mapping_exception",
},
}
`);
});
it('resolves left index_not_found_exception if source index does not exist', async () => {
expect.assertions(1);
@ -879,14 +882,14 @@ describe('migration actions', () => {
})()) as Either.Right<ReindexResponse>;
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"index": "no_such_index",
"type": "index_not_found_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"index": "no_such_index",
"type": "index_not_found_exception",
},
}
`);
});
it('resolves left target_index_had_write_block if all failures are due to a write block', async () => {
expect.assertions(1);
@ -902,13 +905,13 @@ describe('migration actions', () => {
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "target_index_had_write_block",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "target_index_had_write_block",
},
}
`);
});
it('resolves left if requireAlias=true and the target is not an alias', async () => {
expect.assertions(1);
@ -924,14 +927,14 @@ describe('migration actions', () => {
const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' });
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"index": "existing_index_with_write_block",
"type": "index_not_found_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"index": "existing_index_with_write_block",
"type": "index_not_found_exception",
},
}
`);
});
it('resolves left wait_for_task_completion_timeout when the task does not finish within the timeout', async () => {
@ -983,11 +986,11 @@ describe('migration actions', () => {
targetIndex: 'reindex_target_7',
});
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "verify_reindex_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "verify_reindex_succeeded",
}
`);
});
it('resolves left if source and target indices have different amount of documents', async () => {
expect.assertions(1);
@ -997,13 +1000,13 @@ describe('migration actions', () => {
targetIndex: 'existing_index_2',
});
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "verify_reindex_failed",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "verify_reindex_failed",
},
}
`);
});
it('rejects if source or target index does not exist', async () => {
expect.assertions(2);
@ -1630,11 +1633,11 @@ describe('migration actions', () => {
});
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Right",
"right": "bulk_index_succeeded",
}
`);
Object {
"_tag": "Right",
"right": "bulk_index_succeeded",
}
`);
});
it('resolves right even if there were some version_conflict_engine_exception', async () => {
const existingDocs = (
@ -1675,13 +1678,13 @@ describe('migration actions', () => {
refresh: 'wait_for',
})()
).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "target_index_had_write_block",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "target_index_had_write_block",
},
}
`);
});
it('resolves left request_entity_too_large_exception when the payload is too large', async () => {
@ -1697,13 +1700,13 @@ describe('migration actions', () => {
transformedDocs: newDocs,
});
await expect(task()).resolves.toMatchInlineSnapshot(`
Object {
"_tag": "Left",
"left": Object {
"type": "request_entity_too_large_exception",
},
}
`);
Object {
"_tag": "Left",
"left": Object {
"type": "request_entity_too_large_exception",
},
}
`);
});
});
});

4
src/core/server/saved_objects/migrations/initial_state.test.ts
View file

@ -116,6 +116,10 @@ describe('createInitialState', () => {
migrationDocLinks: {
resolveMigrationFailures:
'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html',
repeatedTimeoutRequests:
'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail',
routingAllocationDisabled:
'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled',
},
});
});

6
src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts
View file

@ -114,7 +114,7 @@ describe('unsupported_cluster_routing_allocation', () => {
await root.setup();
await expect(root.start()).rejects.toThrowError(
/Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {"transient": {"cluster\.routing\.allocation\.enable": null}, "persistent": {"cluster\.routing\.allocation\.enable": null}}/
/Unable to complete saved object migrations for the \[\.kibana\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./
);
await retryAsync(
@ -126,7 +126,7 @@ describe('unsupported_cluster_routing_allocation', () => {
.map((str) => JSON5.parse(str)) as LogRecord[];
expect(
records.find((rec) =>
/^Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\./.test(
/^Unable to complete saved object migrations for the \[\.kibana.*\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\./.test(
rec.message
)
)
@ -149,7 +149,7 @@ describe('unsupported_cluster_routing_allocation', () => {
await root.setup();
await expect(root.start()).rejects.toThrowError(
/Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {"transient": {"cluster\.routing\.allocation\.enable": null}, "persistent": {"cluster\.routing\.allocation\.enable": null}}/
/Unable to complete saved object migrations for the \[\.kibana\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./
);
});
});

35
src/core/server/saved_objects/migrations/model/extract_errors.test.ts
View file

@ -6,7 +6,11 @@
* Side Public License, v 1.
*/
import { extractUnknownDocFailureReason } from './extract_errors';
import {
extractUnknownDocFailureReason,
fatalReasonClusterRoutingAllocationUnsupported,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
} from './extract_errors';
describe('extractUnknownDocFailureReason', () => {
it('generates the correct error message', () => {
@ -37,3 +41,32 @@ describe('extractUnknownDocFailureReason', () => {
`);
});
});
describe('fatalReasonDocumentExceedsMaxBatchSizeBytes', () => {
it('generate the correct error message', () => {
expect(
fatalReasonDocumentExceedsMaxBatchSizeBytes({
_id: 'abc',
docSizeBytes: 106954752,
maxBatchSizeBytes: 104857600,
})
).toMatchInlineSnapshot(
`"The document with _id \\"abc\\" is 106954752 bytes which exceeds the configured maximum batch size of 104857600 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value."`
);
});
});
describe('fatalReasonClusterRoutingAllocationUnsupported', () => {
it('generates the correct error message', () => {
const errorMessages = fatalReasonClusterRoutingAllocationUnsupported({
errorMessage: '[some-error] message',
docSectionLink: 'linkToDocsSection',
});
expect(errorMessages.fatalReason).toMatchInlineSnapshot(
`"[some-error] message To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to linkToDocsSection for more information on how to resolve the issue."`
);
expect(errorMessages.logsErrorMessage).toMatchInlineSnapshot(
`"[some-error] message Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to linkToDocsSection for more information on how to resolve the issue."`
);
});
});

29
src/core/server/saved_objects/migrations/model/extract_errors.ts
View file

@ -51,3 +51,32 @@ export function extractUnknownDocFailureReason(
`'`
);
}
/**
* Constructs migration failure message string for doc exceeds max batch size in bytes
*/
export const fatalReasonDocumentExceedsMaxBatchSizeBytes = ({
_id,
docSizeBytes,
maxBatchSizeBytes,
}: {
_id: string;
docSizeBytes: number;
maxBatchSizeBytes: number;
}) =>
`The document with _id "${_id}" is ${docSizeBytes} bytes which exceeds the configured maximum batch size of ${maxBatchSizeBytes} bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`;
/**
* Constructs migration failure message and logs message strings when an unsupported cluster routing allocation is configured.
* The full errorMessage is "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue."
*/
export const fatalReasonClusterRoutingAllocationUnsupported = ({
errorMessage,
docSectionLink,
}: {
errorMessage: string;
docSectionLink: string;
}) => ({
fatalReason: `${errorMessage} To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}. Refer to ${docSectionLink} for more information on how to resolve the issue.`,
logsErrorMessage: `${errorMessage} Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to ${docSectionLink} for more information on how to resolve the issue.`,
});

5
src/core/server/saved_objects/migrations/model/model.test.ts
View file

@ -96,6 +96,8 @@ describe('migrations v2 model', () => {
excludeFromUpgradeFilterHooks: {},
migrationDocLinks: {
resolveMigrationFailures: 'resolveMigrationFailures',
repeatedTimeoutRequests: 'repeatedTimeoutRequests',
routingAllocationDisabled: 'routingAllocationDisabled',
},
};
@ -283,12 +285,13 @@ describe('migrations v2 model', () => {
test('INIT -> FATAL when cluster routing allocation is not enabled', () => {
const res: ResponseType<'INIT'> = Either.left({
type: 'unsupported_cluster_routing_allocation',
message: '[unsupported_cluster_routing_allocation]',
});
const newState = model(initState, res) as FatalState;
expect(newState.controlState).toEqual('FATAL');
expect(newState.reason).toMatchInlineSnapshot(
`"The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}"`
`"[unsupported_cluster_routing_allocation] To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to routingAllocationDisabled for more information on how to resolve the issue."`
);
});
test("INIT -> FATAL when .kibana points to newer version's index", () => {

37
src/core/server/saved_objects/migrations/model/model.ts
View file

@ -21,7 +21,12 @@ import {
setProgressTotal,
} from './progress';
import { delayRetryState, resetRetryState } from './retry_state';
import { extractTransformFailuresReason, extractUnknownDocFailureReason } from './extract_errors';
import {
extractTransformFailuresReason,
extractUnknownDocFailureReason,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
fatalReasonClusterRoutingAllocationUnsupported,
} from './extract_errors';
import type { ExcludeRetryableEsError } from './types';
import {
getAliases,
@ -33,17 +38,7 @@ import {
} from './helpers';
import { createBatches } from './create_batches';
const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`;
const fatalReasonDocumentExceedsMaxBatchSizeBytes = ({
_id,
docSizeBytes,
maxBatchSizeBytes,
}: {
_id: string;
docSizeBytes: number;
maxBatchSizeBytes: number;
}) =>
`The document with _id "${_id}" is ${docSizeBytes} bytes which exceeds the configured maximum batch size of ${maxBatchSizeBytes} bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`;
export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`;
export const model = (currentState: State, resW: ResponseType<AllActionStates>): State => {
// The action response `resW` is weakly typed, the type includes all action
@ -73,15 +68,19 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
if (Either.isLeft(res)) {
const left = res.left;
if (isLeftTypeof(left, 'unsupported_cluster_routing_allocation')) {
const initErrorMessages = fatalReasonClusterRoutingAllocationUnsupported({
errorMessage: left.message,
docSectionLink: stateP.migrationDocLinks.routingAllocationDisabled,
});
return {
...stateP,
controlState: 'FATAL',
reason: `The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}`,
reason: initErrorMessages.fatalReason,
logs: [
...stateP.logs,
{
level: 'error',
message: `The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'.`,
message: initErrorMessages.logsErrorMessage,
},
],
};
@ -244,7 +243,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// we get a response.
// If the cluster hit the low watermark for disk usage the LEGACY_CREATE_REINDEX_TARGET action will
// continue to timeout and eventually lead to a failed migration.
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`;
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
return throwBadResponse(stateP, left);
@ -366,7 +365,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// we get a response.
// In the event of retries running out, we link to the docs to help with diagnosing
// the problem.
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`;
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
return throwBadResponse(stateP, left);
@ -461,7 +460,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
//
// If there is a problem CREATE_REINDEX_TEMP action will
// continue to timeout and eventually lead to a failed migration.
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`;
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
return throwBadResponse(stateP, left);
@ -696,7 +695,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// `_cluster/allocation/explain?index=${targetIndex}` API.
// Unless the root cause is identified and addressed, the request will
// continue to timeout and eventually lead to a failed migration.
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`;
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
throwBadResponse(stateP, left);
@ -951,7 +950,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
// If the cluster hit the low watermark for disk usage the action will continue to timeout.
// Unless the disk space is addressed, the LEGACY_CREATE_REINDEX_TARGET action will
// continue to timeout and eventually lead to a failed migration.
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`;
const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
return throwBadResponse(stateP, left);

3
src/core/server/saved_objects/migrations/state.ts
View file

@ -8,6 +8,7 @@
import * as Option from 'fp-ts/lib/Option';
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { DocLinks } from '@kbn/doc-links';
import { ControlState } from './state_action_machine';
import { AliasAction } from './actions';
import { IndexMapping } from '../mappings';
@ -125,7 +126,7 @@ export interface BaseState extends ControlState {
/**
* DocLinks for savedObjects. to reference online documentation
*/
readonly migrationDocLinks: Record<string, string>;
readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects'];
}
export interface InitState extends BaseState {