github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Retry INIT step when hitting incompatible_cluster_routing_allocation (#131809)

Browse source 
* Add reproducing test case

* Fix and add integration test

* Transient settings should take preference

* Rename unsupported_cluster_routing_allocation error to incompatible_cluster_routing_allocation

* Retry INIT when action fails with [incompatible_cluster_routing_allocation]

* Apply suggestions from code review

Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>

* Fix archive with trial licence and re-enable skipped test

* Integration test for incompatible cluster routing allocation

* Fix types after renaming UnsupportedClusterRoutingAllocation

* Attempt to fix open handle tests

Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>
This commit is contained in:
Rudolf Meijering 2022-05-11 17:05:01 +02:00 • committed by GitHub
parent edb001d36d
commit 575c5599a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 92 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/setup/upgrade/resolving-migration-failures.asciidoc
View file

@ -171,7 +171,7 @@ Upgrade migrations fail because routing allocation is disabled or restricted (`c
[source,sh]
--------------------------------------------
Unable to complete saved object migrations for the [.kibana] index: [unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
Unable to complete saved object migrations for the [.kibana] index: [incompatible_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
--------------------------------------------
To get around the issue, remove the transient and persisted routing allocation settings:

8
src/core/server/saved_objects/migrations/README.md
View file

@ -149,8 +149,12 @@ index.
### New control state
1. Two conditions have to be met before migrations begin:
1. If replica allocation is set as a persistent or transient setting to "perimaries", "new_primaries" or "none" fail the migration. Without replica allocation enabled or not set to 'all', the migration will timeout when waiting for index yellow status before bulk indexing. The check only considers persistent and transient settings and does not take static configuration in `elasticsearch.yml` into account. If `cluster.routing.allocation.enable` is configured in `elaticsearch.yml` and not set to the default of 'all', the migration will timeout. Static settings can only be returned from the `nodes/info` API.
→ `FATAL`
1. The Elasticsearch shard allocation cluster setting `cluster.routing.allocation.enable` needs to be unset or set to 'all'. When set to 'primaries', 'new_primaries' or 'none', the migration will timeout when waiting for index yellow status before bulk indexing because the replica cannot be allocated.
As per the Elasticsearch docs https://www.elastic.co/guide/en/elasticsearch/reference/8.2/restart-cluster.html#restart-cluster-rolling when Cloud performs a rolling restart such as during an upgrade, it will temporarily disable shard allocation. Kibana therefore keeps retrying the INIT step to wait for shard allocation to be enabled again.
The check only considers persistent and transient settings and does not take static configuration in `elasticsearch.yml` into account since there are no known use cases for doing so. If `cluster.routing.allocation.enable` is configured in `elaticsearch.yml` and not set to the default of 'all', the migration will timeout. Static settings can only be returned from the `nodes/info` API.
→ `INIT`
2. If `.kibana` is pointing to an index that belongs to a later version of
Kibana .e.g. a 7.11.0 instance found the `.kibana` alias pointing to

6
src/core/server/saved_objects/migrations/actions/index.ts
View file

@ -20,7 +20,7 @@ export {
export type { RetryableEsClientError };
// actions/* imports
export type { InitActionParams, UnsupportedClusterRoutingAllocation } from './initialize_action';
export type { InitActionParams, IncompatibleClusterRoutingAllocation } from './initialize_action';
export { initAction } from './initialize_action';
export type { FetchIndexResponse, FetchIndicesParams } from './fetch_indices';
@ -87,7 +87,7 @@ export type {
export { updateAndPickupMappings } from './update_and_pickup_mappings';
import type { UnknownDocsFound } from './check_for_unknown_docs';
import type { UnsupportedClusterRoutingAllocation } from './initialize_action';
import type { IncompatibleClusterRoutingAllocation } from './initialize_action';
export type {
CheckForUnknownDocsParams,
@ -151,7 +151,7 @@ export interface ActionErrorTypeMap {
documents_transform_failed: DocumentsTransformFailed;
request_entity_too_large_exception: RequestEntityTooLargeException;
unknown_docs_found: UnknownDocsFound;
unsupported_cluster_routing_allocation: UnsupportedClusterRoutingAllocation;
incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation;
index_not_yellow_timeout: IndexNotYellowTimeout;
}

13
src/core/server/saved_objects/migrations/actions/initialize_action.ts
View file

@ -27,9 +27,8 @@ export interface InitActionParams {
indices: string[];
}
export interface UnsupportedClusterRoutingAllocation {
type: 'unsupported_cluster_routing_allocation';
message: string;
export interface IncompatibleClusterRoutingAllocation {
type: 'incompatible_cluster_routing_allocation';
}
export const checkClusterRoutingAllocationEnabledTask =
@ -37,7 +36,7 @@ export const checkClusterRoutingAllocationEnabledTask =
client,
}: {
client: ElasticsearchClient;
}): TaskEither.TaskEither<RetryableEsClientError | UnsupportedClusterRoutingAllocation, {}> =>
}): TaskEither.TaskEither<RetryableEsClientError | IncompatibleClusterRoutingAllocation, {}> =>
() => {
return client.cluster
.getSettings({
@ -54,9 +53,7 @@ export const checkClusterRoutingAllocationEnabledTask =
if (!clusterRoutingAllocationEnabledIsAll) {
return Either.left({
type: 'unsupported_cluster_routing_allocation' as const,
message:
'[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.',
type: 'incompatible_cluster_routing_allocation' as const,
});
} else {
return Either.right({});
@ -69,7 +66,7 @@ export const initAction = ({
client,
indices,
}: InitActionParams): TaskEither.TaskEither<
RetryableEsClientError | UnsupportedClusterRoutingAllocation,
RetryableEsClientError | IncompatibleClusterRoutingAllocation,
FetchIndexResponse
> => {
return pipe(

9
src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts
View file

@ -167,8 +167,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
"type": "incompatible_cluster_routing_allocation",
},
}
`);
@ -188,8 +187,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
"type": "incompatible_cluster_routing_allocation",
},
}
`);
@ -209,8 +207,7 @@ describe('migration actions', () => {
Object {
"_tag": "Left",
"left": Object {
"message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.",
"type": "unsupported_cluster_routing_allocation",
"type": "incompatible_cluster_routing_allocation",
},
}
`);

BIN
src/core/server/saved_objects/migrations/integration_tests/archives/8.0.0_migrated_with_outdated_docs.zip
View file

Binary file not shown.

59
src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts → src/core/server/saved_objects/migrations/integration_tests/incompatible_cluster_routing_allocation.test.ts
View file

@ -15,7 +15,7 @@ import { ElasticsearchClient } from '../../../elasticsearch';
import { LogRecord } from '@kbn/logging';
import { retryAsync } from '../test_helpers/retry_async';
const logFilePath = Path.join(__dirname, 'unsupported_cluster_routing_allocation.log');
const logFilePath = Path.join(__dirname, 'incompatible_cluster_routing_allocation.log');
async function removeLogFile() {
// ignore errors if it doesn't exist
@ -27,7 +27,11 @@ const { startES } = kbnTestServer.createTestServers({
settings: {
es: {
license: 'basic',
dataArchive: Path.join(__dirname, 'archives', '7.7.2_xpack_100k_obj.zip'),
dataArchive: Path.join(
__dirname,
'archives',
'8.0.0_v1_migrations_sample_data_saved_objects.zip'
),
},
},
});
@ -77,14 +81,14 @@ let esServer: kbnTestServer.TestElasticsearchUtils;
async function updateRoutingAllocations(
esClient: ElasticsearchClient,
settingType: string = 'persistent',
value: string = 'none'
value: string | null
) {
return await esClient.cluster.putSettings({
[settingType]: { cluster: { routing: { allocation: { enable: value } } } },
});
}
describe('unsupported_cluster_routing_allocation', () => {
describe('incompatible_cluster_routing_allocation', () => {
let client: ElasticsearchClient;
let root: Root;
@ -97,7 +101,7 @@ describe('unsupported_cluster_routing_allocation', () => {
await esServer.stop();
});
it('fails with a descriptive message when persistent replica allocation is not enabled', async () => {
it('retries the INIT action with a descriptive message when cluster settings are incompatible', async () => {
const initialSettings = await client.cluster.getSettings({ flat_settings: true });
expect(getClusterRoutingAllocations(initialSettings)).toBe(true);
@ -108,15 +112,14 @@ describe('unsupported_cluster_routing_allocation', () => {
expect(getClusterRoutingAllocations(updatedSettings)).toBe(false);
// now try to start Kibana
// Start Kibana
root = createKbnRoot();
await root.preboot();
await root.setup();
await expect(root.start()).rejects.toThrowError(
/Unable to complete saved object migrations for the \[\.kibana.*\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./
);
root.start();
// Wait for the INIT -> INIT action retry
await retryAsync(
async () => {
const logFileContent = await fs.readFile(logFilePath, 'utf-8');
@ -124,32 +127,38 @@ describe('unsupported_cluster_routing_allocation', () => {
.split('\n')
.filter(Boolean)
.map((str) => JSON5.parse(str)) as LogRecord[];
// Wait for logs of the second failed attempt to be sure we're correctly incrementing retries
expect(
records.find((rec) =>
/^Unable to complete saved object migrations for the \[\.kibana.*\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\./.test(
rec.message
rec.message.includes(
`Action failed with '[incompatible_cluster_routing_allocation] Incompatible Elasticsearch cluster settings detected. Remove the persistent and transient Elasticsearch cluster setting 'cluster.routing.allocation.enable' or set it to a value of 'all' to allow migrations to proceed. Refer to https://www.elastic.co/guide/en/kibana/master/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue.'. Retrying attempt 2 in 4 seconds.`
)
)
).toBeDefined();
},
{ retryAttempts: 10, retryDelayMs: 200 }
{ retryAttempts: 20, retryDelayMs: 500 }
);
});
it('fails with a descriptive message when persistent replica allocation is set to "primaries"', async () => {
await updateRoutingAllocations(client, 'persistent', 'primaries');
// Reset the cluster routing allocation settings
await updateRoutingAllocations(client, 'persistent', null);
const updatedSettings = await client.cluster.getSettings({ flat_settings: true });
// Wait for migrations to succeed
await retryAsync(
async () => {
const logFileContent = await fs.readFile(logFilePath, 'utf-8');
const records = logFileContent
.split('\n')
.filter(Boolean)
.map((str) => JSON5.parse(str)) as LogRecord[];
expect(getClusterRoutingAllocations(updatedSettings)).toBe(false);
// now try to start Kibana
root = createKbnRoot();
await root.preboot();
await root.setup();
await expect(root.start()).rejects.toThrowError(
/Unable to complete saved object migrations for the \[\.kibana.*\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./
expect(
records.find((rec) => rec.message.includes('MARK_VERSION_INDEX_READY -> DONE'))
).toBeDefined();
},
{ retryAttempts: 100, retryDelayMs: 500 }
);
await root.shutdown();
});
});

4
src/core/server/saved_objects/migrations/integration_tests/outdated_docs.test.ts
View file

@ -41,7 +41,7 @@ describe('migration v2', () => {
await new Promise((resolve) => setTimeout(resolve, 10000));
});
it.skip('migrates the documents to the highest version', async () => {
it('migrates the documents to the highest version', async () => {
const migratedIndex = `.kibana_${pkg.version}_001`;
const { startES } = kbnTestServer.createTestServers({
adjustTimeout: (t: number) => jest.setTimeout(t),
@ -86,7 +86,7 @@ describe('migration v2', () => {
expect(migratedDocs.length).toBe(1);
const [doc] = migratedDocs;
expect(doc._source.migrationVersion.foo).toBe('7.14.0');
expect(doc._source.coreMigrationVersion).toBe('8.0.0');
expect(doc._source.coreMigrationVersion).toBe(pkg.version);
});
});

16
src/core/server/saved_objects/migrations/model/extract_errors.test.ts
View file

@ -8,7 +8,6 @@
import {
extractUnknownDocFailureReason,
fatalReasonClusterRoutingAllocationUnsupported,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
} from './extract_errors';
@ -55,18 +54,3 @@ describe('fatalReasonDocumentExceedsMaxBatchSizeBytes', () => {
);
});
});
describe('fatalReasonClusterRoutingAllocationUnsupported', () => {
it('generates the correct error message', () => {
const errorMessages = fatalReasonClusterRoutingAllocationUnsupported({
errorMessage: '[some-error] message',
docSectionLink: 'linkToDocsSection',
});
expect(errorMessages.fatalReason).toMatchInlineSnapshot(
`"[some-error] message To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to linkToDocsSection for more information on how to resolve the issue."`
);
expect(errorMessages.logsErrorMessage).toMatchInlineSnapshot(
`"[some-error] message Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to linkToDocsSection for more information on how to resolve the issue."`
);
});
});

15
src/core/server/saved_objects/migrations/model/extract_errors.ts
View file

@ -65,18 +65,3 @@ export const fatalReasonDocumentExceedsMaxBatchSizeBytes = ({
maxBatchSizeBytes: number;
}) =>
`The document with _id "${_id}" is ${docSizeBytes} bytes which exceeds the configured maximum batch size of ${maxBatchSizeBytes} bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`;
/**
* Constructs migration failure message and logs message strings when an unsupported cluster routing allocation is configured.
* The full errorMessage is "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue."
*/
export const fatalReasonClusterRoutingAllocationUnsupported = ({
errorMessage,
docSectionLink,
}: {
errorMessage: string;
docSectionLink: string;
}) => ({
fatalReason: `${errorMessage} To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}. Refer to ${docSectionLink} for more information on how to resolve the issue.`,
logsErrorMessage: `${errorMessage} Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to ${docSectionLink} for more information on how to resolve the issue.`,
});

42
src/core/server/saved_objects/migrations/model/model.test.ts
View file

@ -282,17 +282,21 @@ describe('migrations v2 model', () => {
expect(newState.retryCount).toEqual(0);
expect(newState.retryDelay).toEqual(0);
});
test('INIT -> FATAL when cluster routing allocation is not enabled', () => {
test('INIT -> INIT when cluster routing allocation is incompatible', () => {
const res: ResponseType<'INIT'> = Either.left({
type: 'unsupported_cluster_routing_allocation',
message: '[unsupported_cluster_routing_allocation]',
type: 'incompatible_cluster_routing_allocation',
});
const newState = model(initState, res) as FatalState;
expect(newState.controlState).toEqual('FATAL');
expect(newState.reason).toMatchInlineSnapshot(
`"[unsupported_cluster_routing_allocation] To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to routingAllocationDisabled for more information on how to resolve the issue."`
);
expect(newState.controlState).toEqual('INIT');
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "error",
"message": "Action failed with '[incompatible_cluster_routing_allocation] Incompatible Elasticsearch cluster settings detected. Remove the persistent and transient Elasticsearch cluster setting 'cluster.routing.allocation.enable' or set it to a value of 'all' to allow migrations to proceed. Refer to routingAllocationDisabled for more information on how to resolve the issue.'. Retrying attempt 1 in 2 seconds.",
}
`);
});
test("INIT -> FATAL when .kibana points to newer version's index", () => {
const res: ResponseType<'INIT'> = Either.right({
@ -575,6 +579,12 @@ describe('migrations v2 model', () => {
expect(newState.controlState).toEqual('LEGACY_CREATE_REINDEX_TARGET');
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "error",
"message": "Action failed with '[index_not_yellow_timeout] Timeout waiting for ... Refer to repeatedTimeoutRequests for information on how to resolve the issue.'. Retrying attempt 1 in 2 seconds.",
}
`);
});
test('LEGACY_CREATE_REINDEX_TARGET -> LEGACY_REINDEX resets retry count and retry delay if action succeeds', () => {
const res: ResponseType<'LEGACY_CREATE_REINDEX_TARGET'> =
@ -743,6 +753,12 @@ describe('migrations v2 model', () => {
expect(newState.controlState).toEqual('WAIT_FOR_YELLOW_SOURCE');
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "error",
"message": "Action failed with '[index_not_yellow_timeout] Timeout waiting for ... Refer to repeatedTimeoutRequests for information on how to resolve the issue.'. Retrying attempt 1 in 2 seconds.",
}
`);
});
test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS resets retry count and delay if action succeeds', () => {
@ -962,6 +978,12 @@ describe('migrations v2 model', () => {
expect(newState.controlState).toEqual('CREATE_REINDEX_TEMP');
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "error",
"message": "Action failed with '[index_not_yellow_timeout] Timeout waiting for ... Refer to repeatedTimeoutRequests for information on how to resolve the issue.'. Retrying attempt 1 in 2 seconds.",
}
`);
});
it('CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT resets retry count if action succeeds', () => {
const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded');
@ -1296,6 +1318,12 @@ describe('migrations v2 model', () => {
expect(newState.controlState).toEqual('CLONE_TEMP_TO_TARGET');
expect(newState.retryCount).toEqual(1);
expect(newState.retryDelay).toEqual(2000);
expect(newState.logs[0]).toMatchInlineSnapshot(`
Object {
"level": "error",
"message": "Action failed with '[index_not_yellow_timeout] Timeout waiting for ... Refer to repeatedTimeoutRequests for information on how to resolve the issue.'. Retrying attempt 1 in 2 seconds.",
}
`);
});
it('CREATE_NEW_TARGET -> MARK_VERSION_INDEX_READY resets the retry count and delay', () => {
const res: ResponseType<'CLONE_TEMP_TO_TARGET'> = Either.right({

21
src/core/server/saved_objects/migrations/model/model.ts
View file

@ -25,7 +25,6 @@ import {
extractTransformFailuresReason,
extractUnknownDocFailureReason,
fatalReasonDocumentExceedsMaxBatchSizeBytes,
fatalReasonClusterRoutingAllocationUnsupported,
} from './extract_errors';
import type { ExcludeRetryableEsError } from './types';
import {
@ -67,23 +66,9 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
if (Either.isLeft(res)) {
const left = res.left;
if (isLeftTypeof(left, 'unsupported_cluster_routing_allocation')) {
const initErrorMessages = fatalReasonClusterRoutingAllocationUnsupported({
errorMessage: left.message,
docSectionLink: stateP.migrationDocLinks.routingAllocationDisabled,
});
return {
...stateP,
controlState: 'FATAL',
reason: initErrorMessages.fatalReason,
logs: [
...stateP.logs,
{
level: 'error',
message: initErrorMessages.logsErrorMessage,
},
],
};
if (isLeftTypeof(left, 'incompatible_cluster_routing_allocation')) {
const retryErrorMessage = `[${left.type}] Incompatible Elasticsearch cluster settings detected. Remove the persistent and transient Elasticsearch cluster setting 'cluster.routing.allocation.enable' or set it to a value of 'all' to allow migrations to proceed. Refer to ${stateP.migrationDocLinks.routingAllocationDisabled} for more information on how to resolve the issue.`;
return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts);
} else {
return throwBadResponse(stateP, left);
}