mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
Prevent rollback for failed upgrades from write_blocking SO indices (#158725)
Tackles https://github.com/elastic/kibana/issues/155136 When an upgrade fails, a cluster might be on a partially migrated state (with some indices already updated to the newer version). When rolling back to the previous version, in ESS, this can cause these indices to be `write_blocked`. This PR aims at detecting this situation and failing early, effectively preventing to `write_block` any indices.
This commit is contained in:
Gerard Soldevila
GitHub
parent
051ac85c07
commit
1c5b09dd06
6 changed files with 221 additions and 10 deletions
|
|
@ -18,7 +18,6 @@ import type { Logger } from '@kbn/logging';
|
|||
import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
type SavedObjectUnsanitizedDoc,
|
||||
type SavedObjectsRawDoc,
|
||||
type ISavedObjectTypeRegistry,
|
||||
|
|
@ -197,7 +196,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
// compare indexTypesMap with the one present (or not) in the .kibana index meta
|
||||
// and check if some SO types have been moved to different indices
|
||||
const indicesWithMovingTypes = await getIndicesInvolvedInRelocation({
|
||||
mainIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
mainIndex: this.kibanaIndex,
|
||||
client: this.client,
|
||||
indexTypesMap,
|
||||
logger: this.log,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ import {
|
|||
MigrationType,
|
||||
getTempIndexName,
|
||||
createBulkIndexOperationTuple,
|
||||
hasLaterVersionAlias,
|
||||
aliasVersion,
|
||||
} from './helpers';
|
||||
|
||||
describe('addExcludedTypesToBoolQuery', () => {
|
||||
|
|
@ -183,6 +185,24 @@ describe('addMustNotClausesToBoolQuery', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('aliasVersion', () => {
|
||||
test('empty', () => {
|
||||
expect(aliasVersion(undefined)).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('not a version alias', () => {
|
||||
expect(aliasVersion('.kibana')).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('supports arbitrary names and versions', () => {
|
||||
expect(aliasVersion('.kibana_task_manager_7.17.0')).toEqual('7.17.0');
|
||||
});
|
||||
|
||||
test('supports index names too', () => {
|
||||
expect(aliasVersion('.kibana_8.8.0_001')).toEqual('8.8.0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAliases', () => {
|
||||
it('returns a right record of alias to index name pairs', () => {
|
||||
const indices: FetchIndexResponse = {
|
||||
|
|
@ -273,6 +293,76 @@ describe('versionMigrationCompleted', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('hasLaterVersionAlias', () => {
|
||||
test('undefined', () => {
|
||||
expect(hasLaterVersionAlias('8.8.0', undefined)).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('empty', () => {
|
||||
expect(hasLaterVersionAlias('8.8.0', {})).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('only previous version alias', () => {
|
||||
expect(
|
||||
hasLaterVersionAlias('8.8.0', {
|
||||
'.kibana_7.17.0': '.kibana_7.17.0_001',
|
||||
'.kibana_8.6.0': '.kibana_8.6.0_001',
|
||||
'.kibana_8.7.2': '.kibana_8.7.2_001',
|
||||
})
|
||||
).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('current version alias', () => {
|
||||
expect(
|
||||
hasLaterVersionAlias('8.8.0', {
|
||||
'.kibana_7.17.0': '.kibana_7.17.0_001',
|
||||
'.kibana_8.6.0': '.kibana_8.6.0_001',
|
||||
'.kibana_8.7.2': '.kibana_8.7.2_001',
|
||||
'.kibana_8.8.0': '.kibana_8.8.0_001',
|
||||
})
|
||||
).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('next build alias', () => {
|
||||
expect(
|
||||
hasLaterVersionAlias('8.8.0', {
|
||||
'.kibana_7.17.0': '.kibana_7.17.0_001',
|
||||
'.kibana_8.6.0': '.kibana_8.6.0_001',
|
||||
'.kibana_8.7.2': '.kibana_8.7.2_001',
|
||||
'.kibana_8.8.0': '.kibana_8.8.0_001',
|
||||
'.kibana_8.8.1': '.kibana_8.8.0_001',
|
||||
})
|
||||
).toEqual('.kibana_8.8.1');
|
||||
});
|
||||
|
||||
test('next minor alias', () => {
|
||||
expect(
|
||||
hasLaterVersionAlias('8.8.1', {
|
||||
'.kibana_8.9.0': '.kibana_8.9.0_001',
|
||||
'.kibana_7.17.0': '.kibana_7.17.0_001',
|
||||
'.kibana_8.6.0': '.kibana_8.6.0_001',
|
||||
'.kibana_8.7.2': '.kibana_8.7.2_001',
|
||||
'.kibana_8.8.0': '.kibana_8.8.0_001',
|
||||
'.kibana_8.8.1': '.kibana_8.8.0_001',
|
||||
})
|
||||
).toEqual('.kibana_8.9.0');
|
||||
});
|
||||
|
||||
test('multiple future versions, return most recent alias', () => {
|
||||
expect(
|
||||
hasLaterVersionAlias('7.17.0', {
|
||||
'.kibana_8.9.0': '.kibana_8.9.0_001',
|
||||
'.kibana_8.9.1': '.kibana_8.9.0_001',
|
||||
'.kibana_7.17.0': '.kibana_7.17.0_001',
|
||||
'.kibana_8.6.0': '.kibana_8.6.0_001',
|
||||
'.kibana_8.7.2': '.kibana_8.7.2_001',
|
||||
'.kibana_8.8.0': '.kibana_8.8.0_001',
|
||||
'.kibana_8.8.1': '.kibana_8.8.0_001',
|
||||
})
|
||||
).toEqual('.kibana_8.9.1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildRemoveAliasActions', () => {
|
||||
test('empty', () => {
|
||||
expect(buildRemoveAliasActions('.kibana_test_123', [], [])).toEqual([]);
|
||||
|
|
|
|||
|
|
@ -93,6 +93,22 @@ export function indexBelongsToLaterVersion(kibanaVersion: string, indexName?: st
|
|||
return version != null ? gt(version, kibanaVersion) : false;
|
||||
}
|
||||
|
||||
export function hasLaterVersionAlias(
|
||||
kibanaVersion: string,
|
||||
aliases?: Partial<Record<string, string>>
|
||||
): string | undefined {
|
||||
const mostRecentAlias = Object.keys(aliases ?? {})
|
||||
.filter(aliasVersion)
|
||||
.sort()
|
||||
.pop();
|
||||
|
||||
const mostRecentAliasVersion = valid(aliasVersion(mostRecentAlias));
|
||||
|
||||
return mostRecentAliasVersion != null && gt(mostRecentAliasVersion, kibanaVersion)
|
||||
? mostRecentAlias
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add new must_not clauses to the given query
|
||||
* in order to filter out the specified types
|
||||
|
|
@ -170,6 +186,14 @@ export function indexVersion(indexName?: string): string | undefined {
|
|||
return (indexName?.match(/.+_(\d+\.\d+\.\d+)_\d+/) || [])[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the version number from a >= 7.11 index alias
|
||||
* @param indexName A >= v7.11 index alias
|
||||
*/
|
||||
export function aliasVersion(alias?: string): string | undefined {
|
||||
return (alias?.match(/.+_(\d+\.\d+\.\d+)/) || [])[1];
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface MultipleIndicesPerAlias {
|
||||
type: 'multiple_indices_per_alias';
|
||||
|
|
|
|||
|
|
@ -330,6 +330,21 @@ describe('migrations v2 model', () => {
|
|||
`"The .kibana alias is pointing to a newer version of Kibana: v7.12.0"`
|
||||
);
|
||||
});
|
||||
test('INIT -> FATAL when later version alias exists', () => {
|
||||
const res: ResponseType<'INIT'> = Either.right({
|
||||
'.kibana_7.11.0_001': {
|
||||
aliases: { '.kibana_7.12.0': {}, '.kibana': {} },
|
||||
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
|
||||
settings: {},
|
||||
},
|
||||
});
|
||||
const newState = model(initState, res) as FatalState;
|
||||
|
||||
expect(newState.controlState).toEqual('FATAL');
|
||||
expect(newState.reason).toMatchInlineSnapshot(
|
||||
`"The .kibana_7.12.0 alias refers to a newer version of Kibana: v7.12.0"`
|
||||
);
|
||||
});
|
||||
test('INIT -> FATAL when .kibana points to multiple indices', () => {
|
||||
const res: ResponseType<'INIT'> = Either.right({
|
||||
'.kibana_7.12.0_001': {
|
||||
|
|
@ -365,13 +380,13 @@ describe('migrations v2 model', () => {
|
|||
'.kibana_7.invalid.0_001': {
|
||||
aliases: {
|
||||
'.kibana': {},
|
||||
'.kibana_7.12.0': {},
|
||||
'.kibana_7.11.0': {},
|
||||
},
|
||||
mappings: mappingsWithUnknownType,
|
||||
settings: {},
|
||||
},
|
||||
'.kibana_7.11.0_001': {
|
||||
aliases: { '.kibana_7.11.0': {} },
|
||||
'.kibana_7.10.0_001': {
|
||||
aliases: { '.kibana_7.10.0': {} },
|
||||
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
|
||||
settings: {},
|
||||
},
|
||||
|
|
@ -571,13 +586,13 @@ describe('migrations v2 model', () => {
|
|||
'.kibana_7.invalid.0_001': {
|
||||
aliases: {
|
||||
'.kibana': {},
|
||||
'.kibana_7.12.0': {},
|
||||
'.kibana_7.11.0': {},
|
||||
},
|
||||
mappings: mappingsWithUnknownType,
|
||||
settings: {},
|
||||
},
|
||||
'.kibana_7.11.0_001': {
|
||||
aliases: { '.kibana_7.11.0': {} },
|
||||
'.kibana_7.10.0_001': {
|
||||
aliases: { '.kibana_7.10.0': {} },
|
||||
mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } },
|
||||
settings: {},
|
||||
},
|
||||
|
|
@ -588,8 +603,8 @@ describe('migrations v2 model', () => {
|
|||
expect(newState.sourceIndex.value).toBe('.kibana_7.invalid.0_001');
|
||||
expect(newState.aliases).toEqual({
|
||||
'.kibana': '.kibana_7.invalid.0_001',
|
||||
'.kibana_7.11.0': '.kibana_7.11.0_001',
|
||||
'.kibana_7.12.0': '.kibana_7.invalid.0_001',
|
||||
'.kibana_7.10.0': '.kibana_7.10.0_001',
|
||||
'.kibana_7.11.0': '.kibana_7.invalid.0_001',
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ import {
|
|||
buildRemoveAliasActions,
|
||||
MigrationType,
|
||||
increaseBatchSize,
|
||||
hasLaterVersionAlias,
|
||||
aliasVersion,
|
||||
} from './helpers';
|
||||
import { buildTempIndexMap, createBatches } from './create_batches';
|
||||
import type { MigrationLog } from '../types';
|
||||
|
|
@ -120,6 +122,22 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
};
|
||||
}
|
||||
|
||||
const laterVersionAlias = hasLaterVersionAlias(stateP.kibanaVersion, aliases);
|
||||
if (
|
||||
// `.kibana_<version>` alias exists, and refers to a later version of Kibana
|
||||
// e.g. `.kibana_8.7.0` exists, and current stack version is 8.6.1
|
||||
// see https://github.com/elastic/kibana/issues/155136
|
||||
laterVersionAlias
|
||||
) {
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'FATAL',
|
||||
reason: `The ${laterVersionAlias} alias refers to a newer version of Kibana: v${aliasVersion(
|
||||
laterVersionAlias
|
||||
)}`,
|
||||
};
|
||||
}
|
||||
|
||||
// The source index .kibana is pointing to. E.g: ".kibana_8.7.0_001"
|
||||
const source = aliases[stateP.currentAlias];
|
||||
// The target index .kibana WILL be pointing to if we reindex. E.g: ".kibana_8.8.0_001"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
|
||||
import {
|
||||
clearLog,
|
||||
startElasticsearch,
|
||||
getKibanaMigratorTestKit,
|
||||
nextMinor,
|
||||
defaultKibanaIndex,
|
||||
} from '../kibana_migrator_test_kit';
|
||||
import '../jest_matchers';
|
||||
import { delay, parseLogFile } from '../test_utils';
|
||||
import { baselineTypes as types } from '../kibana_migrator_test_kit.fixtures';
|
||||
|
||||
export const logFilePath = Path.join(__dirname, 'fail_on_rollback.test.log');
|
||||
|
||||
describe('when rolling back to an older version', () => {
|
||||
let esServer: TestElasticsearchUtils['es'];
|
||||
|
||||
beforeAll(async () => {
|
||||
esServer = await startElasticsearch();
|
||||
});
|
||||
|
||||
beforeEach(async () => {});
|
||||
|
||||
it('kibana should detect that a later version alias exists, and abort', async () => {
|
||||
// create a current version baseline
|
||||
const { runMigrations: createBaseline } = await getKibanaMigratorTestKit({
|
||||
types,
|
||||
logFilePath,
|
||||
});
|
||||
await createBaseline();
|
||||
|
||||
// migrate to next minor
|
||||
const { runMigrations: upgrade } = await getKibanaMigratorTestKit({
|
||||
kibanaVersion: nextMinor,
|
||||
types,
|
||||
logFilePath,
|
||||
});
|
||||
await upgrade();
|
||||
|
||||
// run migrations for the current version again (simulate rollback)
|
||||
const { runMigrations: rollback } = await getKibanaMigratorTestKit({ types, logFilePath });
|
||||
|
||||
await clearLog(logFilePath);
|
||||
await expect(rollback()).rejects.toThrowError(
|
||||
`Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: The ${defaultKibanaIndex}_${nextMinor} alias refers to a newer version of Kibana: v${nextMinor}`
|
||||
);
|
||||
|
||||
const logs = await parseLogFile(logFilePath);
|
||||
expect(logs).toContainLogEntry('[.kibana_migrator_tests] INIT -> FATAL.');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await esServer?.stop();
|
||||
await delay(2);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue