github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add multi-instance tests for the .kibana split (#156756)

Browse source 
Improve _dot kibana split_ integration tests, take into account
https://github.com/elastic/kibana/pull/157356 fix.
This commit is contained in:
Gerard Soldevila 2023-05-12 16:47:21 +02:00 • committed by GitHub
parent ab5338f5da
commit 612064aa8b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 580 additions and 452 deletions
Download patch file Download diff file Expand all files Collapse all files

42
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap generated
View file

@ -209,13 +209,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",
@ -436,13 +435,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",
@ -667,13 +665,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",
@ -902,13 +899,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",
@ -1162,13 +1158,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",
@ -1400,13 +1395,12 @@ Object {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".my-so-index_7.11.0",

7
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts
View file

@ -265,13 +265,12 @@ describe('createInitialState', () => {
"tempIndexMappings": Object {
"dynamic": false,
"properties": Object {
"migrationVersion": Object {
"dynamic": "true",
"type": "object",
},
"type": Object {
"type": "keyword",
},
"typeMigrationVersion": Object {
"type": "version",
},
},
},
"versionAlias": ".kibana_task_manager_8.1.0",

6
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts
View file

@ -89,10 +89,8 @@ export const createInitialState = ({
dynamic: false,
properties: {
type: { type: 'keyword' },
migrationVersion: {
// @ts-expect-error we don't allow plugins to set `dynamic`
dynamic: 'true',
type: 'object',
typeMigrationVersion: {
type: 'version',
},
},
};

48
src/core/server/integration_tests/saved_objects/migrations/group3/__snapshots__/dot_kibana_split.test.ts.snap generated Normal file
View file

@ -0,0 +1,48 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`split .kibana index into multiple system indices when multiple Kibana migrators run in parallel correctly migrates 7.7.2_xpack_100k_obj.zip archive: after migration 1`] = `
Object {
".kibana": Object {
"apm-telemetry": 1,
"config": 1,
"space": 1,
"ui-metric": 5,
},
".kibana_alerting_cases": Object {},
".kibana_analytics": Object {
"dashboard": 52994,
"index-pattern": 1,
"search": 1,
"visualization": 53004,
},
".kibana_ingest": Object {},
".kibana_security_solution": Object {},
".kibana_task_manager": Object {
"task": 5,
},
}
`;
exports[`split .kibana index into multiple system indices when multiple Kibana migrators run in parallel correctly migrates 7.7.2_xpack_100k_obj.zip archive: before migration 1`] = `
Object {
".kibana": Object {
"apm-telemetry": 1,
"application_usage_transactional": 4,
"config": 1,
"dashboard": 52994,
"index-pattern": 1,
"maps-telemetry": 1,
"search": 1,
"space": 1,
"ui-metric": 5,
"visualization": 53004,
},
".kibana_alerting_cases": undefined,
".kibana_analytics": undefined,
".kibana_ingest": undefined,
".kibana_security_solution": undefined,
".kibana_task_manager": Object {
"task": 5,
},
}
`;

453
src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts Normal file
View file

@ -0,0 +1,453 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Path from 'path';
import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
import {
type ISavedObjectTypeRegistry,
type SavedObjectsType,
MAIN_SAVED_OBJECT_INDEX,
} from '@kbn/core-saved-objects-server';
import {
clearLog,
startElasticsearch,
getKibanaMigratorTestKit,
getCurrentVersionTypeRegistry,
overrideTypeRegistry,
getAggregatedTypesCount,
currentVersion,
type KibanaMigratorTestKit,
getEsClient,
getAggregatedTypesCountAllIndices,
} from '../kibana_migrator_test_kit';
import { delay, parseLogFile } from '../test_utils';
import '../jest_matchers';
// define a type => index distribution
const RELOCATE_TYPES: Record<string, string> = {
dashboard: '.kibana_so_ui',
visualization: '.kibana_so_ui',
'canvas-workpad': '.kibana_so_ui',
search: '.kibana_so_search',
task: '.kibana_task_manager',
// the remaining types will be forced to go to '.kibana',
// overriding `indexPattern: foo` defined in the registry
};
const PARALLEL_MIGRATORS = 6;
export const logFilePath = Path.join(__dirname, 'dot_kibana_split.test.log');
describe('split .kibana index into multiple system indices', () => {
let esServer: TestElasticsearchUtils['es'];
let typeRegistry: ISavedObjectTypeRegistry;
beforeAll(async () => {
typeRegistry = await getCurrentVersionTypeRegistry({ oss: false });
});
beforeEach(async () => {
await clearLog(logFilePath);
});
describe('when migrating from a legacy version', () => {
let migratorTestKitFactory: () => Promise<KibanaMigratorTestKit>;
beforeAll(async () => {
esServer = await startElasticsearch({
dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'),
});
});
it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => {
const updatedTypeRegistry = overrideTypeRegistry(
typeRegistry,
(type: SavedObjectsType<any>) => {
return {
...type,
indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX,
};
}
);
migratorTestKitFactory = () =>
getKibanaMigratorTestKit({
types: updatedTypeRegistry.getAllTypes(),
kibanaIndex: '.kibana',
logFilePath,
});
const { runMigrations, client } = await migratorTestKitFactory();
// count of types in the legacy index
expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({
'canvas-workpad': 3,
config: 1,
dashboard: 3,
'index-pattern': 3,
map: 3,
'maps-telemetry': 1,
'sample-data-telemetry': 3,
search: 2,
telemetry: 1,
space: 1,
visualization: 39,
});
await runMigrations();
await client.indices.refresh({
index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'],
});
expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({
'index-pattern': 3,
map: 3,
'sample-data-telemetry': 3,
config: 1,
telemetry: 1,
space: 1,
});
expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({
search: 2,
});
expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({
visualization: 39,
'canvas-workpad': 3,
dashboard: 3,
});
const indicesInfo = await client.indices.get({ index: '.kibana*' });
expect(indicesInfo[`.kibana_${currentVersion}_001`]).toEqual(
expect.objectContaining({
aliases: expect.objectContaining({ '.kibana': expect.any(Object) }),
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
})
);
expect(indicesInfo[`.kibana_so_search_${currentVersion}_001`]).toEqual(
expect.objectContaining({
aliases: expect.objectContaining({ '.kibana_so_search': expect.any(Object) }),
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
})
);
expect(indicesInfo[`.kibana_so_ui_${currentVersion}_001`]).toEqual(
expect.objectContaining({
aliases: expect.objectContaining({ '.kibana_so_ui': expect.any(Object) }),
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
})
);
expect(indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap)
.toMatchInlineSnapshot(`
Object {
".kibana": Array [
"action",
"action_task_params",
"alert",
"api_key_pending_invalidation",
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"canvas-element",
"canvas-workpad-template",
"cases",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"config",
"config-global",
"connector_token",
"core-usage-stats",
"csp-rule-template",
"endpoint:user-artifact-manifest",
"enterprise_search_telemetry",
"epm-packages",
"epm-packages-assets",
"event_loop_delays_daily",
"exception-list",
"exception-list-agnostic",
"file",
"file-upload-usage-collection-telemetry",
"fileShare",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"fleet-uninstall-tokens",
"graph-workspace",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"index-pattern",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"ingest_manager_settings",
"inventory-view",
"kql-telemetry",
"legacy-url-alias",
"lens",
"lens-ui-telemetry",
"maintenance-window",
"map",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"osquery-manager-usage-metric",
"osquery-pack",
"osquery-pack-asset",
"osquery-saved-query",
"query",
"rules-settings",
"sample-data-telemetry",
"search-session",
"search-telemetry",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"workplace_search_telemetry",
],
".kibana_so_search": Array [
"search",
],
".kibana_so_ui": Array [
"canvas-workpad",
"dashboard",
"visualization",
],
".kibana_task_manager": Array [
"task",
],
}
`);
const logs = await parseLogFile(logFilePath);
expect(logs).toContainLogEntries(
[
// .kibana_task_manager index exists and has no aliases => LEGACY_* migration path
'[.kibana_task_manager] INIT -> LEGACY_SET_WRITE_BLOCK.',
'[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.',
'[.kibana_task_manager] LEGACY_DELETE -> SET_SOURCE_WRITE_BLOCK.',
'[.kibana_task_manager] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.',
'[.kibana_task_manager] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.',
'[.kibana_task_manager] CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.',
'[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK.',
'[.kibana_task_manager] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.',
'[.kibana_task_manager] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.',
'[.kibana_task_manager] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.',
'[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.',
'[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.',
'[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.',
'[.kibana_task_manager] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.',
'[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.',
'[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.',
'[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.',
'[.kibana_task_manager] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.',
'[.kibana_task_manager] MARK_VERSION_INDEX_READY -> DONE.',
'[.kibana_task_manager] Migration completed after',
],
{ ordered: true }
);
expect(logs).not.toContainLogEntries([
// .kibana_task_manager migrator is NOT involved in relocation, must not sync with other migrators
'[.kibana_task_manager] READY_TO_REINDEX_SYNC',
'[.kibana_task_manager] DONE_REINDEXING_SYNC',
]);
// new indices migrators did not exist, so they all have to reindex (create temp index + sync)
['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => {
expect(logs).toContainLogEntries(
[
`[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`,
`[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`,
// no docs to reindex, as source index did NOT exist
`[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`,
],
{ ordered: true }
);
});
// the .kibana migrator is involved in a relocation, it must also reindex
expect(logs).toContainLogEntries(
[
'[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.',
'[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS.',
'[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.',
'[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.',
'[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.',
'[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.',
'[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.',
'[.kibana] Starting to process 59 documents.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.',
'[.kibana] Processed 59 documents out of 59.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.',
'[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.',
],
{ ordered: true }
);
// after .kibana migrator is done relocating documents
// the 3 migrators share the final part of the flow
['.kibana', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => {
expect(logs).toContainLogEntries(
[
`[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`,
`[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`,
`[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`,
`[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`,
`[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`,
`[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`,
`[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.`,
`[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`,
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`,
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`,
`[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`,
`[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.`,
`[${index}] MARK_VERSION_INDEX_READY -> DONE.`,
`[${index}] Migration completed after`,
],
{ ordered: true }
);
});
// should NOT retransform anything (we reindexed, thus we transformed already)
['.kibana', '.kibana_task_manager', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => {
expect(logs).not.toContainLogEntry(`[${index}] OUTDATED_DOCUMENTS_TRANSFORM`);
});
});
afterEach(async () => {
// we run the migrator again to ensure that the next time state is loaded everything still works as expected
const { runMigrations } = await migratorTestKitFactory();
await clearLog(logFilePath);
await runMigrations();
const logs = await parseLogFile(logFilePath);
expect(logs).not.toContainLogEntries(['REINDEX', 'CREATE', 'UPDATE_TARGET_MAPPINGS']);
});
afterAll(async () => {
await esServer?.stop();
await delay(2);
});
});
describe('when multiple Kibana migrators run in parallel', () => {
it('correctly migrates 7.7.2_xpack_100k_obj.zip archive', async () => {
esServer = await startElasticsearch({
dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'),
});
const esClient = await getEsClient();
const breakdownBefore = await getAggregatedTypesCountAllIndices(esClient);
expect(breakdownBefore).toMatchSnapshot('before migration');
for (let i = 0; i < PARALLEL_MIGRATORS; ++i) {
await clearLog(Path.join(__dirname, `dot_kibana_split_instance_${i}.log`));
}
const testKits = await Promise.all(
new Array(PARALLEL_MIGRATORS)
.fill({
settings: {
migrations: {
discardUnknownObjects: currentVersion,
discardCorruptObjects: currentVersion,
},
},
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
types: typeRegistry.getAllTypes(),
})
.map((config, index) =>
getKibanaMigratorTestKit({
...config,
logFilePath: Path.join(__dirname, `dot_kibana_split_instance_${index}.log`),
})
)
);
const results = await Promise.all(testKits.map((testKit) => testKit.runMigrations()));
expect(
results
.flat()
.every((result) => result.status === 'migrated' || result.status === 'patched')
).toEqual(true);
const breakdownAfter = await getAggregatedTypesCountAllIndices(esClient);
expect(breakdownAfter).toMatchSnapshot('after migration');
});
afterEach(async () => {
await esServer?.stop();
await delay(2);
});
});
});

386
src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts
View file

@ -1,386 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Path from 'path';
import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
import {
type ISavedObjectTypeRegistry,
type SavedObjectsType,
MAIN_SAVED_OBJECT_INDEX,
} from '@kbn/core-saved-objects-server';
import {
readLog,
startElasticsearch,
getKibanaMigratorTestKit,
getCurrentVersionTypeRegistry,
overrideTypeRegistry,
clearLog,
getAggregatedTypesCount,
currentVersion,
type KibanaMigratorTestKit,
} from '../kibana_migrator_test_kit';
import { delay } from '../test_utils';
// define a type => index distribution
const RELOCATE_TYPES: Record<string, string> = {
dashboard: '.kibana_so_ui',
visualization: '.kibana_so_ui',
'canvas-workpad': '.kibana_so_ui',
search: '.kibana_so_search',
task: '.kibana_task_manager',
// the remaining types will be forced to go to '.kibana',
// overriding `indexPattern: foo` defined in the registry
};
describe('split .kibana index into multiple system indices', () => {
let esServer: TestElasticsearchUtils['es'];
let typeRegistry: ISavedObjectTypeRegistry;
let migratorTestKitFactory: () => Promise<KibanaMigratorTestKit>;
beforeAll(async () => {
typeRegistry = await getCurrentVersionTypeRegistry({ oss: false });
esServer = await startElasticsearch({
dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'),
});
});
beforeEach(async () => {
await clearLog();
});
describe('when migrating from a legacy version', () => {
it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => {
const updatedTypeRegistry = overrideTypeRegistry(
typeRegistry,
(type: SavedObjectsType<any>) => {
return {
...type,
indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX,
};
}
);
migratorTestKitFactory = () =>
getKibanaMigratorTestKit({
types: updatedTypeRegistry.getAllTypes(),
kibanaIndex: '.kibana',
});
const { runMigrations, client } = await migratorTestKitFactory();
// count of types in the legacy index
expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({
'canvas-workpad': 3,
config: 1,
dashboard: 3,
'index-pattern': 3,
map: 3,
'maps-telemetry': 1,
'sample-data-telemetry': 3,
search: 2,
telemetry: 1,
space: 1,
visualization: 39,
});
await runMigrations();
await client.indices.refresh({
index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'],
});
expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({
'index-pattern': 3,
map: 3,
'sample-data-telemetry': 3,
config: 1,
telemetry: 1,
space: 1,
});
expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({
search: 2,
});
expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({
visualization: 39,
'canvas-workpad': 3,
dashboard: 3,
});
const indicesInfo = await client.indices.get({ index: '.kibana*' });
expect(indicesInfo).toEqual(
expect.objectContaining({
'.kibana_8.9.0_001': {
aliases: { '.kibana': expect.any(Object), '.kibana_8.9.0': expect.any(Object) },
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
},
'.kibana_so_search_8.9.0_001': {
aliases: {
'.kibana_so_search': expect.any(Object),
'.kibana_so_search_8.9.0': expect.any(Object),
},
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
},
'.kibana_so_ui_8.9.0_001': {
aliases: {
'.kibana_so_ui': expect.any(Object),
'.kibana_so_ui_8.9.0': expect.any(Object),
},
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: expect.any(Object),
indexTypesMap: expect.any(Object),
},
properties: expect.any(Object),
},
settings: { index: expect.any(Object) },
},
})
);
expect(indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap)
.toMatchInlineSnapshot(`
Object {
".kibana": Array [
"action",
"action_task_params",
"alert",
"api_key_pending_invalidation",
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"canvas-element",
"canvas-workpad-template",
"cases",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"config",
"config-global",
"connector_token",
"core-usage-stats",
"csp-rule-template",
"endpoint:user-artifact-manifest",
"enterprise_search_telemetry",
"epm-packages",
"epm-packages-assets",
"event_loop_delays_daily",
"exception-list",
"exception-list-agnostic",
"file",
"file-upload-usage-collection-telemetry",
"fileShare",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"fleet-uninstall-tokens",
"graph-workspace",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"index-pattern",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"ingest_manager_settings",
"inventory-view",
"kql-telemetry",
"legacy-url-alias",
"lens",
"lens-ui-telemetry",
"maintenance-window",
"map",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"osquery-manager-usage-metric",
"osquery-pack",
"osquery-pack-asset",
"osquery-saved-query",
"query",
"rules-settings",
"sample-data-telemetry",
"search-session",
"search-telemetry",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"workplace_search_telemetry",
],
".kibana_so_search": Array [
"search",
],
".kibana_so_ui": Array [
"canvas-workpad",
"dashboard",
"visualization",
],
".kibana_task_manager": Array [
"task",
],
}
`);
const logs = await readLog();
// .kibana_task_manager index exists and has no aliases => LEGACY_* migration path
expect(logs).toMatch('[.kibana_task_manager] INIT -> LEGACY_SET_WRITE_BLOCK.');
// .kibana_task_manager migrator is NOT involved in relocation, must not sync
expect(logs).not.toMatch('[.kibana_task_manager] READY_TO_REINDEX_SYNC');
// newer indices migrators did not exist, so they all have to reindex (create temp index + sync)
['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => {
expect(logs).toMatch(`[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`);
expect(logs).toMatch(`[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`);
// no docs to reindex, as source index did NOT exist
expect(logs).toMatch(`[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`);
});
// the .kibana migrator is involved in a relocation, it must also reindex
expect(logs).toMatch('[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.');
expect(logs).toMatch('[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS.');
expect(logs).toMatch('[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.');
expect(logs).toMatch('[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.');
expect(logs).toMatch('[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.');
expect(logs).toMatch('[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.');
expect(logs).toMatch('[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.');
expect(logs).toMatch(
'[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.'
);
expect(logs).toMatch('[.kibana] Starting to process 59 documents.');
expect(logs).toMatch(
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.'
);
expect(logs).toMatch(
'[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.'
);
expect(logs).toMatch('[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.');
expect(logs).toMatch(
'[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.'
);
expect(logs).toMatch('[.kibana] Processed 59 documents out of 59.');
expect(logs).toMatch(
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.'
);
expect(logs).toMatch('[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.');
// after .kibana migrator is done relocating documents
// the 3 migrators share the final part of the flow
[
['.kibana', 8],
['.kibana_so_ui', 45],
['.kibana_so_search', 2],
].forEach(([index, docCount]) => {
expect(logs).toMatch(`[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`);
expect(logs).toMatch(`[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`);
expect(logs).toMatch(`[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`);
expect(logs).toMatch(`[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`);
expect(logs).toMatch(
`[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`
);
expect(logs).toMatch(`[${index}] Starting to process ${docCount} documents.`);
expect(logs).toMatch(
`[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_TRANSFORM.`
);
expect(logs).toMatch(
`[${index}] OUTDATED_DOCUMENTS_TRANSFORM -> TRANSFORMED_DOCUMENTS_BULK_INDEX.`
);
expect(logs).toMatch(
`[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`
);
expect(logs).toMatch(
`[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_REFRESH.`
);
expect(logs).toMatch(`[${index}] OUTDATED_DOCUMENTS_REFRESH -> CHECK_TARGET_MAPPINGS.`);
expect(logs).toMatch(
`[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`
);
expect(logs).toMatch(
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`
);
expect(logs).toMatch(
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`
);
expect(logs).toMatch(
`[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`
);
expect(logs).toMatch(
`[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.`
);
expect(logs).toMatch(`[${index}] MARK_VERSION_INDEX_READY -> DONE.`);
expect(logs).toMatch(`[${index}] Migration completed`);
});
});
});
afterEach(async () => {
// we run the migrator again to ensure that the next time state is loaded everything still works as expected
const { runMigrations } = await migratorTestKitFactory();
await clearLog();
await runMigrations();
const logs = await readLog();
expect(logs).not.toMatch('REINDEX');
expect(logs).not.toMatch('CREATE');
expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS');
});
afterAll(async () => {
await esServer?.stop();
await delay(10);
});
});

78
src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts
View file

@ -33,7 +33,11 @@ import {
import { AgentManager, configureClient } from '@kbn/core-elasticsearch-client-server-internal';
import { type LoggingConfigType, LoggingSystem } from '@kbn/core-logging-server-internal';
import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server';
import {
ALL_SAVED_OBJECT_INDICES,
ISavedObjectTypeRegistry,
SavedObjectsType,
} from '@kbn/core-saved-objects-server';
import { esTestConfig, kibanaServerTestUser } from '@kbn/test';
import type { LoggerFactory } from '@kbn/logging';
import { createRootWithCorePlugins, createTestServers } from '@kbn/core-test-helpers-kbn-server';
@ -289,36 +293,60 @@ const getMigrator = async (
});
};
export const getAggregatedTypesCount = async (client: ElasticsearchClient, index: string) => {
await client.indices.refresh();
const response = await client.search<unknown, { typesAggregation: { buckets: any[] } }>({
index,
_source: false,
aggs: {
typesAggregation: {
terms: {
// assign type __UNKNOWN__ to those documents that don't define one
missing: '__UNKNOWN__',
field: 'type',
size: 100,
},
aggs: {
docs: {
top_hits: {
size: 10,
_source: {
excludes: ['*'],
export const getAggregatedTypesCount = async (
client: ElasticsearchClient,
index: string
): Promise<Record<string, number> | undefined> => {
try {
await client.indices.refresh({ index });
const response = await client.search<unknown, { typesAggregation: { buckets: any[] } }>({
index,
_source: false,
aggs: {
typesAggregation: {
terms: {
// assign type __UNKNOWN__ to those documents that don't define one
missing: '__UNKNOWN__',
field: 'type',
size: 100,
},
aggs: {
docs: {
top_hits: {
size: 10,
_source: {
excludes: ['*'],
},
},
},
},
},
},
},
});
});
return (response.aggregations!.typesAggregation.buckets as unknown as any).reduce(
(acc: any, current: any) => {
acc[current.key] = current.doc_count;
return (response.aggregations!.typesAggregation.buckets as unknown as any).reduce(
(acc: any, current: any) => {
acc[current.key] = current.doc_count;
return acc;
},
{}
);
} catch (error) {
if (error.meta?.statusCode === 404) {
return undefined;
}
throw error;
}
};
export const getAggregatedTypesCountAllIndices = async (esClient: ElasticsearchClient) => {
const typeBreakdown = await Promise.all(
ALL_SAVED_OBJECT_INDICES.map((index) => getAggregatedTypesCount(esClient, index))
);
return ALL_SAVED_OBJECT_INDICES.reduce<Record<string, Record<string, number> | undefined>>(
(acc, index, pos) => {
acc[index] = typeBreakdown[pos];
return acc;
},
{}

4
x-pack/test/detection_engine_api_integration/security_and_spaces/group10/resolve_read_rules.ts
View file

@ -138,9 +138,7 @@ export default ({ getService }: FtrProviderContext) => {
references: [],
namespaces: [spaceId],
originId: 'c364e1e0-2615-11ec-811e-db7211397897',
migrationVersion: {
alert: '8.0.0',
},
typeMigrationVersion: '8.0.0',
coreMigrationVersion: '8.0.0',
updated_at: '2021-10-05T19:52:56.014Z',
},

8
x-pack/test/fleet_api_integration/apis/agents/status.ts
View file

@ -39,9 +39,7 @@ export default function ({ getService }: FtrProviderContext) {
updated_by: 'system',
inactivity_timeout: 60,
},
migrationVersion: {
'ingest-agent-policies': '7.10.0',
},
typeMigrationVersion: '7.10.0',
},
});
// 2 agents online
@ -287,9 +285,7 @@ export default function ({ getService }: FtrProviderContext) {
updated_by: 'system',
inactivity_timeout: 60,
},
migrationVersion: {
'ingest-agent-policies': '7.10.0',
},
typeMigrationVersion: '7.10.0',
},
})
)