mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
Split the .kibana saved objects index into multiple indices (#154888)
## Description Fix https://github.com/elastic/kibana/issues/104081 This PR move some of the SO types from the `.kibana` index into the following ones: - `.kibana_alerting_cases` - `.kibana_analytics` - `.kibana_security_solution` - `.kibana_ingest` This split/reallocation will occur during the `8.8.0` Kibana upgrade (*meaning: from any version older than `8.8.0` to any version greater or equal to `8.8.0`*) **This PR main changes are:** - implement the changes required in the SO migration algorithm to support this reallocation - update the FTR tools (looking at you esArchiver) to support these new indices - update hardcoded references to `.kibana` and usage of the `core.savedObjects.getKibanaIndex()` to use new APIs to target the correct index/indices - update FTR datasets, tests and utility accordingly ## To reviewers **Overall estimated risk of regressions: low** But, still, please take the time to review changes in your code. The parts of the production code that were the most impacted are the telemetry collectors, as most of them were performing direct requests against the `.kibana` index, so we had to adapt them. Most other contributor-owned changes are in FTR tests and datasets. If you think a type is misplaced (either we missed some types that should be moved to a specific index, or some types were moved and shouldn't have been) please tell us, and we'll fix the reallocation either in this PR or in a follow-up. ## .Kibana split The following new indices are introduced by this PR, with the following SO types being moved to it. (any SO type not listed here will be staying in its current index) Note: The complete **_type => index_** breakdown is available in [this spreadsheet](https://docs.google.com/spreadsheets/d/1b_MG_E_aBksZ4Vkd9cVayij1oBpdhvH4XC8NVlChiio/edit#gid=145920788). #### `.kibana_alerting_cases` - action - action_task_params - alert - api_key_pending_invalidation - cases - cases-comments - cases-configure - cases-connector-mappings - cases-telemetry - cases-user-actions - connector_token - rules-settings - maintenance-window #### `.kibana_security_solution` - csp-rule-template - endpoint:user-artifact - endpoint:user-artifact-manifest - exception-list - exception-list-agnostic - osquery-manager-usage-metric - osquery-pack - osquery-pack-asset - osquery-saved-query - security-rule - security-solution-signals-migration - siem-detection-engine-rule-actions - siem-ui-timeline - siem-ui-timeline-note - siem-ui-timeline-pinned-event #### `.kibana_analytics` - canvas-element - canvas-workpad-template - canvas-workpad - dashboard - graph-workspace - index-pattern - kql-telemetry - lens - lens-ui-telemetry - map - search - search-session - search-telemetry - visualization #### `.kibana_ingest` - epm-packages - epm-packages-assets - fleet-fleet-server-host - fleet-message-signing-keys - fleet-preconfiguration-deletion-record - fleet-proxy - ingest_manager_settings - ingest-agent-policies - ingest-download-sources - ingest-outputs - ingest-package-policies ## Tasks / PRs ### Sub-PRs **Implementation** - 🟣 https://github.com/elastic/kibana/pull/154846 - 🟣 https://github.com/elastic/kibana/pull/154892 - 🟣 https://github.com/elastic/kibana/pull/154882 - 🟣 https://github.com/elastic/kibana/pull/154884 - 🟣 https://github.com/elastic/kibana/pull/155155 **Individual index split** - 🟣 https://github.com/elastic/kibana/pull/154897 - 🟣 https://github.com/elastic/kibana/pull/155129 - 🟣 https://github.com/elastic/kibana/pull/155140 - 🟣 https://github.com/elastic/kibana/pull/155130 ### Improvements / follow-ups - 👷🏼 Extract logic into [runV2Migration](https://github.com/elastic/kibana/pull/154151#discussion_r1158470566) @gsoldevila - Make `getCurrentIndexTypesMap` resillient to intermittent failures https://github.com/elastic/kibana/pull/154151#discussion_r1169289717 - 🚧 Build a more structured [MigratorSynchronizer](https://github.com/elastic/kibana/pull/154151#discussion_r1158469918) - 🟣 https://github.com/elastic/kibana/pull/155035 - 🟣 https://github.com/elastic/kibana/pull/155116 - 🟣 https://github.com/elastic/kibana/pull/155366 ## Reallocation tweaks Tweaks to the reallocation can be done after the initial merge, as long as it's done before the public release of 8.8 - `url` should get back to `.kibana` (see [comment](https://github.com/elastic/kibana/pull/154888#discussion_r1172317133)) ## Release Note For performance purposes, Kibana is now using more system indices to store its internal data. The following system indices will be created when upgrading to `8.8.0`: - `.kibana_alerting_cases` - `.kibana_analytics` - `.kibana_security_solution` - `.kibana_ingest` --------- Co-authored-by: pgayvallet <pierre.gayvallet@elastic.co> Co-authored-by: Christos Nasikas <christos.nasikas@elastic.co> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Georgii Gorbachev <georgii.gorbachev@elastic.co>
This commit is contained in:
Gerard Soldevila
GitHub
parent
4e4f408a34
commit
21351df953
221 changed files with 13309 additions and 1570 deletions
|
|
@ -245,7 +245,8 @@ export function createPluginSetupContext<TPlugin, TPluginDependencies>(
|
|||
setSecurityExtension: deps.savedObjects.setSecurityExtension,
|
||||
setSpacesExtension: deps.savedObjects.setSpacesExtension,
|
||||
registerType: deps.savedObjects.registerType,
|
||||
getKibanaIndex: deps.savedObjects.getKibanaIndex,
|
||||
getDefaultIndex: deps.savedObjects.getDefaultIndex,
|
||||
getAllIndices: deps.savedObjects.getAllIndices,
|
||||
},
|
||||
status: {
|
||||
core$: deps.status.core$,
|
||||
|
|
@ -313,6 +314,10 @@ export function createPluginStartContext<TPlugin, TPluginDependencies>(
|
|||
createExporter: deps.savedObjects.createExporter,
|
||||
createImporter: deps.savedObjects.createImporter,
|
||||
getTypeRegistry: deps.savedObjects.getTypeRegistry,
|
||||
getDefaultIndex: deps.savedObjects.getDefaultIndex,
|
||||
getIndexForType: deps.savedObjects.getIndexForType,
|
||||
getIndicesForTypes: deps.savedObjects.getIndicesForTypes,
|
||||
getAllIndices: deps.savedObjects.getAllIndices,
|
||||
},
|
||||
metrics: {
|
||||
collectionInterval: deps.metrics.collectionInterval,
|
||||
|
|
|
|||
|
|
@ -22,8 +22,9 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m
|
|||
import { kibanaMigratorMock } from '../mocks';
|
||||
import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import {
|
||||
ISavedObjectsEncryptionExtension,
|
||||
SavedObjectsRawDocSource,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
type ISavedObjectsEncryptionExtension,
|
||||
type SavedObjectsRawDocSource,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
bulkCreateSuccess,
|
||||
|
|
@ -41,8 +42,8 @@ import {
|
|||
mockVersion,
|
||||
mockVersionProps,
|
||||
MULTI_NAMESPACE_ENCRYPTED_TYPE,
|
||||
TypeIdTuple,
|
||||
updateSuccess,
|
||||
type TypeIdTuple,
|
||||
} from '../test_helpers/repository.test.common';
|
||||
import { savedObjectsExtensionsMock } from '../mocks/saved_objects_extensions.mock';
|
||||
|
||||
|
|
@ -633,7 +634,7 @@ describe('SavedObjectsRepository Encryption Extension', () => {
|
|||
total: 2,
|
||||
hits: [
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${space ? `${space}:` : ''}${encryptedSO.type}:${encryptedSO.id}`,
|
||||
_score: 1,
|
||||
...mockVersionProps,
|
||||
|
|
@ -643,7 +644,7 @@ describe('SavedObjectsRepository Encryption Extension', () => {
|
|||
},
|
||||
},
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${space ? `${space}:` : ''}index-pattern:logstash-*`,
|
||||
_score: 2,
|
||||
...mockVersionProps,
|
||||
|
|
|
|||
|
|
@ -47,13 +47,14 @@ import type {
|
|||
SavedObjectsBulkDeleteObject,
|
||||
SavedObjectsBulkDeleteOptions,
|
||||
} from '@kbn/core-saved-objects-api-server';
|
||||
import type {
|
||||
SavedObjectsRawDoc,
|
||||
SavedObjectsRawDocSource,
|
||||
SavedObjectUnsanitizedDoc,
|
||||
SavedObject,
|
||||
SavedObjectReference,
|
||||
BulkResolveError,
|
||||
import {
|
||||
type SavedObjectsRawDoc,
|
||||
type SavedObjectsRawDocSource,
|
||||
type SavedObjectUnsanitizedDoc,
|
||||
type SavedObject,
|
||||
type SavedObjectReference,
|
||||
type BulkResolveError,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import { ALL_NAMESPACES_STRING } from '@kbn/core-saved-objects-utils-server';
|
||||
import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
|
||||
|
|
@ -4364,7 +4365,7 @@ describe('SavedObjectsRepository', () => {
|
|||
body: {
|
||||
_id: params.id,
|
||||
...mockVersionProps,
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
get: {
|
||||
found: true,
|
||||
_source: {
|
||||
|
|
@ -4668,7 +4669,7 @@ describe('SavedObjectsRepository', () => {
|
|||
body: {
|
||||
_id: params.id,
|
||||
...mockVersionProps,
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
get: {
|
||||
found: true,
|
||||
_source: {
|
||||
|
|
|
|||
|
|
@ -9,19 +9,20 @@
|
|||
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
import { Payload } from 'elastic-apm-node';
|
||||
import type {
|
||||
AuthorizationTypeEntry,
|
||||
AuthorizeAndRedactMultiNamespaceReferencesParams,
|
||||
CheckAuthorizationResult,
|
||||
ISavedObjectsSecurityExtension,
|
||||
SavedObjectsMappingProperties,
|
||||
SavedObjectsRawDocSource,
|
||||
SavedObjectsType,
|
||||
SavedObjectsTypeMappingDefinition,
|
||||
SavedObject,
|
||||
SavedObjectReference,
|
||||
AuthorizeFindParams,
|
||||
import type { Payload } from 'elastic-apm-node';
|
||||
import {
|
||||
type AuthorizationTypeEntry,
|
||||
type AuthorizeAndRedactMultiNamespaceReferencesParams,
|
||||
type CheckAuthorizationResult,
|
||||
type ISavedObjectsSecurityExtension,
|
||||
type SavedObjectsMappingProperties,
|
||||
type SavedObjectsRawDocSource,
|
||||
type SavedObjectsType,
|
||||
type SavedObjectsTypeMappingDefinition,
|
||||
type SavedObject,
|
||||
type SavedObjectReference,
|
||||
type AuthorizeFindParams,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import type {
|
||||
SavedObjectsBaseOptions,
|
||||
|
|
@ -47,9 +48,9 @@ import {
|
|||
} from '@kbn/core-elasticsearch-client-server-mocks';
|
||||
import { DocumentMigrator } from '@kbn/core-saved-objects-migration-server-internal';
|
||||
import {
|
||||
AuthorizeAndRedactInternalBulkResolveParams,
|
||||
GetFindRedactTypeMapParams,
|
||||
AuthorizationTypeMap,
|
||||
type AuthorizeAndRedactInternalBulkResolveParams,
|
||||
type GetFindRedactTypeMapParams,
|
||||
type AuthorizationTypeMap,
|
||||
SavedObjectsErrorHelpers,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import { mockGetSearchDsl } from '../lib/repository.test.mock';
|
||||
|
|
@ -601,8 +602,6 @@ export const getMockBulkCreateResponse = (
|
|||
managed: docManaged,
|
||||
}) => ({
|
||||
create: {
|
||||
// status: 1,
|
||||
// _index: '.kibana',
|
||||
_id: `${namespace ? `${namespace}:` : ''}${type}:${id}`,
|
||||
_source: {
|
||||
[type]: attributes,
|
||||
|
|
@ -726,7 +725,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
|
|||
total: 4,
|
||||
hits: [
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${namespace ? `${namespace}:` : ''}index-pattern:logstash-*`,
|
||||
_score: 1,
|
||||
...mockVersionProps,
|
||||
|
|
@ -743,7 +742,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
|
|||
},
|
||||
},
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${namespace ? `${namespace}:` : ''}config:6.0.0-alpha1`,
|
||||
_score: 2,
|
||||
...mockVersionProps,
|
||||
|
|
@ -758,7 +757,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
|
|||
},
|
||||
},
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${namespace ? `${namespace}:` : ''}index-pattern:stocks-*`,
|
||||
_score: 3,
|
||||
...mockVersionProps,
|
||||
|
|
@ -774,7 +773,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => {
|
|||
},
|
||||
},
|
||||
{
|
||||
_index: '.kibana',
|
||||
_index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_id: `${NAMESPACE_AGNOSTIC_TYPE}:something`,
|
||||
_score: 4,
|
||||
...mockVersionProps,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ export {
|
|||
getTypes,
|
||||
type IndexMapping,
|
||||
type IndexMappingMeta,
|
||||
type IndexTypesMap,
|
||||
type SavedObjectsTypeMappingDefinitions,
|
||||
type IndexMappingMigrationStateMeta,
|
||||
} from './src/mappings';
|
||||
|
|
|
|||
|
|
@ -11,5 +11,6 @@ export type {
|
|||
SavedObjectsTypeMappingDefinitions,
|
||||
IndexMappingMeta,
|
||||
IndexMapping,
|
||||
IndexTypesMap,
|
||||
IndexMappingMigrationStateMeta,
|
||||
} from './types';
|
||||
|
|
|
|||
|
|
@ -55,6 +55,9 @@ export interface IndexMapping {
|
|||
_meta?: IndexMappingMeta;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export type IndexTypesMap = Record<string, string[]>;
|
||||
|
||||
/** @internal */
|
||||
export interface IndexMappingMeta {
|
||||
/**
|
||||
|
|
@ -65,6 +68,12 @@ export interface IndexMappingMeta {
|
|||
* @remark: Only defined for indices using the v2 migration algorithm.
|
||||
*/
|
||||
migrationMappingPropertyHashes?: { [k: string]: string };
|
||||
/**
|
||||
* A map that tells what are the SO types stored in each index
|
||||
*
|
||||
* @remark: Only defined for indices using the v2 migration algorithm.
|
||||
*/
|
||||
indexTypesMap?: IndexTypesMap;
|
||||
/**
|
||||
* The current model versions of the mapping of the index.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -69,7 +69,11 @@ export type MigrationStatus =
|
|||
/** @internal */
|
||||
export type MigrationResult =
|
||||
| { status: 'skipped' }
|
||||
| { status: 'patched' }
|
||||
| {
|
||||
status: 'patched';
|
||||
destIndex: string;
|
||||
elapsedMs: number;
|
||||
}
|
||||
| {
|
||||
status: 'migrated';
|
||||
destIndex: string;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
type ISavedObjectTypeRegistry,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import { getIndexForType } from './get_index_for_type';
|
||||
|
||||
const createTypeRegistry = () => {
|
||||
|
|
@ -17,7 +20,7 @@ const createTypeRegistry = () => {
|
|||
|
||||
describe('getIndexForType', () => {
|
||||
const kibanaVersion = '8.0.0';
|
||||
const defaultIndex = '.kibana';
|
||||
const defaultIndex = MAIN_SAVED_OBJECT_INDEX;
|
||||
let typeRegistry: ReturnType<typeof createTypeRegistry>;
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
|
|||
|
|
@ -138,6 +138,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -154,6 +168,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [],
|
||||
"outdatedDocumentsQuery": Object {
|
||||
"bool": Object {
|
||||
|
|
@ -167,6 +182,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -325,6 +356,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -345,6 +390,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [],
|
||||
"outdatedDocumentsQuery": Object {
|
||||
"bool": Object {
|
||||
|
|
@ -358,6 +404,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -516,6 +578,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -540,6 +616,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [],
|
||||
"outdatedDocumentsQuery": Object {
|
||||
"bool": Object {
|
||||
|
|
@ -553,6 +630,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -711,6 +804,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -739,6 +846,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [],
|
||||
"outdatedDocumentsQuery": Object {
|
||||
"bool": Object {
|
||||
|
|
@ -752,6 +860,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -954,6 +1078,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -970,6 +1108,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [
|
||||
Object {
|
||||
"_id": "1234",
|
||||
|
|
@ -988,6 +1127,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -1152,6 +1307,20 @@ Object {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".my-so-index",
|
||||
|
|
@ -1172,6 +1341,7 @@ Object {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocuments": Array [
|
||||
Object {
|
||||
"_id": "1234",
|
||||
|
|
@ -1190,6 +1360,22 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
|
|||
|
|
@ -83,6 +83,9 @@ export { cleanupUnknownAndExcluded } from './cleanup_unknown_and_excluded';
|
|||
export { waitForDeleteByQueryTask } from './wait_for_delete_by_query_task';
|
||||
|
||||
export type { CreateIndexParams, ClusterShardLimitExceeded } from './create_index';
|
||||
|
||||
export { synchronizeMigrators } from './synchronize_migrators';
|
||||
|
||||
export { createIndex } from './create_index';
|
||||
|
||||
export { checkTargetMappings } from './check_target_mappings';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
import { synchronizeMigrators } from './synchronize_migrators';
|
||||
import { type Defer, defer } from '../kibana_migrator_utils';
|
||||
|
||||
describe('synchronizeMigrators', () => {
|
||||
let defers: Array<Defer<void>>;
|
||||
let allDefersPromise: Promise<any>;
|
||||
let migratorsDefers: Array<Defer<void>>;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
defers = ['.kibana_cases', '.kibana_task_manager', '.kibana'].map(defer);
|
||||
allDefersPromise = Promise.all(defers.map(({ promise }) => promise));
|
||||
|
||||
migratorsDefers = defers.map(({ resolve, reject }) => ({
|
||||
resolve: jest.fn(resolve),
|
||||
reject: jest.fn(reject),
|
||||
promise: allDefersPromise,
|
||||
}));
|
||||
});
|
||||
|
||||
describe('when all migrators reach the synchronization point with a correct state', () => {
|
||||
it('unblocks all migrators and resolves Right', async () => {
|
||||
const tasks = migratorsDefers.map((migratorDefer) => synchronizeMigrators(migratorDefer));
|
||||
|
||||
const res = await Promise.all(tasks.map((task) => task()));
|
||||
|
||||
migratorsDefers.forEach((migratorDefer) =>
|
||||
expect(migratorDefer.resolve).toHaveBeenCalledTimes(1)
|
||||
);
|
||||
migratorsDefers.forEach((migratorDefer) =>
|
||||
expect(migratorDefer.reject).not.toHaveBeenCalled()
|
||||
);
|
||||
|
||||
expect(res).toEqual([
|
||||
{ _tag: 'Right', right: 'synchronized_successfully' },
|
||||
{ _tag: 'Right', right: 'synchronized_successfully' },
|
||||
{ _tag: 'Right', right: 'synchronized_successfully' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('migrators are not unblocked until the last one reaches the synchronization point', async () => {
|
||||
let resolved: number = 0;
|
||||
migratorsDefers.forEach((migratorDefer) => migratorDefer.promise.then(() => ++resolved));
|
||||
const [casesDefer, ...otherMigratorsDefers] = migratorsDefers;
|
||||
|
||||
// we simulate that only kibana_task_manager and kibana migrators get to the sync point
|
||||
const tasks = otherMigratorsDefers.map((migratorDefer) =>
|
||||
synchronizeMigrators(migratorDefer)
|
||||
);
|
||||
// we don't await for them, or we would be locked forever
|
||||
Promise.all(tasks.map((task) => task()));
|
||||
|
||||
const [taskManagerDefer, kibanaDefer] = otherMigratorsDefers;
|
||||
expect(taskManagerDefer.resolve).toHaveBeenCalledTimes(1);
|
||||
expect(kibanaDefer.resolve).toHaveBeenCalledTimes(1);
|
||||
expect(casesDefer.resolve).not.toHaveBeenCalled();
|
||||
expect(resolved).toEqual(0);
|
||||
|
||||
// finally, the last migrator gets to the synchronization point
|
||||
await synchronizeMigrators(casesDefer)();
|
||||
expect(resolved).toEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('when one migrator fails and rejects the synchronization defer', () => {
|
||||
describe('before the rest of the migrators reach the synchronization point', () => {
|
||||
it('synchronizedMigrators resolves Left for the rest of migrators', async () => {
|
||||
let resolved: number = 0;
|
||||
let errors: number = 0;
|
||||
migratorsDefers.forEach((migratorDefer) =>
|
||||
migratorDefer.promise.then(() => ++resolved).catch(() => ++errors)
|
||||
);
|
||||
const [casesDefer, ...otherMigratorsDefers] = migratorsDefers;
|
||||
|
||||
// we first make one random migrator fail and not reach the sync point
|
||||
casesDefer.reject('Oops. The cases migrator failed unexpectedly.');
|
||||
|
||||
// the other migrators then try to synchronize
|
||||
const tasks = otherMigratorsDefers.map((migratorDefer) =>
|
||||
synchronizeMigrators(migratorDefer)
|
||||
);
|
||||
|
||||
expect(Promise.all(tasks.map((task) => task()))).resolves.toEqual([
|
||||
{
|
||||
_tag: 'Left',
|
||||
left: {
|
||||
type: 'sync_failed',
|
||||
error: 'Oops. The cases migrator failed unexpectedly.',
|
||||
},
|
||||
},
|
||||
{
|
||||
_tag: 'Left',
|
||||
left: {
|
||||
type: 'sync_failed',
|
||||
error: 'Oops. The cases migrator failed unexpectedly.',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
// force next tick (as we did not await for Promises)
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(resolved).toEqual(0);
|
||||
expect(errors).toEqual(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('after the rest of the migrators reach the synchronization point', () => {
|
||||
it('synchronizedMigrators resolves Left for the rest of migrators', async () => {
|
||||
let resolved: number = 0;
|
||||
let errors: number = 0;
|
||||
migratorsDefers.forEach((migratorDefer) =>
|
||||
migratorDefer.promise.then(() => ++resolved).catch(() => ++errors)
|
||||
);
|
||||
const [casesDefer, ...otherMigratorsDefers] = migratorsDefers;
|
||||
|
||||
// some migrators try to synchronize
|
||||
const tasks = otherMigratorsDefers.map((migratorDefer) =>
|
||||
synchronizeMigrators(migratorDefer)
|
||||
);
|
||||
|
||||
// we then make one random migrator fail and not reach the sync point
|
||||
casesDefer.reject('Oops. The cases migrator failed unexpectedly.');
|
||||
|
||||
expect(Promise.all(tasks.map((task) => task()))).resolves.toEqual([
|
||||
{
|
||||
_tag: 'Left',
|
||||
left: {
|
||||
type: 'sync_failed',
|
||||
error: 'Oops. The cases migrator failed unexpectedly.',
|
||||
},
|
||||
},
|
||||
{
|
||||
_tag: 'Left',
|
||||
left: {
|
||||
type: 'sync_failed',
|
||||
error: 'Oops. The cases migrator failed unexpectedly.',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
// force next tick (as we did not await for Promises)
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(resolved).toEqual(0);
|
||||
expect(errors).toEqual(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import * as Either from 'fp-ts/lib/Either';
|
||||
import * as TaskEither from 'fp-ts/lib/TaskEither';
|
||||
import type { Defer } from '../kibana_migrator_utils';
|
||||
|
||||
export interface SyncFailed {
|
||||
type: 'sync_failed';
|
||||
error: Error;
|
||||
}
|
||||
|
||||
export function synchronizeMigrators(
|
||||
defer: Defer<void>
|
||||
): TaskEither.TaskEither<SyncFailed, 'synchronized_successfully'> {
|
||||
return () => {
|
||||
defer.resolve();
|
||||
return defer.promise
|
||||
.then(() => Either.right('synchronized_successfully' as const))
|
||||
.catch((error) => Either.left({ type: 'sync_failed' as const, error }));
|
||||
};
|
||||
}
|
||||
|
|
@ -13,44 +13,63 @@ import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
|
|||
import {
|
||||
type SavedObjectsMigrationConfigType,
|
||||
SavedObjectTypeRegistry,
|
||||
type IndexMapping,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
|
||||
import { createInitialState } from './initial_state';
|
||||
import { createInitialState, type CreateInitialStateParams } from './initial_state';
|
||||
|
||||
const mockLogger = loggingSystemMock.create();
|
||||
|
||||
const migrationsConfig = {
|
||||
retryAttempts: 15,
|
||||
batchSize: 1000,
|
||||
maxBatchSizeBytes: ByteSizeValue.parse('100mb'),
|
||||
} as unknown as SavedObjectsMigrationConfigType;
|
||||
|
||||
const createInitialStateCommonParams = {
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: true,
|
||||
indexTypesMap: {
|
||||
'.kibana': ['typeA', 'typeB', 'typeC'],
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana_cases': ['typeD', 'typeE'],
|
||||
},
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
} as IndexMapping,
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
};
|
||||
|
||||
describe('createInitialState', () => {
|
||||
let typeRegistry: SavedObjectTypeRegistry;
|
||||
let docLinks: DocLinksServiceSetup;
|
||||
let logger: Logger;
|
||||
let createInitialStateParams: CreateInitialStateParams;
|
||||
|
||||
beforeEach(() => {
|
||||
typeRegistry = new SavedObjectTypeRegistry();
|
||||
docLinks = docLinksServiceMock.createSetupContract();
|
||||
logger = mockLogger.get();
|
||||
createInitialStateParams = {
|
||||
...createInitialStateCommonParams,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => jest.clearAllMocks());
|
||||
|
||||
const migrationsConfig = {
|
||||
retryAttempts: 15,
|
||||
batchSize: 1000,
|
||||
maxBatchSizeBytes: ByteSizeValue.parse('100mb'),
|
||||
} as unknown as SavedObjectsMigrationConfigType;
|
||||
|
||||
it('creates the initial state for the model based on the passed in parameters', () => {
|
||||
expect(
|
||||
createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
...createInitialStateParams,
|
||||
waitForMigrationCompletion: true,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
})
|
||||
).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
|
@ -172,6 +191,20 @@ describe('createInitialState', () => {
|
|||
},
|
||||
},
|
||||
"indexPrefix": ".kibana_task_manager",
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
"kibanaVersion": "8.1.0",
|
||||
"knownTypes": Array [],
|
||||
"legacyIndex": ".kibana_task_manager",
|
||||
|
|
@ -183,6 +216,7 @@ describe('createInitialState', () => {
|
|||
"resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html",
|
||||
"routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled",
|
||||
},
|
||||
"mustRelocateDocuments": true,
|
||||
"outdatedDocumentsQuery": Object {
|
||||
"bool": Object {
|
||||
"should": Array [],
|
||||
|
|
@ -195,6 +229,22 @@ describe('createInitialState', () => {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": Object {
|
||||
"my_type": Object {
|
||||
|
|
@ -227,22 +277,7 @@ describe('createInitialState', () => {
|
|||
});
|
||||
|
||||
it('creates the initial state for the model with waitForMigrationCompletion false,', () => {
|
||||
expect(
|
||||
createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
})
|
||||
).toMatchObject({
|
||||
expect(createInitialState(createInitialStateParams)).toMatchObject({
|
||||
waitForMigrationCompletion: false,
|
||||
});
|
||||
});
|
||||
|
|
@ -262,18 +297,10 @@ describe('createInitialState', () => {
|
|||
});
|
||||
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
...createInitialStateParams,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(initialState.knownTypes).toEqual(['foo', 'bar']);
|
||||
|
|
@ -289,40 +316,15 @@ describe('createInitialState', () => {
|
|||
excludeOnUpgrade: fooExcludeOnUpgradeHook,
|
||||
});
|
||||
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
});
|
||||
|
||||
const initialState = createInitialState(createInitialStateParams);
|
||||
expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ foo: fooExcludeOnUpgradeHook });
|
||||
});
|
||||
|
||||
it('returns state with a preMigration script', () => {
|
||||
const preMigrationScript = "ctx._id = ctx._source.type + ':' + ctx._id";
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
...createInitialStateParams,
|
||||
preMigrationScript,
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
});
|
||||
|
||||
expect(Option.isSome(initialState.preMigrationScript)).toEqual(true);
|
||||
|
|
@ -334,19 +336,8 @@ describe('createInitialState', () => {
|
|||
expect(
|
||||
Option.isNone(
|
||||
createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
...createInitialStateParams,
|
||||
preMigrationScript: undefined,
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
}).preMigrationScript
|
||||
)
|
||||
).toEqual(true);
|
||||
|
|
@ -354,19 +345,9 @@ describe('createInitialState', () => {
|
|||
it('returns state with an outdatedDocumentsQuery', () => {
|
||||
expect(
|
||||
createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
...createInitialStateParams,
|
||||
preMigrationScript: "ctx._id = ctx._source.type + ':' + ctx._id",
|
||||
migrationVersionPerType: { my_dashboard: '7.10.1', my_viz: '8.0.0' },
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
}).outdatedDocumentsQuery
|
||||
).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
|
@ -473,44 +454,19 @@ describe('createInitialState', () => {
|
|||
});
|
||||
|
||||
it('initializes the `discardUnknownObjects` flag to false if the flag is not provided in the config', () => {
|
||||
const logger = mockLogger.get();
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
});
|
||||
const initialState = createInitialState(createInitialStateParams);
|
||||
|
||||
expect(logger.warn).not.toBeCalled();
|
||||
expect(initialState.discardUnknownObjects).toEqual(false);
|
||||
});
|
||||
|
||||
it('initializes the `discardUnknownObjects` flag to false if the value provided in the config does not match the current kibana version', () => {
|
||||
const logger = mockLogger.get();
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
...createInitialStateParams,
|
||||
migrationsConfig: {
|
||||
...migrationsConfig,
|
||||
discardUnknownObjects: '8.0.0',
|
||||
},
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(initialState.discardUnknownObjects).toEqual(false);
|
||||
|
|
@ -522,44 +478,23 @@ describe('createInitialState', () => {
|
|||
|
||||
it('initializes the `discardUnknownObjects` flag to true if the value provided in the config matches the current kibana version', () => {
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
...createInitialStateParams,
|
||||
migrationsConfig: {
|
||||
...migrationsConfig,
|
||||
discardUnknownObjects: '8.1.0',
|
||||
},
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
});
|
||||
|
||||
expect(initialState.discardUnknownObjects).toEqual(true);
|
||||
});
|
||||
|
||||
it('initializes the `discardCorruptObjects` flag to false if the value provided in the config does not match the current kibana version', () => {
|
||||
const logger = mockLogger.get();
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
...createInitialStateParams,
|
||||
migrationsConfig: {
|
||||
...migrationsConfig,
|
||||
discardCorruptObjects: '8.0.0',
|
||||
},
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(initialState.discardCorruptObjects).toEqual(false);
|
||||
|
|
@ -571,21 +506,11 @@ describe('createInitialState', () => {
|
|||
|
||||
it('initializes the `discardCorruptObjects` flag to true if the value provided in the config matches the current kibana version', () => {
|
||||
const initialState = createInitialState({
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
targetMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.kibana_task_manager',
|
||||
...createInitialStateParams,
|
||||
migrationsConfig: {
|
||||
...migrationsConfig,
|
||||
discardCorruptObjects: '8.1.0',
|
||||
},
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger: mockLogger.get(),
|
||||
});
|
||||
|
||||
expect(initialState.discardCorruptObjects).toEqual(true);
|
||||
|
|
|
|||
|
|
@ -14,28 +14,18 @@ import type { SavedObjectsMigrationVersion } from '@kbn/core-saved-objects-commo
|
|||
import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
|
||||
import type {
|
||||
IndexMapping,
|
||||
IndexTypesMap,
|
||||
SavedObjectsMigrationConfigType,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { InitState } from './state';
|
||||
import { excludeUnusedTypesQuery } from './core';
|
||||
import { getTempIndexName } from './model/helpers';
|
||||
|
||||
/**
|
||||
* Construct the initial state for the model
|
||||
*/
|
||||
export const createInitialState = ({
|
||||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
targetMappings,
|
||||
preMigrationScript,
|
||||
migrationVersionPerType,
|
||||
indexPrefix,
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
}: {
|
||||
export interface CreateInitialStateParams {
|
||||
kibanaVersion: string;
|
||||
waitForMigrationCompletion: boolean;
|
||||
mustRelocateDocuments: boolean;
|
||||
indexTypesMap: IndexTypesMap;
|
||||
targetMappings: IndexMapping;
|
||||
preMigrationScript?: string;
|
||||
migrationVersionPerType: SavedObjectsMigrationVersion;
|
||||
|
|
@ -44,7 +34,25 @@ export const createInitialState = ({
|
|||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
docLinks: DocLinksServiceStart;
|
||||
logger: Logger;
|
||||
}): InitState => {
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the initial state for the model
|
||||
*/
|
||||
export const createInitialState = ({
|
||||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
preMigrationScript,
|
||||
migrationVersionPerType,
|
||||
indexPrefix,
|
||||
migrationsConfig,
|
||||
typeRegistry,
|
||||
docLinks,
|
||||
logger,
|
||||
}: CreateInitialStateParams): InitState => {
|
||||
const outdatedDocumentsQuery: QueryDslQueryContainer = {
|
||||
bool: {
|
||||
should: Object.entries(migrationVersionPerType).map(([type, latestVersion]) => ({
|
||||
|
|
@ -117,18 +125,28 @@ export const createInitialState = ({
|
|||
);
|
||||
}
|
||||
|
||||
const targetIndexMappings: IndexMapping = {
|
||||
...targetMappings,
|
||||
_meta: {
|
||||
...targetMappings._meta,
|
||||
indexTypesMap,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
controlState: 'INIT',
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypesMap,
|
||||
indexPrefix,
|
||||
legacyIndex: indexPrefix,
|
||||
currentAlias: indexPrefix,
|
||||
versionAlias: `${indexPrefix}_${kibanaVersion}`,
|
||||
versionIndex: `${indexPrefix}_${kibanaVersion}_001`,
|
||||
tempIndex: `${indexPrefix}_${kibanaVersion}_reindex_temp`,
|
||||
tempIndex: getTempIndexName(indexPrefix, kibanaVersion),
|
||||
kibanaVersion,
|
||||
preMigrationScript: Option.fromNullable(preMigrationScript),
|
||||
targetIndexMappings: targetMappings,
|
||||
targetIndexMappings,
|
||||
tempIndexMappings: reindexTargetMappings,
|
||||
outdatedDocumentsQuery,
|
||||
retryCount: 0,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,15 @@ import { DocumentMigrator } from './document_migrator';
|
|||
import { ByteSizeValue } from '@kbn/config-schema';
|
||||
import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import { runResilientMigrator } from './run_resilient_migrator';
|
||||
|
||||
jest.mock('./run_resilient_migrator', () => {
|
||||
const actual = jest.requireActual('./run_resilient_migrator');
|
||||
|
||||
return {
|
||||
runResilientMigrator: jest.fn(actual.runResilientMigrator),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock('./document_migrator', () => {
|
||||
return {
|
||||
|
|
@ -29,6 +38,21 @@ jest.mock('./document_migrator', () => {
|
|||
};
|
||||
});
|
||||
|
||||
const mappingsResponseWithoutIndexTypesMap: estypes.IndicesGetMappingResponse = {
|
||||
'.kibana_8.7.0_001': {
|
||||
mappings: {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
references: '7997cf5a56cc02bdc9c93361bde732b0',
|
||||
// ...
|
||||
},
|
||||
// we do not add a `indexTypesMap`
|
||||
// simulating a Kibana < 8.8.0 that does not have one yet
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const createRegistry = (types: Array<Partial<SavedObjectsType>>) => {
|
||||
const registry = new SavedObjectTypeRegistry();
|
||||
types.forEach((type) =>
|
||||
|
|
@ -47,6 +71,7 @@ const createRegistry = (types: Array<Partial<SavedObjectsType>>) => {
|
|||
describe('KibanaMigrator', () => {
|
||||
beforeEach(() => {
|
||||
(DocumentMigrator as jest.Mock).mockClear();
|
||||
(runResilientMigrator as jest.MockedFunction<typeof runResilientMigrator>).mockClear();
|
||||
});
|
||||
describe('getActiveMappings', () => {
|
||||
it('returns full index mappings w/ core properties', () => {
|
||||
|
|
@ -60,7 +85,7 @@ describe('KibanaMigrator', () => {
|
|||
},
|
||||
{
|
||||
name: 'bmap',
|
||||
indexPattern: 'other-index',
|
||||
indexPattern: '.other-index',
|
||||
mappings: {
|
||||
properties: { field: { type: 'text' } },
|
||||
},
|
||||
|
|
@ -98,19 +123,34 @@ describe('KibanaMigrator', () => {
|
|||
describe('runMigrations', () => {
|
||||
it('throws if prepareMigrations is not called first', async () => {
|
||||
const options = mockOptions();
|
||||
|
||||
options.client.indices.get.mockResponse({}, { statusCode: 200 });
|
||||
|
||||
const migrator = new KibanaMigrator(options);
|
||||
|
||||
await expect(() => migrator.runMigrations()).toThrowErrorMatchingInlineSnapshot(
|
||||
`"Migrations are not ready. Make sure prepareMigrations is called first."`
|
||||
await expect(migrator.runMigrations()).rejects.toThrowError(
|
||||
'Migrations are not ready. Make sure prepareMigrations is called first.'
|
||||
);
|
||||
});
|
||||
|
||||
it('only runs migrations once if called multiple times', async () => {
|
||||
const successfulRun: typeof runResilientMigrator = ({ indexPrefix }) =>
|
||||
Promise.resolve({
|
||||
sourceIndex: indexPrefix,
|
||||
destIndex: indexPrefix,
|
||||
elapsedMs: 28,
|
||||
status: 'migrated',
|
||||
});
|
||||
const mockRunResilientMigrator = runResilientMigrator as jest.MockedFunction<
|
||||
typeof runResilientMigrator
|
||||
>;
|
||||
|
||||
mockRunResilientMigrator.mockImplementationOnce(successfulRun);
|
||||
mockRunResilientMigrator.mockImplementationOnce(successfulRun);
|
||||
mockRunResilientMigrator.mockImplementationOnce(successfulRun);
|
||||
mockRunResilientMigrator.mockImplementationOnce(successfulRun);
|
||||
const options = mockOptions();
|
||||
options.client.indices.get.mockResponse({}, { statusCode: 200 });
|
||||
options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
|
||||
statusCode: 200,
|
||||
});
|
||||
|
||||
options.client.cluster.getSettings.mockResponse(
|
||||
{
|
||||
|
|
@ -127,11 +167,42 @@ describe('KibanaMigrator', () => {
|
|||
await migrator.runMigrations();
|
||||
|
||||
// indices.get is called twice during a single migration
|
||||
expect(options.client.indices.get).toHaveBeenCalledTimes(2);
|
||||
expect(runResilientMigrator).toHaveBeenCalledTimes(4);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
indexPrefix: '.my-index',
|
||||
mustRelocateDocuments: true,
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
indexPrefix: '.other-index',
|
||||
mustRelocateDocuments: true,
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
indexPrefix: '.my-task-index',
|
||||
mustRelocateDocuments: false,
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
expect.objectContaining({
|
||||
indexPrefix: '.my-complementary-index',
|
||||
mustRelocateDocuments: true,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('emits results on getMigratorResult$()', async () => {
|
||||
const options = mockV2MigrationOptions();
|
||||
options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
|
||||
statusCode: 200,
|
||||
});
|
||||
const migrator = new KibanaMigrator(options);
|
||||
const migratorStatus = lastValueFrom(migrator.getStatus$().pipe(take(3)));
|
||||
migrator.prepareMigrations();
|
||||
|
|
@ -146,12 +217,12 @@ describe('KibanaMigrator', () => {
|
|||
status: 'migrated',
|
||||
});
|
||||
expect(result![1]).toMatchObject({
|
||||
destIndex: 'other-index_8.2.3_001',
|
||||
destIndex: '.other-index_8.2.3_001',
|
||||
elapsedMs: expect.any(Number),
|
||||
status: 'patched',
|
||||
});
|
||||
});
|
||||
it('rejects when the migration state machine terminates in a FATAL state', () => {
|
||||
it('rejects when the migration state machine terminates in a FATAL state', async () => {
|
||||
const options = mockV2MigrationOptions();
|
||||
options.client.indices.get.mockResponse(
|
||||
{
|
||||
|
|
@ -166,6 +237,9 @@ describe('KibanaMigrator', () => {
|
|||
},
|
||||
{ statusCode: 200 }
|
||||
);
|
||||
options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
|
||||
statusCode: 200,
|
||||
});
|
||||
|
||||
const migrator = new KibanaMigrator(options);
|
||||
migrator.prepareMigrations();
|
||||
|
|
@ -181,6 +255,9 @@ describe('KibanaMigrator', () => {
|
|||
error: { type: 'elasticsearch_exception', reason: 'task failed with an error' },
|
||||
task: { description: 'task description' } as any,
|
||||
});
|
||||
options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
|
||||
statusCode: 200,
|
||||
});
|
||||
|
||||
const migrator = new KibanaMigrator(options);
|
||||
migrator.prepareMigrations();
|
||||
|
|
@ -193,6 +270,160 @@ describe('KibanaMigrator', () => {
|
|||
{"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}]
|
||||
`);
|
||||
});
|
||||
|
||||
describe('for V2 migrations', () => {
|
||||
describe('where some SO types must be relocated', () => {
|
||||
it('runs successfully', async () => {
|
||||
const options = mockV2MigrationOptions();
|
||||
options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
|
||||
statusCode: 200,
|
||||
});
|
||||
|
||||
const migrator = new KibanaMigrator(options);
|
||||
migrator.prepareMigrations();
|
||||
const results = await migrator.runMigrations();
|
||||
|
||||
expect(results.length).toEqual(4);
|
||||
expect(results[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
sourceIndex: '.my-index_pre8.2.3_001',
|
||||
destIndex: '.my-index_8.2.3_001',
|
||||
elapsedMs: expect.any(Number),
|
||||
status: 'migrated',
|
||||
})
|
||||
);
|
||||
expect(results[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
destIndex: '.other-index_8.2.3_001',
|
||||
elapsedMs: expect.any(Number),
|
||||
status: 'patched',
|
||||
})
|
||||
);
|
||||
expect(results[2]).toEqual(
|
||||
expect.objectContaining({
|
||||
destIndex: '.my-task-index_8.2.3_001',
|
||||
elapsedMs: expect.any(Number),
|
||||
status: 'patched',
|
||||
})
|
||||
);
|
||||
expect(results[3]).toEqual(
|
||||
expect.objectContaining({
|
||||
destIndex: '.my-complementary-index_8.2.3_001',
|
||||
elapsedMs: expect.any(Number),
|
||||
status: 'patched',
|
||||
})
|
||||
);
|
||||
|
||||
expect(runResilientMigrator).toHaveBeenCalledTimes(4);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
kibanaVersion: '8.2.3',
|
||||
indexPrefix: '.my-index',
|
||||
indexTypesMap: {
|
||||
'.my-index': ['testtype', 'testtype3'],
|
||||
'.other-index': ['testtype2'],
|
||||
'.my-task-index': ['testtasktype'],
|
||||
},
|
||||
targetMappings: expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
testtype: expect.anything(),
|
||||
testtype3: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
readyToReindex: expect.objectContaining({
|
||||
promise: expect.anything(),
|
||||
resolve: expect.anything(),
|
||||
reject: expect.anything(),
|
||||
}),
|
||||
mustRelocateDocuments: true,
|
||||
doneReindexing: expect.objectContaining({
|
||||
promise: expect.anything(),
|
||||
resolve: expect.anything(),
|
||||
reject: expect.anything(),
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
kibanaVersion: '8.2.3',
|
||||
indexPrefix: '.other-index',
|
||||
indexTypesMap: {
|
||||
'.my-index': ['testtype', 'testtype3'],
|
||||
'.other-index': ['testtype2'],
|
||||
'.my-task-index': ['testtasktype'],
|
||||
},
|
||||
targetMappings: expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
testtype2: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
readyToReindex: expect.objectContaining({
|
||||
promise: expect.anything(),
|
||||
resolve: expect.anything(),
|
||||
reject: expect.anything(),
|
||||
}),
|
||||
mustRelocateDocuments: true,
|
||||
doneReindexing: expect.objectContaining({
|
||||
promise: expect.anything(),
|
||||
resolve: expect.anything(),
|
||||
reject: expect.anything(),
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
kibanaVersion: '8.2.3',
|
||||
indexPrefix: '.my-task-index',
|
||||
indexTypesMap: {
|
||||
'.my-index': ['testtype', 'testtype3'],
|
||||
'.other-index': ['testtype2'],
|
||||
'.my-task-index': ['testtasktype'],
|
||||
},
|
||||
targetMappings: expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
testtasktype: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
// this migrator is NOT involved in any relocation,
|
||||
// thus, it must not synchronize with other migrators
|
||||
mustRelocateDocuments: false,
|
||||
readyToReindex: undefined,
|
||||
doneReindexing: undefined,
|
||||
})
|
||||
);
|
||||
expect(runResilientMigrator).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
expect.objectContaining({
|
||||
kibanaVersion: '8.2.3',
|
||||
indexPrefix: '.my-complementary-index',
|
||||
indexTypesMap: {
|
||||
'.my-index': ['testtype', 'testtype3'],
|
||||
'.other-index': ['testtype2'],
|
||||
'.my-task-index': ['testtasktype'],
|
||||
},
|
||||
targetMappings: expect.objectContaining({
|
||||
properties: expect.not.objectContaining({
|
||||
// this index does no longer have any types associated to it
|
||||
testtype: expect.anything(),
|
||||
testtype2: expect.anything(),
|
||||
testtype3: expect.anything(),
|
||||
testtasktype: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
mustRelocateDocuments: true,
|
||||
doneReindexing: expect.objectContaining({
|
||||
promise: expect.anything(),
|
||||
resolve: expect.anything(),
|
||||
reject: expect.anything(),
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -254,7 +485,19 @@ const mockOptions = () => {
|
|||
logger: loggingSystemMock.create().get(),
|
||||
kibanaVersion: '8.2.3',
|
||||
waitForMigrationCompletion: false,
|
||||
defaultIndexTypesMap: {
|
||||
'.my-index': ['testtype', 'testtype2'],
|
||||
'.my-task-index': ['testtasktype'],
|
||||
// this index no longer has any types registered in typeRegistry
|
||||
// but we still need a migrator for it, so that 'testtype3' documents
|
||||
// are moved over to their new index (.my_index)
|
||||
'.my-complementary-index': ['testtype3'],
|
||||
},
|
||||
typeRegistry: createRegistry([
|
||||
// typeRegistry depicts an updated index map:
|
||||
// .my-index: ['testtype', 'testtype3'],
|
||||
// .my-other-index: ['testtype2'],
|
||||
// .my-task-index': ['testtasktype'],
|
||||
{
|
||||
name: 'testtype',
|
||||
hidden: false,
|
||||
|
|
@ -270,7 +513,32 @@ const mockOptions = () => {
|
|||
name: 'testtype2',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
indexPattern: 'other-index',
|
||||
// We are moving 'testtype2' from '.my-index' to '.other-index'
|
||||
indexPattern: '.other-index',
|
||||
mappings: {
|
||||
properties: {
|
||||
name: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
migrations: {},
|
||||
},
|
||||
{
|
||||
name: 'testtasktype',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
indexPattern: '.my-task-index',
|
||||
mappings: {
|
||||
properties: {
|
||||
name: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
migrations: {},
|
||||
},
|
||||
{
|
||||
// We are moving 'testtype3' from '.my-complementary-index' to '.my-index'
|
||||
name: 'testtype3',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
mappings: {
|
||||
properties: {
|
||||
name: { type: 'keyword' },
|
||||
|
|
|
|||
|
|
@ -16,10 +16,11 @@ import Semver from 'semver';
|
|||
import type { Logger } from '@kbn/logging';
|
||||
import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type {
|
||||
SavedObjectUnsanitizedDoc,
|
||||
SavedObjectsRawDoc,
|
||||
ISavedObjectTypeRegistry,
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
type SavedObjectUnsanitizedDoc,
|
||||
type SavedObjectsRawDoc,
|
||||
type ISavedObjectTypeRegistry,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
SavedObjectsSerializer,
|
||||
|
|
@ -29,17 +30,17 @@ import {
|
|||
type IKibanaMigrator,
|
||||
type KibanaMigratorStatus,
|
||||
type MigrationResult,
|
||||
type IndexTypesMap,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getIndicesInvolvedInRelocation } from './kibana_migrator_utils';
|
||||
import { buildActiveMappings, buildTypesMappings } from './core';
|
||||
import { DocumentMigrator } from './document_migrator';
|
||||
import { createIndexMap } from './core/build_index_map';
|
||||
import { runResilientMigrator } from './run_resilient_migrator';
|
||||
import { migrateRawDocsSafely } from './core/migrate_raw_docs';
|
||||
import { runZeroDowntimeMigration } from './zdt';
|
||||
|
||||
// ensure plugins don't try to convert SO namespaceTypes after 8.0.0
|
||||
// see https://github.com/elastic/kibana/issues/147344
|
||||
const ALLOWED_CONVERT_VERSION = '8.0.0';
|
||||
import { createMultiPromiseDefer, indexMapToIndexTypesMap } from './kibana_migrator_utils';
|
||||
import { ALLOWED_CONVERT_VERSION, DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants';
|
||||
|
||||
export interface KibanaMigratorOptions {
|
||||
client: ElasticsearchClient;
|
||||
|
|
@ -50,6 +51,7 @@ export interface KibanaMigratorOptions {
|
|||
logger: Logger;
|
||||
docLinks: DocLinksServiceStart;
|
||||
waitForMigrationCompletion: boolean;
|
||||
defaultIndexTypesMap?: IndexTypesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -71,6 +73,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
private readonly soMigrationsConfig: SavedObjectsMigrationConfigType;
|
||||
private readonly docLinks: DocLinksServiceStart;
|
||||
private readonly waitForMigrationCompletion: boolean;
|
||||
private readonly defaultIndexTypesMap: IndexTypesMap;
|
||||
public readonly kibanaVersion: string;
|
||||
|
||||
/**
|
||||
|
|
@ -84,6 +87,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
kibanaVersion,
|
||||
logger,
|
||||
docLinks,
|
||||
defaultIndexTypesMap = DEFAULT_INDEX_TYPES_MAP,
|
||||
waitForMigrationCompletion,
|
||||
}: KibanaMigratorOptions) {
|
||||
this.client = client;
|
||||
|
|
@ -105,6 +109,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
// operation so we cache the result
|
||||
this.activeMappings = buildActiveMappings(this.mappingProperties);
|
||||
this.docLinks = docLinks;
|
||||
this.defaultIndexTypesMap = defaultIndexTypesMap;
|
||||
}
|
||||
|
||||
public runMigrations({ rerun = false }: { rerun?: boolean } = {}): Promise<MigrationResult[]> {
|
||||
|
|
@ -134,12 +139,12 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
return this.status$.asObservable();
|
||||
}
|
||||
|
||||
private runMigrationsInternal(): Promise<MigrationResult[]> {
|
||||
private async runMigrationsInternal(): Promise<MigrationResult[]> {
|
||||
const migrationAlgorithm = this.soMigrationsConfig.algorithm;
|
||||
if (migrationAlgorithm === 'zdt') {
|
||||
return this.runMigrationZdt();
|
||||
return await this.runMigrationZdt();
|
||||
} else {
|
||||
return this.runMigrationV2();
|
||||
return await this.runMigrationV2();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -157,7 +162,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
});
|
||||
}
|
||||
|
||||
private runMigrationV2(): Promise<MigrationResult[]> {
|
||||
private async runMigrationV2(): Promise<MigrationResult[]> {
|
||||
const indexMap = createIndexMap({
|
||||
kibanaIndexName: this.kibanaIndex,
|
||||
indexMap: this.mappingProperties,
|
||||
|
|
@ -173,16 +178,59 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
this.log.debug(`migrationVersion: ${migrationVersion} saved object type: ${type}`);
|
||||
});
|
||||
|
||||
const migrators = Object.keys(indexMap).map((index) => {
|
||||
// build a indexTypesMap from the info present in tye typeRegistry, e.g.:
|
||||
// {
|
||||
// '.kibana': ['typeA', 'typeB', ...]
|
||||
// '.kibana_task_manager': ['task', ...]
|
||||
// '.kibana_cases': ['typeC', 'typeD', ...]
|
||||
// ...
|
||||
// }
|
||||
const indexTypesMap = indexMapToIndexTypesMap(indexMap);
|
||||
|
||||
// compare indexTypesMap with the one present (or not) in the .kibana index meta
|
||||
// and check if some SO types have been moved to different indices
|
||||
const indicesWithMovingTypes = await getIndicesInvolvedInRelocation({
|
||||
mainIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
client: this.client,
|
||||
indexTypesMap,
|
||||
logger: this.log,
|
||||
defaultIndexTypesMap: this.defaultIndexTypesMap,
|
||||
});
|
||||
|
||||
// we create 2 synchronization objects (2 synchronization points) for each of the
|
||||
// migrators involved in relocations, aka each of the migrators that will:
|
||||
// A) reindex some documents TO other indices
|
||||
// B) receive some documents FROM other indices
|
||||
// C) both
|
||||
const readyToReindexDefers = createMultiPromiseDefer(indicesWithMovingTypes);
|
||||
const doneReindexingDefers = createMultiPromiseDefer(indicesWithMovingTypes);
|
||||
|
||||
// build a list of all migrators that must be started
|
||||
const migratorIndices = new Set(Object.keys(indexMap));
|
||||
// indices involved in a relocation might no longer be present in current mappings
|
||||
// but if their SOs must be relocated to another index, we still need a migrator to do the job
|
||||
indicesWithMovingTypes.forEach((index) => migratorIndices.add(index));
|
||||
|
||||
const migrators = Array.from(migratorIndices).map((indexName, i) => {
|
||||
return {
|
||||
migrate: (): Promise<MigrationResult> => {
|
||||
const readyToReindex = readyToReindexDefers[indexName];
|
||||
const doneReindexing = doneReindexingDefers[indexName];
|
||||
// check if this migrator's index is involved in some document redistribution
|
||||
const mustRelocateDocuments = !!readyToReindex;
|
||||
|
||||
return runResilientMigrator({
|
||||
client: this.client,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
mustRelocateDocuments,
|
||||
indexTypesMap,
|
||||
waitForMigrationCompletion: this.waitForMigrationCompletion,
|
||||
targetMappings: buildActiveMappings(indexMap[index].typeMappings),
|
||||
// a migrator's index might no longer have any associated types to it
|
||||
targetMappings: buildActiveMappings(indexMap[indexName]?.typeMappings ?? {}),
|
||||
logger: this.log,
|
||||
preMigrationScript: indexMap[index].script,
|
||||
preMigrationScript: indexMap[indexName]?.script,
|
||||
readyToReindex,
|
||||
doneReindexing,
|
||||
transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) =>
|
||||
migrateRawDocsSafely({
|
||||
serializer: this.serializer,
|
||||
|
|
@ -190,7 +238,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
rawDocs,
|
||||
}),
|
||||
migrationVersionPerType: this.documentMigrator.migrationVersion,
|
||||
indexPrefix: index,
|
||||
indexPrefix: indexName,
|
||||
migrationsConfig: this.soMigrationsConfig,
|
||||
typeRegistry: this.typeRegistry,
|
||||
docLinks: this.docLinks,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
|
||||
export enum TypeStatus {
|
||||
Added = 'added',
|
||||
Removed = 'removed',
|
||||
Moved = 'moved',
|
||||
Untouched = 'untouched',
|
||||
}
|
||||
|
||||
export interface TypeStatusDetails {
|
||||
currentIndex?: string;
|
||||
targetIndex?: string;
|
||||
status: TypeStatus;
|
||||
}
|
||||
|
||||
// ensure plugins don't try to convert SO namespaceTypes after 8.0.0
|
||||
// see https://github.com/elastic/kibana/issues/147344
|
||||
export const ALLOWED_CONVERT_VERSION = '8.0.0';
|
||||
|
||||
export const DEFAULT_INDEX_TYPES_MAP: IndexTypesMap = {
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana': [
|
||||
'action',
|
||||
'action_task_params',
|
||||
'alert',
|
||||
'api_key_pending_invalidation',
|
||||
'apm-indices',
|
||||
'apm-server-schema',
|
||||
'apm-service-group',
|
||||
'apm-telemetry',
|
||||
'app_search_telemetry',
|
||||
'application_usage_daily',
|
||||
'application_usage_totals',
|
||||
'book',
|
||||
'canvas-element',
|
||||
'canvas-workpad',
|
||||
'canvas-workpad-template',
|
||||
'cases',
|
||||
'cases-comments',
|
||||
'cases-configure',
|
||||
'cases-connector-mappings',
|
||||
'cases-telemetry',
|
||||
'cases-user-actions',
|
||||
'config',
|
||||
'config-global',
|
||||
'connector_token',
|
||||
'core-usage-stats',
|
||||
'csp-rule-template',
|
||||
'dashboard',
|
||||
'endpoint:user-artifact',
|
||||
'endpoint:user-artifact-manifest',
|
||||
'enterprise_search_telemetry',
|
||||
'epm-packages',
|
||||
'epm-packages-assets',
|
||||
'event_loop_delays_daily',
|
||||
'exception-list',
|
||||
'exception-list-agnostic',
|
||||
'file',
|
||||
'file-upload-usage-collection-telemetry',
|
||||
'fileShare',
|
||||
'fleet-fleet-server-host',
|
||||
'fleet-message-signing-keys',
|
||||
'fleet-preconfiguration-deletion-record',
|
||||
'fleet-proxy',
|
||||
'graph-workspace',
|
||||
'guided-onboarding-guide-state',
|
||||
'guided-onboarding-plugin-state',
|
||||
'index-pattern',
|
||||
'infrastructure-monitoring-log-view',
|
||||
'infrastructure-ui-source',
|
||||
'ingest-agent-policies',
|
||||
'ingest-download-sources',
|
||||
'ingest-outputs',
|
||||
'ingest-package-policies',
|
||||
'ingest_manager_settings',
|
||||
'inventory-view',
|
||||
'kql-telemetry',
|
||||
'legacy-url-alias',
|
||||
'lens',
|
||||
'lens-ui-telemetry',
|
||||
'map',
|
||||
'metrics-explorer-view',
|
||||
'ml-job',
|
||||
'ml-module',
|
||||
'ml-trained-model',
|
||||
'monitoring-telemetry',
|
||||
'osquery-manager-usage-metric',
|
||||
'osquery-pack',
|
||||
'osquery-pack-asset',
|
||||
'osquery-saved-query',
|
||||
'query',
|
||||
'rules-settings',
|
||||
'sample-data-telemetry',
|
||||
'search',
|
||||
'search-session',
|
||||
'search-telemetry',
|
||||
'searchableList',
|
||||
'security-rule',
|
||||
'security-solution-signals-migration',
|
||||
'siem-detection-engine-rule-actions',
|
||||
'siem-ui-timeline',
|
||||
'siem-ui-timeline-note',
|
||||
'siem-ui-timeline-pinned-event',
|
||||
'slo',
|
||||
'space',
|
||||
'spaces-usage-stats',
|
||||
'synthetics-monitor',
|
||||
'synthetics-param',
|
||||
'synthetics-privates-locations',
|
||||
'tag',
|
||||
'telemetry',
|
||||
'todo',
|
||||
'ui-metric',
|
||||
'upgrade-assistant-ml-upgrade-operation',
|
||||
'upgrade-assistant-reindex-operation',
|
||||
'uptime-dynamic-settings',
|
||||
'uptime-synthetics-api-key',
|
||||
'url',
|
||||
'usage-counters',
|
||||
'visualization',
|
||||
'workplace_search_telemetry',
|
||||
],
|
||||
};
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,257 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { errors } from '@elastic/elasticsearch';
|
||||
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
import { DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants';
|
||||
import {
|
||||
calculateTypeStatuses,
|
||||
createMultiPromiseDefer,
|
||||
getIndicesInvolvedInRelocation,
|
||||
indexMapToIndexTypesMap,
|
||||
} from './kibana_migrator_utils';
|
||||
import { INDEX_MAP_BEFORE_SPLIT } from './kibana_migrator_utils.fixtures';
|
||||
|
||||
describe('createMultiPromiseDefer', () => {
|
||||
it('creates defer objects with the same Promise', () => {
|
||||
const defers = createMultiPromiseDefer(['.kibana', '.kibana_cases']);
|
||||
expect(Object.keys(defers)).toHaveLength(2);
|
||||
expect(defers['.kibana'].promise).toEqual(defers['.kibana_cases'].promise);
|
||||
expect(defers['.kibana'].resolve).not.toEqual(defers['.kibana_cases'].resolve);
|
||||
expect(defers['.kibana'].reject).not.toEqual(defers['.kibana_cases'].reject);
|
||||
});
|
||||
|
||||
it('the common Promise resolves when all defers resolve', async () => {
|
||||
const defers = createMultiPromiseDefer(['.kibana', '.kibana_cases']);
|
||||
let resolved = 0;
|
||||
Object.values(defers).forEach((defer) => defer.promise.then(() => ++resolved));
|
||||
defers['.kibana'].resolve();
|
||||
await new Promise((resolve) => setImmediate(resolve)); // next tick
|
||||
expect(resolved).toEqual(0);
|
||||
defers['.kibana_cases'].resolve();
|
||||
await new Promise((resolve) => setImmediate(resolve)); // next tick
|
||||
expect(resolved).toEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIndicesInvolvedInRelocation', () => {
|
||||
const getIndicesInvolvedInRelocationParams = () => {
|
||||
const client = elasticsearchClientMock.createElasticsearchClient();
|
||||
(client as any).child = jest.fn().mockImplementation(() => client);
|
||||
|
||||
return {
|
||||
client,
|
||||
mainIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
indexTypesMap: {},
|
||||
defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP,
|
||||
logger: loggerMock.create(),
|
||||
};
|
||||
};
|
||||
|
||||
it('tries to get the indexTypesMap from the mainIndex', async () => {
|
||||
const params = getIndicesInvolvedInRelocationParams();
|
||||
try {
|
||||
await getIndicesInvolvedInRelocation(params);
|
||||
} catch (err) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
expect(params.client.indices.getMapping).toHaveBeenCalledTimes(1);
|
||||
expect(params.client.indices.getMapping).toHaveBeenCalledWith({
|
||||
index: MAIN_SAVED_OBJECT_INDEX,
|
||||
});
|
||||
});
|
||||
|
||||
it('fails if the query to get indexTypesMap fails with critical error', async () => {
|
||||
const params = getIndicesInvolvedInRelocationParams();
|
||||
params.client.indices.getMapping.mockImplementation(() =>
|
||||
elasticsearchClientMock.createErrorTransportRequestPromise(
|
||||
new errors.ResponseError({
|
||||
statusCode: 500,
|
||||
body: {
|
||||
error: {
|
||||
type: 'error_type',
|
||||
reason: 'error_reason',
|
||||
},
|
||||
},
|
||||
warnings: [],
|
||||
headers: {},
|
||||
meta: {} as any,
|
||||
})
|
||||
)
|
||||
);
|
||||
expect(getIndicesInvolvedInRelocation(params)).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"error_type"`
|
||||
);
|
||||
});
|
||||
|
||||
it('assumes fresh deployment if the mainIndex does not exist, returns an empty list of moving types', async () => {
|
||||
const params = getIndicesInvolvedInRelocationParams();
|
||||
params.client.indices.getMapping.mockImplementation(() =>
|
||||
elasticsearchClientMock.createErrorTransportRequestPromise(
|
||||
new errors.ResponseError({
|
||||
statusCode: 404,
|
||||
body: {
|
||||
error: {
|
||||
type: 'error_type',
|
||||
reason: 'error_reason',
|
||||
},
|
||||
},
|
||||
warnings: [],
|
||||
headers: {},
|
||||
meta: {} as any,
|
||||
})
|
||||
)
|
||||
);
|
||||
expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
describe('if mainIndex exists', () => {
|
||||
describe('but it does not have an indexTypeMap stored', () => {
|
||||
it('uses the defaultIndexTypeMap and finds out which indices are involved in a relocation', async () => {
|
||||
const params = getIndicesInvolvedInRelocationParams();
|
||||
params.client.indices.getMapping.mockReturnValue(
|
||||
Promise.resolve({
|
||||
'.kibana_8.7.0_001': {
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
someType: '7997cf5a56cc02bdc9c93361bde732b0',
|
||||
},
|
||||
},
|
||||
properties: {
|
||||
someProperty: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
params.defaultIndexTypesMap = {
|
||||
'.indexA': ['type1', 'type2', 'type3'],
|
||||
'.indexB': ['type4', 'type5', 'type6'],
|
||||
};
|
||||
|
||||
params.indexTypesMap = {
|
||||
'.indexA': ['type1'], // move type2 and type 3 over to new indexC
|
||||
'.indexB': ['type4', 'type5', 'type6'], // stays the same
|
||||
'.indexC': ['type2', 'type3'],
|
||||
};
|
||||
|
||||
expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual(['.indexA', '.indexC']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('and it has an indexTypeMap stored', () => {
|
||||
it('compares stored indexTypeMap against desired one, and finds out which indices are involved in a relocation', async () => {
|
||||
const params = getIndicesInvolvedInRelocationParams();
|
||||
params.client.indices.getMapping.mockReturnValue(
|
||||
Promise.resolve({
|
||||
'.kibana_8.8.0_001': {
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
someType: '7997cf5a56cc02bdc9c93361bde732b0',
|
||||
},
|
||||
// map stored on index
|
||||
indexTypesMap: {
|
||||
'.indexA': ['type1'],
|
||||
'.indexB': ['type4', 'type5', 'type6'],
|
||||
'.indexC': ['type2', 'type3'],
|
||||
},
|
||||
},
|
||||
properties: {
|
||||
someProperty: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
// exists on index, so this one will NOT be taken into account
|
||||
params.defaultIndexTypesMap = {
|
||||
'.indexA': ['type1', 'type2', 'type3'],
|
||||
'.indexB': ['type4', 'type5', 'type6'],
|
||||
};
|
||||
|
||||
params.indexTypesMap = {
|
||||
'.indexA': ['type1'],
|
||||
'.indexB': ['type4'],
|
||||
'.indexC': ['type2', 'type3'],
|
||||
'.indexD': ['type5', 'type6'],
|
||||
};
|
||||
|
||||
expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual(['.indexB', '.indexD']);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('indexMapToIndexTypesMap', () => {
|
||||
it('converts IndexMap to IndexTypesMap', () => {
|
||||
expect(indexMapToIndexTypesMap(INDEX_MAP_BEFORE_SPLIT)).toEqual(DEFAULT_INDEX_TYPES_MAP);
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateTypeStatuses', () => {
|
||||
it('takes two indexTypesMaps and checks what types have been added, removed and relocated', () => {
|
||||
const currentIndexTypesMap = {
|
||||
'.indexA': ['type1', 'type2', 'type3'],
|
||||
'.indexB': ['type4', 'type5', 'type6'],
|
||||
};
|
||||
const desiredIndexTypesMap = {
|
||||
'.indexA': ['type2'],
|
||||
'.indexB': ['type3', 'type5'],
|
||||
'.indexC': ['type4', 'type6', 'type7'],
|
||||
'.indexD': ['type8'],
|
||||
};
|
||||
|
||||
expect(calculateTypeStatuses(currentIndexTypesMap, desiredIndexTypesMap)).toEqual({
|
||||
type1: {
|
||||
currentIndex: '.indexA',
|
||||
status: 'removed',
|
||||
},
|
||||
type2: {
|
||||
currentIndex: '.indexA',
|
||||
status: 'untouched',
|
||||
targetIndex: '.indexA',
|
||||
},
|
||||
type3: {
|
||||
currentIndex: '.indexA',
|
||||
status: 'moved',
|
||||
targetIndex: '.indexB',
|
||||
},
|
||||
type4: {
|
||||
currentIndex: '.indexB',
|
||||
status: 'moved',
|
||||
targetIndex: '.indexC',
|
||||
},
|
||||
type5: {
|
||||
currentIndex: '.indexB',
|
||||
status: 'untouched',
|
||||
targetIndex: '.indexB',
|
||||
},
|
||||
type6: {
|
||||
currentIndex: '.indexB',
|
||||
status: 'moved',
|
||||
targetIndex: '.indexC',
|
||||
},
|
||||
type7: {
|
||||
status: 'added',
|
||||
targetIndex: '.indexC',
|
||||
},
|
||||
type8: {
|
||||
status: 'added',
|
||||
targetIndex: '.indexD',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { IndexMap } from './core';
|
||||
import { TypeStatus, type TypeStatusDetails } from './kibana_migrator_constants';
|
||||
|
||||
// even though this utility class is present in @kbn/kibana-utils-plugin, we can't easily import it from Core
|
||||
// aka. one does not simply reuse code
|
||||
export class Defer<T> {
|
||||
public resolve!: (data: T) => void;
|
||||
public reject!: (error: any) => void;
|
||||
public promise: Promise<any> = new Promise<any>((resolve, reject) => {
|
||||
(this as any).resolve = resolve;
|
||||
(this as any).reject = reject;
|
||||
});
|
||||
}
|
||||
|
||||
export const defer = () => new Defer<void>();
|
||||
|
||||
export function createMultiPromiseDefer(indices: string[]): Record<string, Defer<void>> {
|
||||
const defers: Array<Defer<void>> = indices.map(defer);
|
||||
const all = Promise.all(defers.map(({ promise }) => promise));
|
||||
return indices.reduce<Record<string, Defer<any>>>((acc, indexName, i) => {
|
||||
const { resolve, reject } = defers[i];
|
||||
acc[indexName] = { resolve, reject, promise: all };
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
export async function getCurrentIndexTypesMap({
|
||||
client,
|
||||
mainIndex,
|
||||
defaultIndexTypesMap,
|
||||
logger,
|
||||
}: {
|
||||
client: ElasticsearchClient;
|
||||
mainIndex: string;
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
logger: Logger;
|
||||
}): Promise<IndexTypesMap | undefined> {
|
||||
try {
|
||||
// check if the main index (i.e. .kibana) exists
|
||||
const mapping = await client.indices.getMapping({
|
||||
index: mainIndex,
|
||||
});
|
||||
|
||||
// main index exists, try to extract the indexTypesMap from _meta
|
||||
const meta = Object.values(mapping)?.[0]?.mappings._meta;
|
||||
return meta?.indexTypesMap ?? defaultIndexTypesMap;
|
||||
} catch (error) {
|
||||
if (error.meta?.statusCode === 404) {
|
||||
logger.debug(`The ${mainIndex} index do NOT exist. Assuming this is a fresh deployment`);
|
||||
return undefined;
|
||||
} else {
|
||||
logger.fatal(`Cannot query the meta information on the ${mainIndex} saved object index`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function getIndicesInvolvedInRelocation({
|
||||
client,
|
||||
mainIndex,
|
||||
indexTypesMap,
|
||||
defaultIndexTypesMap,
|
||||
logger,
|
||||
}: {
|
||||
client: ElasticsearchClient;
|
||||
mainIndex: string;
|
||||
indexTypesMap: IndexTypesMap;
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
logger: Logger;
|
||||
}): Promise<string[]> {
|
||||
const indicesWithMovingTypesSet = new Set<string>();
|
||||
|
||||
const currentIndexTypesMap = await getCurrentIndexTypesMap({
|
||||
client,
|
||||
mainIndex,
|
||||
defaultIndexTypesMap,
|
||||
logger,
|
||||
});
|
||||
|
||||
if (!currentIndexTypesMap) {
|
||||
// this is a fresh deployment, no indices must be relocated
|
||||
return [];
|
||||
}
|
||||
|
||||
const typeIndexDistribution = calculateTypeStatuses(currentIndexTypesMap, indexTypesMap);
|
||||
|
||||
Object.values(typeIndexDistribution)
|
||||
.filter(({ status }) => status === TypeStatus.Moved)
|
||||
.forEach(({ currentIndex, targetIndex }) => {
|
||||
indicesWithMovingTypesSet.add(currentIndex!);
|
||||
indicesWithMovingTypesSet.add(targetIndex!);
|
||||
});
|
||||
|
||||
return Array.from(indicesWithMovingTypesSet);
|
||||
}
|
||||
|
||||
export function indexMapToIndexTypesMap(indexMap: IndexMap): IndexTypesMap {
|
||||
return Object.entries(indexMap).reduce<IndexTypesMap>((acc, [indexAlias, { typeMappings }]) => {
|
||||
acc[indexAlias] = Object.keys(typeMappings).sort();
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
export function calculateTypeStatuses(
|
||||
currentIndexTypesMap: IndexTypesMap,
|
||||
desiredIndexTypesMap: IndexTypesMap
|
||||
): Record<string, TypeStatusDetails> {
|
||||
const statuses: Record<string, TypeStatusDetails> = {};
|
||||
|
||||
Object.entries(currentIndexTypesMap).forEach(([currentIndex, types]) => {
|
||||
types.forEach((type) => {
|
||||
statuses[type] = {
|
||||
currentIndex,
|
||||
status: TypeStatus.Removed, // type is removed unless we still have it
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
Object.entries(desiredIndexTypesMap).forEach(([targetIndex, types]) => {
|
||||
types.forEach((type) => {
|
||||
if (!statuses[type]) {
|
||||
statuses[type] = {
|
||||
targetIndex,
|
||||
status: TypeStatus.Added, // type didn't exist, it must be new
|
||||
};
|
||||
} else {
|
||||
statuses[type].targetIndex = targetIndex;
|
||||
statuses[type].status =
|
||||
statuses[type].currentIndex === targetIndex ? TypeStatus.Untouched : TypeStatus.Moved;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return statuses;
|
||||
}
|
||||
|
|
@ -6,13 +6,11 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { cleanupMock } from './migrations_state_machine_cleanup.mocks';
|
||||
import { migrationStateActionMachine } from './migrations_state_action_machine';
|
||||
import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
|
||||
import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
|
||||
import { LoggerAdapter } from '@kbn/core-logging-server-internal';
|
||||
import { typeRegistryMock } from '@kbn/core-saved-objects-base-server-mocks';
|
||||
import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
|
||||
import * as Either from 'fp-ts/lib/Either';
|
||||
import * as Option from 'fp-ts/lib/Option';
|
||||
import { errors } from '@elastic/elasticsearch';
|
||||
|
|
@ -21,8 +19,6 @@ import type { AllControlStates, State } from './state';
|
|||
import { createInitialState } from './initial_state';
|
||||
import { ByteSizeValue } from '@kbn/config-schema';
|
||||
|
||||
const esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
|
||||
describe('migrationsStateActionMachine', () => {
|
||||
beforeAll(() => {
|
||||
jest
|
||||
|
|
@ -33,6 +29,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
const abort = jest.fn();
|
||||
const mockLogger = loggingSystemMock.create();
|
||||
const typeRegistry = typeRegistryMock.create();
|
||||
const docLinks = docLinksServiceMock.createSetupContract();
|
||||
|
|
@ -40,6 +37,12 @@ describe('migrationsStateActionMachine', () => {
|
|||
const initialState = createInitialState({
|
||||
kibanaVersion: '7.11.0',
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: true,
|
||||
indexTypesMap: {
|
||||
'.kibana': ['typeA', 'typeB', 'typeC'],
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana_cases': ['typeD', 'typeE'],
|
||||
},
|
||||
targetMappings: { properties: {} },
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.my-so-index',
|
||||
|
|
@ -92,7 +95,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
});
|
||||
const logs = loggingSystemMock.collect(mockLogger);
|
||||
const doneLog = logs.info.splice(8, 1)[0][0];
|
||||
|
|
@ -113,7 +116,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_DELETE', 'FATAL']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
}).catch((err) => err);
|
||||
expect(loggingSystemMock.collect(mockLogger)).toMatchSnapshot();
|
||||
});
|
||||
|
|
@ -129,7 +132,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger,
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).resolves.toEqual(expect.anything());
|
||||
|
||||
|
|
@ -155,7 +158,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).resolves.toEqual(expect.anything());
|
||||
});
|
||||
|
|
@ -167,7 +170,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).resolves.toEqual(expect.objectContaining({ status: 'migrated' }));
|
||||
});
|
||||
|
|
@ -179,7 +182,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).resolves.toEqual(expect.objectContaining({ status: 'patched' }));
|
||||
});
|
||||
|
|
@ -191,7 +194,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Unable to complete saved object migrations for the [.my-so-index] index: the fatal reason]`
|
||||
|
|
@ -219,7 +222,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
})
|
||||
);
|
||||
},
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Unable to complete saved object migrations for the [.my-so-index] index. Please check the health of your Elasticsearch cluster and try again. Unexpected Elasticsearch ResponseError: statusCode: 200, method: POST, url: /mock error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted,]`
|
||||
|
|
@ -249,7 +252,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
next: () => {
|
||||
throw new Error('this action throws');
|
||||
},
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Unable to complete saved object migrations for the [.my-so-index] index. Error: this action throws]`
|
||||
|
|
@ -271,10 +274,7 @@ describe('migrationsStateActionMachine', () => {
|
|||
`);
|
||||
});
|
||||
describe('cleanup', () => {
|
||||
beforeEach(() => {
|
||||
cleanupMock.mockClear();
|
||||
});
|
||||
it('calls cleanup function when an action throws', async () => {
|
||||
it('calls abort function when an action throws', async () => {
|
||||
await expect(
|
||||
migrationStateActionMachine({
|
||||
initialState: { ...initialState, reason: 'the fatal reason' } as State,
|
||||
|
|
@ -283,24 +283,24 @@ describe('migrationsStateActionMachine', () => {
|
|||
next: () => {
|
||||
throw new Error('this action throws');
|
||||
},
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(cleanupMock).toHaveBeenCalledTimes(1);
|
||||
expect(abort).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('calls cleanup function when reaching the FATAL state', async () => {
|
||||
it('calls abort function when reaching the FATAL state', async () => {
|
||||
await expect(
|
||||
migrationStateActionMachine({
|
||||
initialState: { ...initialState, reason: 'the fatal reason' } as State,
|
||||
logger: mockLogger.get(),
|
||||
model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']),
|
||||
next,
|
||||
client: esClient,
|
||||
abort,
|
||||
})
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(cleanupMock).toHaveBeenCalledTimes(1);
|
||||
expect(abort).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,15 +9,14 @@
|
|||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
import * as Option from 'fp-ts/lib/Option';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import {
|
||||
getErrorMessage,
|
||||
getRequestDebugMeta,
|
||||
} from '@kbn/core-elasticsearch-client-server-internal';
|
||||
import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server';
|
||||
import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { logActionResponse, logStateTransition } from './common/utils/logs';
|
||||
import { type Model, type Next, stateActionMachine } from './state_action_machine';
|
||||
import { cleanup } from './migrations_state_machine_cleanup';
|
||||
import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state';
|
||||
import { redactBulkOperationBatches } from './common/redact_state';
|
||||
|
||||
|
|
@ -35,14 +34,14 @@ export async function migrationStateActionMachine({
|
|||
logger,
|
||||
next,
|
||||
model,
|
||||
client,
|
||||
abort,
|
||||
}: {
|
||||
initialState: State;
|
||||
logger: Logger;
|
||||
next: Next<State>;
|
||||
model: Model<State>;
|
||||
client: ElasticsearchClient;
|
||||
}) {
|
||||
abort: (state?: State) => Promise<void>;
|
||||
}): Promise<MigrationResult> {
|
||||
const startTime = Date.now();
|
||||
// Since saved object index names usually start with a `.` and can be
|
||||
// configured by users to include several `.`'s we can't use a logger tag to
|
||||
|
|
@ -112,22 +111,28 @@ export async function migrationStateActionMachine({
|
|||
}
|
||||
} else if (finalState.controlState === 'FATAL') {
|
||||
try {
|
||||
await cleanup(client, finalState);
|
||||
await abort(finalState);
|
||||
} catch (e) {
|
||||
logger.warn('Failed to cleanup after migrations:', e.message);
|
||||
}
|
||||
return Promise.reject(
|
||||
new Error(
|
||||
`Unable to complete saved object migrations for the [${initialState.indexPrefix}] index: ` +
|
||||
finalState.reason
|
||||
)
|
||||
);
|
||||
|
||||
const errorMessage =
|
||||
`Unable to complete saved object migrations for the [${initialState.indexPrefix}] index: ` +
|
||||
finalState.reason;
|
||||
|
||||
if (finalState.throwDelayMillis) {
|
||||
return new Promise((_, reject) =>
|
||||
setTimeout(() => reject(errorMessage), finalState.throwDelayMillis)
|
||||
);
|
||||
}
|
||||
|
||||
return Promise.reject(new Error(errorMessage));
|
||||
} else {
|
||||
throw new Error('Invalid terminating control state');
|
||||
}
|
||||
} catch (e) {
|
||||
try {
|
||||
await cleanup(client, lastState);
|
||||
await abort(lastState);
|
||||
} catch (err) {
|
||||
logger.warn('Failed to cleanup after migrations:', err.message);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
*/
|
||||
import * as Either from 'fp-ts/lib/Either';
|
||||
import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server';
|
||||
import { createBatches } from './create_batches';
|
||||
import { buildTempIndexMap, createBatches } from './create_batches';
|
||||
|
||||
describe('createBatches', () => {
|
||||
const documentToOperation = (document: SavedObjectsRawDoc) => [
|
||||
|
|
@ -17,56 +17,145 @@ describe('createBatches', () => {
|
|||
|
||||
const DOCUMENT_SIZE_BYTES = 77; // 76 + \n
|
||||
|
||||
it('returns right one batch if all documents fit in maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
];
|
||||
describe('when indexTypesMap and kibanaVersion are not provided', () => {
|
||||
it('returns right one batch if all documents fit in maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
];
|
||||
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 3 })).toEqual(
|
||||
Either.right([documents.map(documentToOperation)])
|
||||
);
|
||||
});
|
||||
it('creates multiple batches with each batch limited to maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title 44' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title 55' } },
|
||||
];
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 2 })).toEqual(
|
||||
Either.right([
|
||||
documents.slice(0, 2).map(documentToOperation),
|
||||
documents.slice(2, 4).map(documentToOperation),
|
||||
documents.slice(4).map(documentToOperation),
|
||||
])
|
||||
);
|
||||
});
|
||||
it('creates a single empty batch if there are no documents', () => {
|
||||
const documents = [] as SavedObjectsRawDoc[];
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: 100 })).toEqual(Either.right([[]]));
|
||||
});
|
||||
it('throws if any one document exceeds the maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: 'foo', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{
|
||||
_id: 'bar',
|
||||
_source: {
|
||||
type: 'dashboard',
|
||||
title: 'my saved object title ² with a very long title that exceeds max size bytes',
|
||||
expect(
|
||||
createBatches({
|
||||
documents,
|
||||
maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 3,
|
||||
})
|
||||
).toEqual(Either.right([documents.map(documentToOperation)]));
|
||||
});
|
||||
it('creates multiple batches with each batch limited to maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title 44' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title 55' } },
|
||||
];
|
||||
expect(
|
||||
createBatches({
|
||||
documents,
|
||||
maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 2,
|
||||
})
|
||||
).toEqual(
|
||||
Either.right([
|
||||
documents.slice(0, 2).map(documentToOperation),
|
||||
documents.slice(2, 4).map(documentToOperation),
|
||||
documents.slice(4).map(documentToOperation),
|
||||
])
|
||||
);
|
||||
});
|
||||
it('creates a single empty batch if there are no documents', () => {
|
||||
const documents = [] as SavedObjectsRawDoc[];
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: 100 })).toEqual(Either.right([[]]));
|
||||
});
|
||||
it('throws if any one document exceeds the maxBatchSizeBytes', () => {
|
||||
const documents = [
|
||||
{ _id: 'foo', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{
|
||||
_id: 'bar',
|
||||
_source: {
|
||||
type: 'dashboard',
|
||||
title: 'my saved object title ² with a very long title that exceeds max size bytes',
|
||||
},
|
||||
},
|
||||
},
|
||||
{ _id: 'baz', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
];
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: 120 })).toEqual(
|
||||
Either.left({
|
||||
maxBatchSizeBytes: 120,
|
||||
docSizeBytes: 130,
|
||||
type: 'document_exceeds_batch_size_bytes',
|
||||
documentId: documents[1]._id,
|
||||
})
|
||||
);
|
||||
{ _id: 'baz', _source: { type: 'dashboard', title: 'my saved object title ®' } },
|
||||
];
|
||||
expect(createBatches({ documents, maxBatchSizeBytes: 120 })).toEqual(
|
||||
Either.left({
|
||||
maxBatchSizeBytes: 120,
|
||||
docSizeBytes: 130,
|
||||
type: 'document_exceeds_batch_size_bytes',
|
||||
documentId: documents[1]._id,
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('when a type index map is provided', () => {
|
||||
it('creates batches that contain the target index information for each type', () => {
|
||||
const documents = [
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } },
|
||||
{ _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } },
|
||||
{ _id: '', _source: { type: 'cases', title: 'a case' } },
|
||||
{ _id: '', _source: { type: 'cases-comments', title: 'a case comment #1' } },
|
||||
{ _id: '', _source: { type: 'cases-user-actions', title: 'a case user action' } },
|
||||
];
|
||||
expect(
|
||||
createBatches({
|
||||
documents,
|
||||
maxBatchSizeBytes: (DOCUMENT_SIZE_BYTES + 43) * 2, // add extra length for 'index' property
|
||||
typeIndexMap: buildTempIndexMap(
|
||||
{
|
||||
'.kibana': ['dashboard'],
|
||||
'.kibana_cases': ['cases', 'cases-comments', 'cases-user-actions'],
|
||||
},
|
||||
'8.8.0'
|
||||
),
|
||||
})
|
||||
).toEqual(
|
||||
Either.right([
|
||||
[
|
||||
[
|
||||
{
|
||||
index: {
|
||||
_id: '',
|
||||
_index: '.kibana_8.8.0_reindex_temp',
|
||||
},
|
||||
},
|
||||
{ type: 'dashboard', title: 'my saved object title ¹' },
|
||||
],
|
||||
[
|
||||
{
|
||||
index: {
|
||||
_id: '',
|
||||
_index: '.kibana_8.8.0_reindex_temp',
|
||||
},
|
||||
},
|
||||
{ type: 'dashboard', title: 'my saved object title ²' },
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
{
|
||||
index: {
|
||||
_id: '',
|
||||
_index: '.kibana_cases_8.8.0_reindex_temp',
|
||||
},
|
||||
},
|
||||
{ type: 'cases', title: 'a case' },
|
||||
],
|
||||
[
|
||||
{
|
||||
index: {
|
||||
_id: '',
|
||||
_index: '.kibana_cases_8.8.0_reindex_temp',
|
||||
},
|
||||
},
|
||||
{ type: 'cases-comments', title: 'a case comment #1' },
|
||||
],
|
||||
],
|
||||
[
|
||||
[
|
||||
{
|
||||
index: {
|
||||
_id: '',
|
||||
_index: '.kibana_cases_8.8.0_reindex_temp',
|
||||
},
|
||||
},
|
||||
{ type: 'cases-user-actions', title: 'a case user action' },
|
||||
],
|
||||
],
|
||||
])
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,7 +9,12 @@
|
|||
import * as Either from 'fp-ts/lib/Either';
|
||||
import type { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '@kbn/core-saved-objects-server';
|
||||
import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { createBulkDeleteOperationBody, createBulkIndexOperationTuple } from './helpers';
|
||||
import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import {
|
||||
createBulkDeleteOperationBody,
|
||||
createBulkIndexOperationTuple,
|
||||
getTempIndexName,
|
||||
} from './helpers';
|
||||
import type { TransformErrorObjects } from '../core';
|
||||
|
||||
export type BulkIndexOperationTuple = [BulkOperationContainer, SavedObjectsRawDocSource];
|
||||
|
|
@ -21,6 +26,12 @@ export interface CreateBatchesParams {
|
|||
corruptDocumentIds?: string[];
|
||||
transformErrors?: TransformErrorObjects[];
|
||||
maxBatchSizeBytes: number;
|
||||
/** This map holds a list of temporary index names for each SO type, e.g.:
|
||||
* 'cases': '.kibana_cases_8.8.0_reindex_temp'
|
||||
* 'task': '.kibana_task_manager_8.8.0_reindex_temp'
|
||||
* ...
|
||||
*/
|
||||
typeIndexMap?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface DocumentExceedsBatchSize {
|
||||
|
|
@ -30,6 +41,32 @@ export interface DocumentExceedsBatchSize {
|
|||
maxBatchSizeBytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a relationship of temporary index names for each SO type, e.g.:
|
||||
* 'cases': '.kibana_cases_8.8.0_reindex_temp'
|
||||
* 'task': '.kibana_task_manager_8.8.0_reindex_temp'
|
||||
* ...
|
||||
*
|
||||
* @param indexTypesMap information about which types are stored in each index
|
||||
* @param kibanaVersion the target version of the indices
|
||||
*/
|
||||
export function buildTempIndexMap(
|
||||
indexTypesMap: IndexTypesMap,
|
||||
kibanaVersion: string
|
||||
): Record<string, string> {
|
||||
return Object.entries(indexTypesMap || {}).reduce<Record<string, string>>(
|
||||
(acc, [indexAlias, types]) => {
|
||||
const tempIndex = getTempIndexName(indexAlias, kibanaVersion!);
|
||||
|
||||
types.forEach((type) => {
|
||||
acc[type] = tempIndex;
|
||||
});
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates batches of documents to be used by the bulk API. Each batch will
|
||||
* have a request body content length that's <= maxBatchSizeBytes
|
||||
|
|
@ -39,6 +76,7 @@ export function createBatches({
|
|||
corruptDocumentIds = [],
|
||||
transformErrors = [],
|
||||
maxBatchSizeBytes,
|
||||
typeIndexMap,
|
||||
}: CreateBatchesParams): Either.Either<DocumentExceedsBatchSize, BulkOperation[][]> {
|
||||
/* To build up the NDJSON request body we construct an array of objects like:
|
||||
* [
|
||||
|
|
@ -92,7 +130,7 @@ export function createBatches({
|
|||
|
||||
// create index (update) operations for all transformed documents
|
||||
for (const document of documents) {
|
||||
const bulkIndexOperationBody = createBulkIndexOperationTuple(document);
|
||||
const bulkIndexOperationBody = createBulkIndexOperationTuple(document, typeIndexMap);
|
||||
// take into account that this tuple's surrounding brackets `[]` won't be present in the NDJSON
|
||||
const docSizeBytes =
|
||||
Buffer.byteLength(JSON.stringify(bulkIndexOperationBody), 'utf8') - BRACKETS_BYTES;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import {
|
|||
buildRemoveAliasActions,
|
||||
versionMigrationCompleted,
|
||||
MigrationType,
|
||||
getTempIndexName,
|
||||
createBulkIndexOperationTuple,
|
||||
} from './helpers';
|
||||
|
||||
describe('addExcludedTypesToBoolQuery', () => {
|
||||
|
|
@ -290,6 +292,46 @@ describe('buildRemoveAliasActions', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('createBulkIndexOperationTuple', () => {
|
||||
it('creates the proper request body to bulk index a document', () => {
|
||||
const document = { _id: '', _source: { type: 'cases', title: 'a case' } };
|
||||
const typeIndexMap = {
|
||||
cases: '.kibana_cases_8.8.0_reindex_temp',
|
||||
};
|
||||
expect(createBulkIndexOperationTuple(document, typeIndexMap)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"index": Object {
|
||||
"_id": "",
|
||||
"_index": ".kibana_cases_8.8.0_reindex_temp",
|
||||
},
|
||||
},
|
||||
Object {
|
||||
"title": "a case",
|
||||
"type": "cases",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it('does not include the index property if it is not specified in the typeIndexMap', () => {
|
||||
const document = { _id: '', _source: { type: 'cases', title: 'a case' } };
|
||||
expect(createBulkIndexOperationTuple(document)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"index": Object {
|
||||
"_id": "",
|
||||
},
|
||||
},
|
||||
Object {
|
||||
"title": "a case",
|
||||
"type": "cases",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMigrationType', () => {
|
||||
it.each`
|
||||
isMappingsCompatible | isVersionMigrationCompleted | expected
|
||||
|
|
@ -306,3 +348,9 @@ describe('getMigrationType', () => {
|
|||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('getTempIndexName', () => {
|
||||
it('composes a temporary index name for reindexing', () => {
|
||||
expect(getTempIndexName('.kibana_cases', '8.8.0')).toEqual('.kibana_cases_8.8.0_reindex_temp');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ export function mergeMigrationMappingPropertyHashes(
|
|||
return {
|
||||
...targetMappings,
|
||||
_meta: {
|
||||
...targetMappings._meta,
|
||||
migrationMappingPropertyHashes: {
|
||||
...indexMappings._meta?.migrationMappingPropertyHashes,
|
||||
...targetMappings._meta?.migrationMappingPropertyHashes,
|
||||
|
|
@ -218,11 +219,15 @@ export function buildRemoveAliasActions(
|
|||
/**
|
||||
* Given a document, creates a valid body to index the document using the Bulk API.
|
||||
*/
|
||||
export const createBulkIndexOperationTuple = (doc: SavedObjectsRawDoc): BulkIndexOperationTuple => {
|
||||
export const createBulkIndexOperationTuple = (
|
||||
doc: SavedObjectsRawDoc,
|
||||
typeIndexMap: Record<string, string> = {}
|
||||
): BulkIndexOperationTuple => {
|
||||
return [
|
||||
{
|
||||
index: {
|
||||
_id: doc._id,
|
||||
...(typeIndexMap[doc._source.type] && { _index: typeIndexMap[doc._source.type] }),
|
||||
// use optimistic concurrency control to ensure that outdated
|
||||
// documents are only overwritten once with the latest version
|
||||
...(typeof doc._seq_no !== 'undefined' && { if_seq_no: doc._seq_no }),
|
||||
|
|
@ -271,3 +276,12 @@ export function getMigrationType({
|
|||
|
||||
return MigrationType.Invalid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a temporary index name, to reindex documents into it
|
||||
* @param index The name of the SO index
|
||||
* @param kibanaVersion The current kibana version
|
||||
* @returns A temporary index name to reindex documents
|
||||
*/
|
||||
export const getTempIndexName = (indexPrefix: string, kibanaVersion: string): string =>
|
||||
`${indexPrefix}_${kibanaVersion}_reindex_temp`;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import type {
|
|||
CheckVersionIndexReadyActions,
|
||||
CleanupUnknownAndExcluded,
|
||||
CleanupUnknownAndExcludedWaitForTaskState,
|
||||
CloneTempToSource,
|
||||
CloneTempToTarget,
|
||||
CreateNewTargetState,
|
||||
CreateReindexTempState,
|
||||
FatalState,
|
||||
|
|
@ -51,6 +51,8 @@ import type {
|
|||
UpdateTargetMappingsPropertiesState,
|
||||
UpdateTargetMappingsPropertiesWaitForTaskState,
|
||||
WaitForYellowSourceState,
|
||||
ReadyToReindexSyncState,
|
||||
DoneReindexingSyncState,
|
||||
} from '../state';
|
||||
import { type TransformErrorObjects, TransformSavedObjectDocumentError } from '../core';
|
||||
import type { AliasAction, RetryableEsClientError } from '../actions';
|
||||
|
|
@ -58,6 +60,7 @@ import type { ResponseType } from '../next';
|
|||
import { createInitialProgress } from './progress';
|
||||
import { model } from './model';
|
||||
import type { BulkIndexOperationTuple, BulkOperation } from './create_batches';
|
||||
import { DEFAULT_INDEX_TYPES_MAP } from '../kibana_migrator_constants';
|
||||
|
||||
describe('migrations v2 model', () => {
|
||||
const indexMapping: IndexMapping = {
|
||||
|
|
@ -115,6 +118,8 @@ describe('migrations v2 model', () => {
|
|||
clusterShardLimitExceeded: 'clusterShardLimitExceeded',
|
||||
},
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: false,
|
||||
indexTypesMap: DEFAULT_INDEX_TYPES_MAP,
|
||||
};
|
||||
const postInitState = {
|
||||
...baseState,
|
||||
|
|
@ -732,7 +737,7 @@ describe('migrations v2 model', () => {
|
|||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
test('INIT -> CREATE_NEW_TARGET when no indices/aliases exist', () => {
|
||||
test('INIT -> CREATE_NEW_TARGET when the index does not exist and the migrator is NOT involved in a relocation', () => {
|
||||
const res: ResponseType<'INIT'> = Either.right({});
|
||||
const newState = model(initState, res);
|
||||
|
||||
|
|
@ -744,6 +749,29 @@ describe('migrations v2 model', () => {
|
|||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
test('INIT -> CREATE_REINDEX_TEMP when the index does not exist and the migrator is involved in a relocation', () => {
|
||||
const res: ResponseType<'INIT'> = Either.right({});
|
||||
const newState = model(
|
||||
{
|
||||
...initState,
|
||||
mustRelocateDocuments: true,
|
||||
},
|
||||
res
|
||||
);
|
||||
|
||||
expect(newState).toMatchObject({
|
||||
controlState: 'CREATE_REINDEX_TEMP',
|
||||
sourceIndex: Option.none,
|
||||
targetIndex: '.kibana_7.11.0_001',
|
||||
versionIndexReadyActions: Option.some([
|
||||
{ add: { index: '.kibana_7.11.0_001', alias: '.kibana' } },
|
||||
{ add: { index: '.kibana_7.11.0_001', alias: '.kibana_7.11.0' } },
|
||||
{ remove_index: { index: '.kibana_7.11.0_reindex_temp' } },
|
||||
]),
|
||||
});
|
||||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1146,12 +1174,29 @@ describe('migrations v2 model', () => {
|
|||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
|
||||
test('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES', () => {
|
||||
const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({});
|
||||
const newState = model(waitForYellowSourceState, res);
|
||||
describe('if the migrator is NOT involved in a relocation', () => {
|
||||
test('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES', () => {
|
||||
const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({});
|
||||
const newState = model(waitForYellowSourceState, res);
|
||||
|
||||
expect(newState).toMatchObject({
|
||||
controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES',
|
||||
expect(newState).toMatchObject({
|
||||
controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('if the migrator is involved in a relocation', () => {
|
||||
// no need to attempt to update the mappings, we are going to reindex
|
||||
test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS', () => {
|
||||
const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({});
|
||||
const newState = model(
|
||||
{ ...waitForYellowSourceState, mustRelocateDocuments: true },
|
||||
res
|
||||
);
|
||||
|
||||
expect(newState).toMatchObject({
|
||||
controlState: 'CHECK_UNKNOWN_DOCUMENTS',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1630,13 +1675,27 @@ describe('migrations v2 model', () => {
|
|||
sourceIndexMappings: Option.some({}) as Option.Some<IndexMapping>,
|
||||
tempIndexMappings: { properties: {} },
|
||||
};
|
||||
it('CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT if action succeeds', () => {
|
||||
const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded');
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT');
|
||||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
|
||||
describe('if the migrator is NOT involved in a relocation', () => {
|
||||
it('CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT if action succeeds', () => {
|
||||
const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded');
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT');
|
||||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('if the migrator is involved in a relocation', () => {
|
||||
it('CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC if action succeeds', () => {
|
||||
const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded');
|
||||
const newState = model({ ...state, mustRelocateDocuments: true }, res);
|
||||
expect(newState.controlState).toEqual('READY_TO_REINDEX_SYNC');
|
||||
expect(newState.retryCount).toEqual(0);
|
||||
expect(newState.retryDelay).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
it('CREATE_REINDEX_TEMP -> CREATE_REINDEX_TEMP if action fails with index_not_green_timeout', () => {
|
||||
const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.left({
|
||||
message: '[index_not_green_timeout] Timeout waiting for ...',
|
||||
|
|
@ -1677,6 +1736,52 @@ describe('migrations v2 model', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('READY_TO_REINDEX_SYNC', () => {
|
||||
const state: ReadyToReindexSyncState = {
|
||||
...postInitState,
|
||||
controlState: 'READY_TO_REINDEX_SYNC',
|
||||
};
|
||||
|
||||
describe('if the migrator source index did NOT exist', () => {
|
||||
test('READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC', () => {
|
||||
const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.right(
|
||||
'synchronized_successfully' as const
|
||||
);
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('DONE_REINDEXING_SYNC');
|
||||
});
|
||||
});
|
||||
|
||||
describe('if the migrator source index did exist', () => {
|
||||
test('READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT', () => {
|
||||
const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.right(
|
||||
'synchronized_successfully' as const
|
||||
);
|
||||
const newState = model(
|
||||
{
|
||||
...state,
|
||||
sourceIndex: Option.fromNullable('.kibana'),
|
||||
sourceIndexMappings: Option.fromNullable({} as IndexMapping),
|
||||
},
|
||||
res
|
||||
);
|
||||
expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT');
|
||||
});
|
||||
});
|
||||
|
||||
test('READY_TO_REINDEX_SYNC -> FATAL if the synchronization between migrators fails', () => {
|
||||
const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.left({
|
||||
type: 'sync_failed',
|
||||
error: new Error('Other migrators failed to reach the synchronization point'),
|
||||
});
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('FATAL');
|
||||
expect((newState as FatalState).reason).toMatchInlineSnapshot(
|
||||
`"An error occurred whilst waiting for other migrators to get to this step."`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('REINDEX_SOURCE_TO_TEMP_OPEN_PIT', () => {
|
||||
const state: ReindexSourceToTempOpenPit = {
|
||||
...postInitState,
|
||||
|
|
@ -1812,11 +1917,50 @@ describe('migrations v2 model', () => {
|
|||
tempIndexMappings: { properties: {} },
|
||||
};
|
||||
|
||||
it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK if action succeeded', () => {
|
||||
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({});
|
||||
const newState = model(state, res) as ReindexSourceToTempTransform;
|
||||
expect(newState.controlState).toBe('SET_TEMP_WRITE_BLOCK');
|
||||
expect(newState.sourceIndex).toEqual(state.sourceIndex);
|
||||
describe('if the migrator is NOT involved in a relocation', () => {
|
||||
it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK if action succeeded', () => {
|
||||
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({});
|
||||
const newState = model(state, res) as ReindexSourceToTempTransform;
|
||||
expect(newState.controlState).toBe('SET_TEMP_WRITE_BLOCK');
|
||||
expect(newState.sourceIndex).toEqual(state.sourceIndex);
|
||||
});
|
||||
});
|
||||
|
||||
describe('if the migrator is involved in a relocation', () => {
|
||||
it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC if action succeeded', () => {
|
||||
const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({});
|
||||
const newState = model(
|
||||
{ ...state, mustRelocateDocuments: true },
|
||||
res
|
||||
) as ReindexSourceToTempTransform;
|
||||
expect(newState.controlState).toBe('DONE_REINDEXING_SYNC');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('DONE_REINDEXING_SYNC', () => {
|
||||
const state: DoneReindexingSyncState = {
|
||||
...postInitState,
|
||||
controlState: 'DONE_REINDEXING_SYNC',
|
||||
};
|
||||
|
||||
test('DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK if synchronization succeeds', () => {
|
||||
const res: ResponseType<'DONE_REINDEXING_SYNC'> = Either.right(
|
||||
'synchronized_successfully' as const
|
||||
);
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('SET_TEMP_WRITE_BLOCK');
|
||||
});
|
||||
test('DONE_REINDEXING_SYNC -> FATAL if the synchronization between migrators fails', () => {
|
||||
const res: ResponseType<'DONE_REINDEXING_SYNC'> = Either.left({
|
||||
type: 'sync_failed',
|
||||
error: new Error('Other migrators failed to reach the synchronization point'),
|
||||
});
|
||||
const newState = model(state, res);
|
||||
expect(newState.controlState).toEqual('FATAL');
|
||||
expect((newState as FatalState).reason).toMatchInlineSnapshot(
|
||||
`"An error occurred whilst waiting for other migrators to get to this step."`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1977,7 +2121,7 @@ describe('migrations v2 model', () => {
|
|||
});
|
||||
|
||||
describe('CLONE_TEMP_TO_TARGET', () => {
|
||||
const state: CloneTempToSource = {
|
||||
const state: CloneTempToTarget = {
|
||||
...postInitState,
|
||||
controlState: 'CLONE_TEMP_TO_TARGET',
|
||||
sourceIndex: Option.some('.kibana') as Option.Some<string>,
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ import {
|
|||
buildRemoveAliasActions,
|
||||
MigrationType,
|
||||
} from './helpers';
|
||||
import { createBatches } from './create_batches';
|
||||
import { buildTempIndexMap, createBatches } from './create_batches';
|
||||
import type { MigrationLog } from '../types';
|
||||
import {
|
||||
CLUSTER_SHARD_LIMIT_EXCEEDED_REASON,
|
||||
|
|
@ -121,6 +121,8 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
|
||||
// The source index .kibana is pointing to. E.g: ".kibana_8.7.0_001"
|
||||
const source = aliases[stateP.currentAlias];
|
||||
// The target index .kibana WILL be pointing to if we reindex. E.g: ".kibana_8.8.0_001"
|
||||
const newVersionTarget = stateP.versionIndex;
|
||||
|
||||
const postInitState = {
|
||||
aliases,
|
||||
|
|
@ -137,7 +139,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
...stateP,
|
||||
...postInitState,
|
||||
sourceIndex: Option.none,
|
||||
targetIndex: `${stateP.indexPrefix}_${stateP.kibanaVersion}_001`,
|
||||
targetIndex: newVersionTarget,
|
||||
controlState: 'WAIT_FOR_MIGRATION_COMPLETION',
|
||||
// Wait for 2s before checking again if the migration has completed
|
||||
retryDelay: 2000,
|
||||
|
|
@ -153,7 +155,6 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
// If the `.kibana` alias exists
|
||||
Option.isSome(postInitState.sourceIndex)
|
||||
) {
|
||||
// CHECKPOINT here we decide to go for yellow source
|
||||
return {
|
||||
...stateP,
|
||||
...postInitState,
|
||||
|
|
@ -182,7 +183,6 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
|
||||
const legacyReindexTarget = `${stateP.indexPrefix}_${legacyVersion}_001`;
|
||||
|
||||
const target = stateP.versionIndex;
|
||||
return {
|
||||
...stateP,
|
||||
...postInitState,
|
||||
|
|
@ -191,7 +191,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
sourceIndexMappings: Option.some(
|
||||
indices[stateP.legacyIndex].mappings
|
||||
) as Option.Some<IndexMapping>,
|
||||
targetIndex: target,
|
||||
targetIndex: newVersionTarget,
|
||||
legacyPreMigrationDoneActions: [
|
||||
{ remove_index: { index: stateP.legacyIndex } },
|
||||
{
|
||||
|
|
@ -209,24 +209,40 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
must_exist: true,
|
||||
},
|
||||
},
|
||||
{ add: { index: target, alias: stateP.currentAlias } },
|
||||
{ add: { index: target, alias: stateP.versionAlias } },
|
||||
{ add: { index: newVersionTarget, alias: stateP.currentAlias } },
|
||||
{ add: { index: newVersionTarget, alias: stateP.versionAlias } },
|
||||
{ remove_index: { index: stateP.tempIndex } },
|
||||
]),
|
||||
};
|
||||
} else if (
|
||||
// if we must relocate documents to this migrator's index, but the index does NOT yet exist:
|
||||
// this migrator must create a temporary index and synchronize with other migrators
|
||||
// this is a similar flow to the reindex one, but this migrator will not reindexing anything
|
||||
stateP.mustRelocateDocuments
|
||||
) {
|
||||
return {
|
||||
...stateP,
|
||||
...postInitState,
|
||||
controlState: 'CREATE_REINDEX_TEMP',
|
||||
sourceIndex: Option.none as Option.None,
|
||||
targetIndex: newVersionTarget,
|
||||
versionIndexReadyActions: Option.some([
|
||||
{ add: { index: newVersionTarget, alias: stateP.currentAlias } },
|
||||
{ add: { index: newVersionTarget, alias: stateP.versionAlias } },
|
||||
{ remove_index: { index: stateP.tempIndex } },
|
||||
]),
|
||||
};
|
||||
} else {
|
||||
// This cluster doesn't have an existing Saved Object index, create a
|
||||
// new version specific index.
|
||||
const target = stateP.versionIndex;
|
||||
// no need to copy anything over from other indices, we can start with a clean, empty index
|
||||
return {
|
||||
...stateP,
|
||||
...postInitState,
|
||||
controlState: 'CREATE_NEW_TARGET',
|
||||
sourceIndex: Option.none as Option.None,
|
||||
targetIndex: target,
|
||||
targetIndex: newVersionTarget,
|
||||
versionIndexReadyActions: Option.some([
|
||||
{ add: { index: target, alias: stateP.currentAlias } },
|
||||
{ add: { index: target, alias: stateP.versionAlias } },
|
||||
{ add: { index: newVersionTarget, alias: stateP.currentAlias } },
|
||||
{ add: { index: newVersionTarget, alias: stateP.versionAlias } },
|
||||
]) as Option.Some<AliasAction[]>,
|
||||
};
|
||||
}
|
||||
|
|
@ -240,6 +256,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
if (
|
||||
// If this version's migration has already been completed we can proceed
|
||||
Either.isRight(aliasesRes) &&
|
||||
// TODO check that this behaves correctly when skipping reindexing
|
||||
versionMigrationCompleted(stateP.currentAlias, stateP.versionAlias, aliasesRes.right)
|
||||
) {
|
||||
return {
|
||||
|
|
@ -414,10 +431,21 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
} else if (stateP.controlState === 'WAIT_FOR_YELLOW_SOURCE') {
|
||||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES',
|
||||
};
|
||||
if (stateP.mustRelocateDocuments) {
|
||||
// this migrator's index must dispatch documents to other indices,
|
||||
// and/or it must receive documents from other indices
|
||||
// we must reindex and synchronize with other migrators
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'CHECK_UNKNOWN_DOCUMENTS',
|
||||
};
|
||||
} else {
|
||||
// this migrator is not involved in a relocation, we can proceed with the standard flow
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES',
|
||||
};
|
||||
}
|
||||
} else if (Either.isLeft(res)) {
|
||||
const left = res.left;
|
||||
if (isTypeof(left, 'index_not_yellow_timeout')) {
|
||||
|
|
@ -711,7 +739,18 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
} else if (stateP.controlState === 'CREATE_REINDEX_TEMP') {
|
||||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT' };
|
||||
if (stateP.mustRelocateDocuments) {
|
||||
// we are reindexing, and this migrator's index is involved in document relocations
|
||||
return { ...stateP, controlState: 'READY_TO_REINDEX_SYNC' };
|
||||
} else {
|
||||
// we are reindexing but this migrator's index is not involved in any document relocation
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT',
|
||||
sourceIndex: stateP.sourceIndex as Option.Some<string>,
|
||||
sourceIndexMappings: stateP.sourceIndexMappings as Option.Some<IndexMapping>,
|
||||
};
|
||||
}
|
||||
} else if (Either.isLeft(res)) {
|
||||
const left = res.left;
|
||||
if (isTypeof(left, 'index_not_green_timeout')) {
|
||||
|
|
@ -738,6 +777,32 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
// left responses to handle here.
|
||||
throwBadResponse(stateP, res);
|
||||
}
|
||||
} else if (stateP.controlState === 'READY_TO_REINDEX_SYNC') {
|
||||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
if (Option.isSome(stateP.sourceIndex) && Option.isSome(stateP.sourceIndexMappings)) {
|
||||
// this migrator's source index exist, reindex its entries
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT',
|
||||
sourceIndex: stateP.sourceIndex as Option.Some<string>,
|
||||
sourceIndexMappings: stateP.sourceIndexMappings as Option.Some<IndexMapping>,
|
||||
};
|
||||
} else {
|
||||
// this migrator's source index did NOT exist
|
||||
// this migrator does not need to reindex anything (others might need to)
|
||||
return { ...stateP, controlState: 'DONE_REINDEXING_SYNC' };
|
||||
}
|
||||
} else if (Either.isLeft(res)) {
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'FATAL',
|
||||
reason: 'An error occurred whilst waiting for other migrators to get to this step.',
|
||||
throwDelayMillis: 1000, // another migrator has failed for a reason, let it take Kibana down and log its problem
|
||||
};
|
||||
} else {
|
||||
return throwBadResponse(stateP, res as never);
|
||||
}
|
||||
} else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT') {
|
||||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
|
|
@ -816,14 +881,41 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
const { sourceIndexPitId, ...state } = stateP;
|
||||
|
||||
if (stateP.mustRelocateDocuments) {
|
||||
return {
|
||||
...state,
|
||||
controlState: 'DONE_REINDEXING_SYNC',
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'SET_TEMP_WRITE_BLOCK',
|
||||
sourceIndex: stateP.sourceIndex as Option.Some<string>,
|
||||
sourceIndexMappings: Option.none,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
throwBadResponse(stateP, res);
|
||||
}
|
||||
} else if (stateP.controlState === 'DONE_REINDEXING_SYNC') {
|
||||
const res = resW as ExcludeRetryableEsError<ResponseType<typeof stateP.controlState>>;
|
||||
if (Either.isRight(res)) {
|
||||
return {
|
||||
...state,
|
||||
...stateP,
|
||||
controlState: 'SET_TEMP_WRITE_BLOCK',
|
||||
sourceIndex: stateP.sourceIndex as Option.Some<string>,
|
||||
sourceIndexMappings: Option.none,
|
||||
};
|
||||
} else if (Either.isLeft(res)) {
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'FATAL',
|
||||
reason: 'An error occurred whilst waiting for other migrators to get to this step.',
|
||||
throwDelayMillis: 1000, // another migrator has failed for a reason, let it take Kibana down and log its problem
|
||||
};
|
||||
} else {
|
||||
throwBadResponse(stateP, res);
|
||||
return throwBadResponse(stateP, res as never);
|
||||
}
|
||||
} else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_TRANSFORM') {
|
||||
// We follow a similar control flow as for
|
||||
|
|
@ -845,7 +937,11 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
stateP.discardCorruptObjects
|
||||
) {
|
||||
const documents = Either.isRight(res) ? res.right.processedDocs : res.left.processedDocs;
|
||||
const batches = createBatches({ documents, maxBatchSizeBytes: stateP.maxBatchSizeBytes });
|
||||
const batches = createBatches({
|
||||
documents,
|
||||
maxBatchSizeBytes: stateP.maxBatchSizeBytes,
|
||||
typeIndexMap: buildTempIndexMap(stateP.indexTypesMap, stateP.kibanaVersion),
|
||||
});
|
||||
if (Either.isRight(batches)) {
|
||||
let corruptDocumentIds = stateP.corruptDocumentIds;
|
||||
let transformErrors = stateP.transformErrors;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
*/
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { defer } from './kibana_migrator_utils';
|
||||
import { next } from './next';
|
||||
import type { State } from './state';
|
||||
|
||||
|
|
@ -14,12 +15,12 @@ describe('migrations v2 next', () => {
|
|||
it.todo('when state.retryDelay > 0 delays execution of the next action');
|
||||
it('DONE returns null', () => {
|
||||
const state = { controlState: 'DONE' } as State;
|
||||
const action = next({} as ElasticsearchClient, (() => {}) as any)(state);
|
||||
const action = next({} as ElasticsearchClient, (() => {}) as any, defer(), defer())(state);
|
||||
expect(action).toEqual(null);
|
||||
});
|
||||
it('FATAL returns null', () => {
|
||||
const state = { controlState: 'FATAL', reason: '' } as State;
|
||||
const action = next({} as ElasticsearchClient, (() => {}) as any)(state);
|
||||
const action = next({} as ElasticsearchClient, (() => {}) as any, defer(), defer())(state);
|
||||
expect(action).toEqual(null);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,8 +7,9 @@
|
|||
*/
|
||||
|
||||
import * as Option from 'fp-ts/lib/Option';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { omit } from 'lodash';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { Defer } from './kibana_migrator_utils';
|
||||
import type {
|
||||
AllActionStates,
|
||||
CalculateExcludeFiltersState,
|
||||
|
|
@ -16,7 +17,7 @@ import type {
|
|||
CheckUnknownDocumentsState,
|
||||
CleanupUnknownAndExcluded,
|
||||
CleanupUnknownAndExcludedWaitForTaskState,
|
||||
CloneTempToSource,
|
||||
CloneTempToTarget,
|
||||
CreateNewTargetState,
|
||||
CreateReindexTempState,
|
||||
InitState,
|
||||
|
|
@ -68,7 +69,12 @@ export type ResponseType<ControlState extends AllActionStates> = Awaited<
|
|||
ReturnType<ReturnType<ActionMap[ControlState]>>
|
||||
>;
|
||||
|
||||
export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => {
|
||||
export const nextActionMap = (
|
||||
client: ElasticsearchClient,
|
||||
transformRawDocs: TransformRawDocs,
|
||||
readyToReindex: Defer<void>,
|
||||
doneReindexing: Defer<void>
|
||||
) => {
|
||||
return {
|
||||
INIT: (state: InitState) =>
|
||||
Actions.initAction({ client, indices: [state.currentAlias, state.versionAlias] }),
|
||||
|
|
@ -135,6 +141,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
|
|||
indexName: state.tempIndex,
|
||||
mappings: state.tempIndexMappings,
|
||||
}),
|
||||
READY_TO_REINDEX_SYNC: () => Actions.synchronizeMigrators(readyToReindex),
|
||||
REINDEX_SOURCE_TO_TEMP_OPEN_PIT: (state: ReindexSourceToTempOpenPit) =>
|
||||
Actions.openPit({ client, index: state.sourceIndex.value }),
|
||||
REINDEX_SOURCE_TO_TEMP_READ: (state: ReindexSourceToTempRead) =>
|
||||
|
|
@ -167,9 +174,10 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
|
|||
*/
|
||||
refresh: false,
|
||||
}),
|
||||
DONE_REINDEXING_SYNC: () => Actions.synchronizeMigrators(doneReindexing),
|
||||
SET_TEMP_WRITE_BLOCK: (state: SetTempWriteBlock) =>
|
||||
Actions.setWriteBlock({ client, index: state.tempIndex }),
|
||||
CLONE_TEMP_TO_TARGET: (state: CloneTempToSource) =>
|
||||
CLONE_TEMP_TO_TARGET: (state: CloneTempToTarget) =>
|
||||
Actions.cloneIndex({ client, source: state.tempIndex, target: state.targetIndex }),
|
||||
REFRESH_TARGET: (state: RefreshTarget) =>
|
||||
Actions.refreshIndex({ client, index: state.targetIndex }),
|
||||
|
|
@ -192,12 +200,13 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
|
|||
taskId: state.updateTargetMappingsTaskId,
|
||||
timeout: '60s',
|
||||
}),
|
||||
UPDATE_TARGET_MAPPINGS_META: (state: UpdateTargetMappingsMeta) =>
|
||||
Actions.updateMappings({
|
||||
UPDATE_TARGET_MAPPINGS_META: (state: UpdateTargetMappingsMeta) => {
|
||||
return Actions.updateMappings({
|
||||
client,
|
||||
index: state.targetIndex,
|
||||
mappings: omit(state.targetIndexMappings, 'properties'),
|
||||
}),
|
||||
mappings: omit(state.targetIndexMappings, ['properties']), // properties already updated on a previous step
|
||||
});
|
||||
},
|
||||
CHECK_VERSION_INDEX_READY_ACTIONS: () => Actions.noop,
|
||||
OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: (state: OutdatedDocumentsSearchOpenPit) =>
|
||||
Actions.openPit({ client, index: state.targetIndex }),
|
||||
|
|
@ -257,8 +266,13 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
|
|||
};
|
||||
};
|
||||
|
||||
export const next = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => {
|
||||
const map = nextActionMap(client, transformRawDocs);
|
||||
export const next = (
|
||||
client: ElasticsearchClient,
|
||||
transformRawDocs: TransformRawDocs,
|
||||
readyToReindex: Defer<void>,
|
||||
doneReindexing: Defer<void>
|
||||
) => {
|
||||
const map = nextActionMap(client, transformRawDocs, readyToReindex, doneReindexing);
|
||||
return (state: State) => {
|
||||
const delay = createDelayFn(state);
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,16 @@ import type {
|
|||
IndexMapping,
|
||||
SavedObjectsMigrationConfigType,
|
||||
MigrationResult,
|
||||
IndexTypesMap,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { Defer } from './kibana_migrator_utils';
|
||||
import type { TransformRawDocs } from './types';
|
||||
import { next } from './next';
|
||||
import { model } from './model';
|
||||
import { createInitialState } from './initial_state';
|
||||
import { migrationStateActionMachine } from './migrations_state_action_machine';
|
||||
import { cleanup } from './migrations_state_machine_cleanup';
|
||||
import type { State } from './state';
|
||||
|
||||
/**
|
||||
* To avoid the Elasticsearch-js client aborting our requests before we
|
||||
|
|
@ -45,9 +49,13 @@ export async function runResilientMigrator({
|
|||
client,
|
||||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
logger,
|
||||
preMigrationScript,
|
||||
readyToReindex,
|
||||
doneReindexing,
|
||||
transformRawDocs,
|
||||
migrationVersionPerType,
|
||||
indexPrefix,
|
||||
|
|
@ -58,8 +66,12 @@ export async function runResilientMigrator({
|
|||
client: ElasticsearchClient;
|
||||
kibanaVersion: string;
|
||||
waitForMigrationCompletion: boolean;
|
||||
mustRelocateDocuments: boolean;
|
||||
indexTypesMap: IndexTypesMap;
|
||||
targetMappings: IndexMapping;
|
||||
preMigrationScript?: string;
|
||||
readyToReindex: Defer<any>;
|
||||
doneReindexing: Defer<any>;
|
||||
logger: Logger;
|
||||
transformRawDocs: TransformRawDocs;
|
||||
migrationVersionPerType: SavedObjectsMigrationVersion;
|
||||
|
|
@ -71,6 +83,8 @@ export async function runResilientMigrator({
|
|||
const initialState = createInitialState({
|
||||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
preMigrationScript,
|
||||
migrationVersionPerType,
|
||||
|
|
@ -84,8 +98,12 @@ export async function runResilientMigrator({
|
|||
return migrationStateActionMachine({
|
||||
initialState,
|
||||
logger,
|
||||
next: next(migrationClient, transformRawDocs),
|
||||
next: next(migrationClient, transformRawDocs, readyToReindex, doneReindexing),
|
||||
model,
|
||||
client: migrationClient,
|
||||
abort: async (state?: State) => {
|
||||
// At this point, we could reject this migrator's defers and unblock other migrators
|
||||
// but we are going to throw and shutdown Kibana anyway, so there's no real point in it
|
||||
await cleanup(client, state);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import type {
|
|||
SavedObjectsRawDoc,
|
||||
SavedObjectTypeExcludeFromUpgradeFilterHook,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { IndexMapping, IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { ControlState } from './state_action_machine';
|
||||
import type { AliasAction } from './actions';
|
||||
import type { TransformErrorObjects } from './core';
|
||||
|
|
@ -152,6 +152,23 @@ export interface BaseState extends ControlState {
|
|||
*/
|
||||
readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects'];
|
||||
readonly waitForMigrationCompletion: boolean;
|
||||
|
||||
/**
|
||||
* This flag tells the migrator that SO documents must be redistributed,
|
||||
* i.e. stored in different system indices, compared to where they are currently stored.
|
||||
* This requires reindexing documents.
|
||||
*/
|
||||
readonly mustRelocateDocuments: boolean;
|
||||
|
||||
/**
|
||||
* This object holds a relation of all the types that are stored in each index, e.g.:
|
||||
* {
|
||||
* '.kibana': [ 'type_1', 'type_2', ... 'type_N' ],
|
||||
* '.kibana_cases': [ 'type_N+1', 'type_N+2', ... 'type_N+M' ],
|
||||
* ...
|
||||
* }
|
||||
*/
|
||||
readonly indexTypesMap: IndexTypesMap;
|
||||
}
|
||||
|
||||
export interface InitState extends BaseState {
|
||||
|
|
@ -231,6 +248,8 @@ export interface FatalState extends BaseState {
|
|||
readonly controlState: 'FATAL';
|
||||
/** The reason the migration was terminated */
|
||||
readonly reason: string;
|
||||
/** The delay in milliseconds before throwing the FATAL exception */
|
||||
readonly throwDelayMillis?: number;
|
||||
}
|
||||
|
||||
export interface WaitForYellowSourceState extends SourceExistsState {
|
||||
|
|
@ -263,7 +282,7 @@ export interface CreateNewTargetState extends PostInitState {
|
|||
readonly versionIndexReadyActions: Option.Some<AliasAction[]>;
|
||||
}
|
||||
|
||||
export interface CreateReindexTempState extends SourceExistsState {
|
||||
export interface CreateReindexTempState extends PostInitState {
|
||||
/**
|
||||
* Create a target index with mappings from the source index and registered
|
||||
* plugins
|
||||
|
|
@ -271,6 +290,11 @@ export interface CreateReindexTempState extends SourceExistsState {
|
|||
readonly controlState: 'CREATE_REINDEX_TEMP';
|
||||
}
|
||||
|
||||
export interface ReadyToReindexSyncState extends PostInitState {
|
||||
/** Open PIT to the source index */
|
||||
readonly controlState: 'READY_TO_REINDEX_SYNC';
|
||||
}
|
||||
|
||||
export interface ReindexSourceToTempOpenPit extends SourceExistsState {
|
||||
/** Open PIT to the source index */
|
||||
readonly controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT';
|
||||
|
|
@ -304,11 +328,16 @@ export interface ReindexSourceToTempIndexBulk extends ReindexSourceToTempBatch {
|
|||
readonly currentBatch: number;
|
||||
}
|
||||
|
||||
export interface DoneReindexingSyncState extends PostInitState {
|
||||
/** Open PIT to the source index */
|
||||
readonly controlState: 'DONE_REINDEXING_SYNC';
|
||||
}
|
||||
|
||||
export interface SetTempWriteBlock extends PostInitState {
|
||||
readonly controlState: 'SET_TEMP_WRITE_BLOCK';
|
||||
}
|
||||
|
||||
export interface CloneTempToSource extends PostInitState {
|
||||
export interface CloneTempToTarget extends PostInitState {
|
||||
/**
|
||||
* Clone the temporary reindex index into
|
||||
*/
|
||||
|
|
@ -482,9 +511,10 @@ export type State = Readonly<
|
|||
| CheckVersionIndexReadyActions
|
||||
| CleanupUnknownAndExcluded
|
||||
| CleanupUnknownAndExcludedWaitForTaskState
|
||||
| CloneTempToSource
|
||||
| CloneTempToTarget
|
||||
| CreateNewTargetState
|
||||
| CreateReindexTempState
|
||||
| DoneReindexingSyncState
|
||||
| DoneState
|
||||
| FatalState
|
||||
| InitState
|
||||
|
|
@ -501,6 +531,7 @@ export type State = Readonly<
|
|||
| OutdatedDocumentsSearchRead
|
||||
| OutdatedDocumentsTransform
|
||||
| PrepareCompatibleMigration
|
||||
| ReadyToReindexSyncState
|
||||
| RefreshSource
|
||||
| RefreshTarget
|
||||
| ReindexSourceToTempClosePit
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@
|
|||
"@kbn/core-doc-links-server-mocks",
|
||||
"@kbn/core-logging-server-internal",
|
||||
"@kbn/core-saved-objects-base-server-mocks",
|
||||
"@kbn/core-elasticsearch-server-mocks",
|
||||
"@kbn/doc-links",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/logging-mocks",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types';
|
|||
import { deleteUnknownTypeObjects, getUnknownTypesDeprecations } from './unknown_object_types';
|
||||
import { typeRegistryMock } from '@kbn/core-saved-objects-base-server-mocks';
|
||||
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
|
||||
import type { SavedObjectsType } from '@kbn/core-saved-objects-server';
|
||||
import { type SavedObjectsType, MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
|
||||
const createAggregateTypesSearchResponse = (
|
||||
typesIds: Record<string, string[]> = {}
|
||||
|
|
@ -50,7 +50,7 @@ describe('unknown saved object types deprecation', () => {
|
|||
|
||||
let typeRegistry: ReturnType<typeof typeRegistryMock.create>;
|
||||
let esClient: ReturnType<typeof elasticsearchClientMock.createScopedClusterClient>;
|
||||
const kibanaIndex = '.kibana';
|
||||
const kibanaIndex = MAIN_SAVED_OBJECT_INDEX;
|
||||
|
||||
beforeEach(() => {
|
||||
typeRegistry = typeRegistryMock.create();
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import { type RawPackageInfo, Env } from '@kbn/config';
|
|||
import { ByteSizeValue } from '@kbn/config-schema';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { getEnvOptions } from '@kbn/config-mocks';
|
||||
import { SavedObjectsType, MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
|
||||
import { nodeServiceMock } from '@kbn/core-node-server-mocks';
|
||||
import { mockCoreContext } from '@kbn/core-base-server-mocks';
|
||||
|
|
@ -55,6 +56,14 @@ import { getSavedObjectsDeprecationsProvider } from './deprecations';
|
|||
jest.mock('./object_types');
|
||||
jest.mock('./deprecations');
|
||||
|
||||
const createType = (parts: Partial<SavedObjectsType>): SavedObjectsType => ({
|
||||
name: 'test-type',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
mappings: { properties: {} },
|
||||
...parts,
|
||||
});
|
||||
|
||||
describe('SavedObjectsService', () => {
|
||||
let deprecationsSetup: ReturnType<typeof createDeprecationRegistryProviderMock>;
|
||||
|
||||
|
|
@ -630,5 +639,85 @@ describe('SavedObjectsService', () => {
|
|||
expect(includedHiddenTypes).toEqual(['someHiddenType']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('index retrieval APIs', () => {
|
||||
let soService: SavedObjectsService;
|
||||
|
||||
beforeEach(async () => {
|
||||
const coreContext = createCoreContext({ skipMigration: false });
|
||||
soService = new SavedObjectsService(coreContext);
|
||||
|
||||
typeRegistryInstanceMock.getType.mockImplementation((type: string) => {
|
||||
if (type === 'dashboard') {
|
||||
return createType({
|
||||
name: 'dashboard',
|
||||
});
|
||||
} else if (type === 'foo') {
|
||||
return createType({
|
||||
name: 'foo',
|
||||
indexPattern: '.kibana_foo',
|
||||
});
|
||||
} else if (type === 'bar') {
|
||||
return createType({
|
||||
name: 'bar',
|
||||
indexPattern: '.kibana_bar',
|
||||
});
|
||||
} else if (type === 'bar_too') {
|
||||
return createType({
|
||||
name: 'bar_too',
|
||||
indexPattern: '.kibana_bar',
|
||||
});
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
});
|
||||
|
||||
await soService.setup(createSetupDeps());
|
||||
});
|
||||
|
||||
describe('#getDefaultIndex', () => {
|
||||
it('return the default index', async () => {
|
||||
const { getDefaultIndex } = await soService.start(createStartDeps());
|
||||
expect(getDefaultIndex()).toEqual(MAIN_SAVED_OBJECT_INDEX);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#getIndexForType', () => {
|
||||
it('return the correct index for type specifying its indexPattern', async () => {
|
||||
const { getIndexForType } = await soService.start(createStartDeps());
|
||||
expect(getIndexForType('bar')).toEqual('.kibana_bar');
|
||||
});
|
||||
it('return the correct index for type not specifying its indexPattern', async () => {
|
||||
const { getIndexForType } = await soService.start(createStartDeps());
|
||||
expect(getIndexForType('dashboard')).toEqual(MAIN_SAVED_OBJECT_INDEX);
|
||||
});
|
||||
it('return the default index for unknown type', async () => {
|
||||
const { getIndexForType } = await soService.start(createStartDeps());
|
||||
expect(getIndexForType('unknown_type')).toEqual(MAIN_SAVED_OBJECT_INDEX);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#getIndicesForTypes', () => {
|
||||
it('return the correct indices for specified types', async () => {
|
||||
const { getIndicesForTypes } = await soService.start(createStartDeps());
|
||||
expect(getIndicesForTypes(['dashboard', 'foo', 'bar'])).toEqual([
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
'.kibana_foo',
|
||||
'.kibana_bar',
|
||||
]);
|
||||
});
|
||||
it('ignore duplicate indices', async () => {
|
||||
const { getIndicesForTypes } = await soService.start(createStartDeps());
|
||||
expect(getIndicesForTypes(['bar', 'bar_too'])).toEqual(['.kibana_bar']);
|
||||
});
|
||||
it('return the default index for unknown type', async () => {
|
||||
const { getIndicesForTypes } = await soService.start(createStartDeps());
|
||||
expect(getIndicesForTypes(['unknown', 'foo'])).toEqual([
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
'.kibana_foo',
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -52,13 +52,12 @@ import {
|
|||
import type { InternalCoreUsageDataSetup } from '@kbn/core-usage-data-base-server-internal';
|
||||
import type { DeprecationRegistryProvider } from '@kbn/core-deprecations-server';
|
||||
import type { NodeInfo } from '@kbn/core-node-server';
|
||||
import { MAIN_SAVED_OBJECT_INDEX, ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
|
||||
import { registerRoutes } from './routes';
|
||||
import { calculateStatus$ } from './status';
|
||||
import { registerCoreObjectTypes } from './object_types';
|
||||
import { getSavedObjectsDeprecationsProvider } from './deprecations';
|
||||
|
||||
const kibanaIndex = '.kibana';
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
|
|
@ -125,7 +124,7 @@ export class SavedObjectsService
|
|||
this.config = new SavedObjectConfig(savedObjectsConfig, savedObjectsMigrationConfig);
|
||||
deprecations.getRegistry('savedObjects').registerDeprecations(
|
||||
getSavedObjectsDeprecationsProvider({
|
||||
kibanaIndex,
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
savedObjectsConfig: this.config,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
typeRegistry: this.typeRegistry,
|
||||
|
|
@ -140,7 +139,7 @@ export class SavedObjectsService
|
|||
logger: this.logger,
|
||||
config: this.config,
|
||||
migratorPromise: firstValueFrom(this.migrator$),
|
||||
kibanaIndex,
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
});
|
||||
|
||||
|
|
@ -198,7 +197,8 @@ export class SavedObjectsService
|
|||
this.typeRegistry.registerType(type);
|
||||
},
|
||||
getTypeRegistry: () => this.typeRegistry,
|
||||
getKibanaIndex: () => kibanaIndex,
|
||||
getDefaultIndex: () => MAIN_SAVED_OBJECT_INDEX,
|
||||
getAllIndices: () => [...ALL_SAVED_OBJECT_INDICES],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -280,7 +280,7 @@ export class SavedObjectsService
|
|||
return SavedObjectsRepository.createRepository(
|
||||
migrator,
|
||||
this.typeRegistry,
|
||||
kibanaIndex,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
esClient,
|
||||
this.logger.get('repository'),
|
||||
includedHiddenTypes,
|
||||
|
|
@ -341,6 +341,21 @@ export class SavedObjectsService
|
|||
importSizeLimit: options?.importSizeLimit ?? this.config!.maxImportExportSize,
|
||||
}),
|
||||
getTypeRegistry: () => this.typeRegistry,
|
||||
getDefaultIndex: () => MAIN_SAVED_OBJECT_INDEX,
|
||||
getIndexForType: (type: string) => {
|
||||
const definition = this.typeRegistry.getType(type);
|
||||
return definition?.indexPattern ?? MAIN_SAVED_OBJECT_INDEX;
|
||||
},
|
||||
getIndicesForTypes: (types: string[]) => {
|
||||
const indices = new Set<string>();
|
||||
types.forEach((type) => {
|
||||
const definition = this.typeRegistry.getType(type);
|
||||
const index = definition?.indexPattern ?? MAIN_SAVED_OBJECT_INDEX;
|
||||
indices.add(index);
|
||||
});
|
||||
return [...indices];
|
||||
},
|
||||
getAllIndices: () => [...ALL_SAVED_OBJECT_INDICES],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -357,7 +372,7 @@ export class SavedObjectsService
|
|||
logger: this.logger,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
soMigrationsConfig,
|
||||
kibanaIndex,
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
client,
|
||||
docLinks,
|
||||
waitForMigrationCompletion,
|
||||
|
|
|
|||
|
|
@ -81,7 +81,13 @@ describe('calculateStatus$', () => {
|
|||
it('is available after migrations have ran', async () => {
|
||||
await expect(
|
||||
calculateStatus$(
|
||||
of({ status: 'completed', result: [{ status: 'skipped' }, { status: 'patched' }] }),
|
||||
of({
|
||||
status: 'completed',
|
||||
result: [
|
||||
{ status: 'skipped' },
|
||||
{ status: 'patched', destIndex: '.kibana', elapsedMs: 28 },
|
||||
],
|
||||
}),
|
||||
esStatus$
|
||||
)
|
||||
.pipe(take(2))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
savedObjectsImporterMock,
|
||||
} from '@kbn/core-saved-objects-import-export-server-mocks';
|
||||
import { migrationMocks } from '@kbn/core-saved-objects-migration-server-mocks';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
|
||||
type SavedObjectsServiceContract = PublicMethodsOf<SavedObjectsService>;
|
||||
|
||||
|
|
@ -41,6 +42,10 @@ const createStartContractMock = (typeRegistry?: jest.Mocked<ISavedObjectTypeRegi
|
|||
createExporter: jest.fn(),
|
||||
createImporter: jest.fn(),
|
||||
getTypeRegistry: jest.fn(),
|
||||
getDefaultIndex: jest.fn(),
|
||||
getIndexForType: jest.fn(),
|
||||
getIndicesForTypes: jest.fn(),
|
||||
getAllIndices: jest.fn(),
|
||||
};
|
||||
|
||||
startContrat.getScopedClient.mockReturnValue(savedObjectsClientMock.create());
|
||||
|
|
@ -49,6 +54,10 @@ const createStartContractMock = (typeRegistry?: jest.Mocked<ISavedObjectTypeRegi
|
|||
startContrat.getTypeRegistry.mockReturnValue(typeRegistry ?? typeRegistryMock.create());
|
||||
startContrat.createExporter.mockReturnValue(savedObjectsExporterMock.create());
|
||||
startContrat.createImporter.mockReturnValue(savedObjectsImporterMock.create());
|
||||
startContrat.getDefaultIndex.mockReturnValue(MAIN_SAVED_OBJECT_INDEX);
|
||||
startContrat.getIndexForType.mockReturnValue(MAIN_SAVED_OBJECT_INDEX);
|
||||
startContrat.getIndicesForTypes.mockReturnValue([MAIN_SAVED_OBJECT_INDEX]);
|
||||
startContrat.getAllIndices.mockReturnValue([MAIN_SAVED_OBJECT_INDEX]);
|
||||
|
||||
return startContrat;
|
||||
};
|
||||
|
|
@ -67,10 +76,12 @@ const createSetupContractMock = () => {
|
|||
setSecurityExtension: jest.fn(),
|
||||
setSpacesExtension: jest.fn(),
|
||||
registerType: jest.fn(),
|
||||
getKibanaIndex: jest.fn(),
|
||||
getDefaultIndex: jest.fn(),
|
||||
getAllIndices: jest.fn(),
|
||||
};
|
||||
|
||||
setupContract.getKibanaIndex.mockReturnValue('.kibana');
|
||||
setupContract.getDefaultIndex.mockReturnValue(MAIN_SAVED_OBJECT_INDEX);
|
||||
setupContract.getAllIndices.mockReturnValue([MAIN_SAVED_OBJECT_INDEX]);
|
||||
|
||||
return setupContract;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -52,6 +52,15 @@ export type {
|
|||
SavedObjectsExportablePredicate,
|
||||
} from './src/saved_objects_management';
|
||||
export type { SavedObjectStatusMeta } from './src/saved_objects_status';
|
||||
export {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
INGEST_SAVED_OBJECT_INDEX,
|
||||
ALERTING_CASES_SAVED_OBJECT_INDEX,
|
||||
SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
|
||||
ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
ALL_SAVED_OBJECT_INDICES,
|
||||
} from './src/saved_objects_index_pattern';
|
||||
export type {
|
||||
SavedObjectsType,
|
||||
SavedObjectTypeExcludeFromUpgradeFilterHook,
|
||||
|
|
|
|||
|
|
@ -136,7 +136,14 @@ export interface SavedObjectsServiceSetup {
|
|||
/**
|
||||
* Returns the default index used for saved objects.
|
||||
*/
|
||||
getKibanaIndex: () => string;
|
||||
getDefaultIndex: () => string;
|
||||
|
||||
/**
|
||||
* Returns all (aliases to) kibana system indices used for saved object storage.
|
||||
*
|
||||
* @deprecated use the `start` contract counterpart.
|
||||
*/
|
||||
getAllIndices: () => string[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -209,4 +216,25 @@ export interface SavedObjectsServiceStart {
|
|||
* {@link SavedObjectsType | saved object types}
|
||||
*/
|
||||
getTypeRegistry: () => ISavedObjectTypeRegistry;
|
||||
/**
|
||||
* Returns the (alias to the) index that the specified saved object type is stored in.
|
||||
*
|
||||
* @param type The SO type to retrieve the index/alias for.
|
||||
*/
|
||||
getIndexForType: (type: string) => string;
|
||||
/**
|
||||
* Returns the (alias to the) index that the specified saved object type is stored in.
|
||||
*
|
||||
* @remark if multiple types are living in the same index, duplicates will be removed.
|
||||
* @param types The SO types to retrieve the index/alias for.
|
||||
*/
|
||||
getIndicesForTypes: (types: string[]) => string[];
|
||||
/**
|
||||
* Returns the default index used for saved objects.
|
||||
*/
|
||||
getDefaultIndex: () => string;
|
||||
/**
|
||||
* Returns all (aliases to) kibana system indices used for saved object storage.
|
||||
*/
|
||||
getAllIndices: () => string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Collect and centralize the names of the different saved object indices.
|
||||
* Note that all of them start with the '.kibana' prefix.
|
||||
* There are multiple places in the code that these indices have the form .kibana*.
|
||||
* However, beware that there are some system indices that have the same prefix
|
||||
* but are NOT used to store saved objects, e.g.: .kibana_security_session_1
|
||||
*/
|
||||
export const MAIN_SAVED_OBJECT_INDEX = '.kibana';
|
||||
export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`;
|
||||
export const INGEST_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_ingest`;
|
||||
export const ALERTING_CASES_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_alerting_cases`;
|
||||
export const SECURITY_SOLUTION_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_security_solution`;
|
||||
export const ANALYTICS_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_analytics`;
|
||||
export const ALL_SAVED_OBJECT_INDICES = [
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
ALERTING_CASES_SAVED_OBJECT_INDEX,
|
||||
INGEST_SAVED_OBJECT_INDEX,
|
||||
SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
|
||||
ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
];
|
||||
|
|
@ -40,7 +40,10 @@ import {
|
|||
type InternalCoreUsageDataSetup,
|
||||
} from '@kbn/core-usage-data-base-server-internal';
|
||||
import type { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { SavedObjectsServiceStart } from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
type SavedObjectsServiceStart,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
|
||||
import { isConfigured } from './is_configured';
|
||||
import { coreUsageStatsType } from './saved_objects';
|
||||
|
|
@ -61,26 +64,6 @@ export interface StartDeps {
|
|||
exposedConfigsToUsage: ExposedConfigsToUsage;
|
||||
}
|
||||
|
||||
const kibanaIndex = '.kibana';
|
||||
|
||||
/**
|
||||
* Because users can configure their Saved Object to any arbitrary index name,
|
||||
* we need to map customized index names back to a "standard" index name.
|
||||
*
|
||||
* e.g. If a user configures `kibana.index: .my_saved_objects` we want to the
|
||||
* collected data to be grouped under `.kibana` not ".my_saved_objects".
|
||||
*
|
||||
* This is rather brittle, but the option to configure index names might go
|
||||
* away completely anyway (see #60053).
|
||||
*
|
||||
* @param index The index name configured for this SO type
|
||||
* @param kibanaConfigIndex The default kibana index as configured by the user
|
||||
* with `kibana.index`
|
||||
*/
|
||||
const kibanaOrTaskManagerIndex = (index: string, kibanaConfigIndex: string) => {
|
||||
return index === kibanaConfigIndex ? '.kibana' : '.kibana_task_manager';
|
||||
};
|
||||
|
||||
interface UsageDataAggs extends AggregationsMultiBucketAggregateBase {
|
||||
buckets: {
|
||||
disabled: AggregationsSingleBucketAggregateBase;
|
||||
|
|
@ -133,7 +116,7 @@ export class CoreUsageDataService
|
|||
.getTypeRegistry()
|
||||
.getAllTypes()
|
||||
.reduce((acc, type) => {
|
||||
const index = type.indexPattern ?? kibanaIndex;
|
||||
const index = type.indexPattern ?? MAIN_SAVED_OBJECT_INDEX;
|
||||
return acc.add(index);
|
||||
}, new Set<string>())
|
||||
.values()
|
||||
|
|
@ -151,7 +134,7 @@ export class CoreUsageDataService
|
|||
const stats = body[0];
|
||||
|
||||
return {
|
||||
alias: kibanaOrTaskManagerIndex(index, kibanaIndex),
|
||||
alias: index,
|
||||
docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0,
|
||||
docsDeleted: stats['docs.deleted'] ? parseInt(stats['docs.deleted'], 10) : 0,
|
||||
storeSizeBytes: stats['store.size'] ? parseInt(stats['store.size'], 10) : 0,
|
||||
|
|
@ -192,7 +175,7 @@ export class CoreUsageDataService
|
|||
unknown,
|
||||
{ aliases: UsageDataAggs }
|
||||
>({
|
||||
index: kibanaIndex,
|
||||
index: MAIN_SAVED_OBJECT_INDEX, // depends on the .kibana split (assuming 'legacy-url-alias' is stored in '.kibana')
|
||||
body: {
|
||||
track_total_hits: true,
|
||||
query: { match: { type: LEGACY_URL_ALIAS_TYPE } },
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import type { Client } from '@elastic/elasticsearch';
|
|||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { KbnClient } from '@kbn/test';
|
||||
|
||||
import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib';
|
||||
import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
|
||||
import { migrateSavedObjectIndices, createStats, cleanSavedObjectIndices } from '../lib';
|
||||
|
||||
export async function emptyKibanaIndexAction({
|
||||
client,
|
||||
|
|
@ -23,8 +24,8 @@ export async function emptyKibanaIndexAction({
|
|||
}) {
|
||||
const stats = createStats('emptyKibanaIndex', log);
|
||||
|
||||
await cleanKibanaIndices({ client, stats, log });
|
||||
await migrateKibanaIndex(kbnClient);
|
||||
stats.createdIndex('.kibana');
|
||||
await cleanSavedObjectIndices({ client, stats, log });
|
||||
await migrateSavedObjectIndices(kbnClient);
|
||||
ALL_SAVED_OBJECT_INDICES.forEach((indexPattern) => stats.createdIndex(indexPattern));
|
||||
return stats.toJSON();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { REPO_ROOT } from '@kbn/repo-info';
|
|||
import type { KbnClient } from '@kbn/test';
|
||||
import type { Client } from '@elastic/elasticsearch';
|
||||
import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { ES_CLIENT_HEADERS } from '../client_headers';
|
||||
|
||||
import {
|
||||
|
|
@ -24,7 +25,7 @@ import {
|
|||
createParseArchiveStreams,
|
||||
createCreateIndexStream,
|
||||
createIndexDocRecordsStream,
|
||||
migrateKibanaIndex,
|
||||
migrateSavedObjectIndices,
|
||||
Progress,
|
||||
createDefaultSpace,
|
||||
} from '../lib';
|
||||
|
|
@ -104,14 +105,15 @@ export async function loadAction({
|
|||
}
|
||||
);
|
||||
|
||||
// If we affected the Kibana index, we need to ensure it's migrated...
|
||||
if (Object.keys(result).some((k) => k.startsWith('.kibana'))) {
|
||||
await migrateKibanaIndex(kbnClient);
|
||||
// If we affected saved objects indices, we need to ensure they are migrated...
|
||||
if (Object.keys(result).some((k) => k.startsWith(MAIN_SAVED_OBJECT_INDEX))) {
|
||||
await migrateSavedObjectIndices(kbnClient);
|
||||
log.debug('[%s] Migrated Kibana index after loading Kibana data', name);
|
||||
|
||||
if (kibanaPluginIds.includes('spaces')) {
|
||||
await createDefaultSpace({ client, index: '.kibana' });
|
||||
log.debug('[%s] Ensured that default space exists in .kibana', name);
|
||||
// WARNING affected by #104081. Assumes 'spaces' saved objects are stored in MAIN_SAVED_OBJECT_INDEX
|
||||
await createDefaultSpace({ client, index: MAIN_SAVED_OBJECT_INDEX });
|
||||
log.debug(`[%s] Ensured that default space exists in ${MAIN_SAVED_OBJECT_INDEX}`, name);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
import { Transform } from 'stream';
|
||||
import type { Client } from '@elastic/elasticsearch';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { Stats } from '../stats';
|
||||
import { Progress } from '../progress';
|
||||
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
||||
|
|
@ -78,7 +79,9 @@ export function createGenerateDocRecordsStream({
|
|||
// if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that
|
||||
// when it is loaded it can skip migration, if possible
|
||||
index:
|
||||
hit._index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : hit._index,
|
||||
hit._index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames
|
||||
? `${MAIN_SAVED_OBJECT_INDEX}_1`
|
||||
: hit._index,
|
||||
data_stream: dataStream,
|
||||
id: hit._id,
|
||||
source: hit._source,
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ export {
|
|||
createCreateIndexStream,
|
||||
createDeleteIndexStream,
|
||||
createGenerateIndexRecordsStream,
|
||||
deleteKibanaIndices,
|
||||
migrateKibanaIndex,
|
||||
cleanKibanaIndices,
|
||||
deleteSavedObjectIndices,
|
||||
migrateSavedObjectIndices,
|
||||
cleanSavedObjectIndices,
|
||||
createDefaultSpace,
|
||||
} from './indices';
|
||||
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { deleteKibanaIndices } from './kibana_index';
|
||||
import type { deleteSavedObjectIndices } from './kibana_index';
|
||||
|
||||
export const mockDeleteKibanaIndices = jest.fn() as jest.MockedFunction<typeof deleteKibanaIndices>;
|
||||
export const mockdeleteSavedObjectIndices = jest.fn() as jest.MockedFunction<
|
||||
typeof deleteSavedObjectIndices
|
||||
>;
|
||||
|
||||
jest.mock('./kibana_index', () => ({
|
||||
deleteKibanaIndices: mockDeleteKibanaIndices,
|
||||
deleteSavedObjectIndices: mockdeleteSavedObjectIndices,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { mockDeleteKibanaIndices } from './create_index_stream.test.mock';
|
||||
import { mockdeleteSavedObjectIndices } from './create_index_stream.test.mock';
|
||||
|
||||
import sinon from 'sinon';
|
||||
import Chance from 'chance';
|
||||
|
|
@ -28,7 +28,7 @@ const chance = new Chance();
|
|||
const log = createStubLogger();
|
||||
|
||||
beforeEach(() => {
|
||||
mockDeleteKibanaIndices.mockClear();
|
||||
mockdeleteSavedObjectIndices.mockClear();
|
||||
});
|
||||
|
||||
describe('esArchiver: createCreateIndexStream()', () => {
|
||||
|
|
@ -187,7 +187,7 @@ describe('esArchiver: createCreateIndexStream()', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('deleteKibanaIndices', () => {
|
||||
describe('deleteSavedObjectIndices', () => {
|
||||
function doTest(...indices: string[]) {
|
||||
return createPromiseFromStreams([
|
||||
createListStream(indices.map((index) => createStubIndexRecord(index))),
|
||||
|
|
@ -199,15 +199,15 @@ describe('esArchiver: createCreateIndexStream()', () => {
|
|||
it('does not delete Kibana indices for indexes that do not start with .kibana', async () => {
|
||||
await doTest('.foo');
|
||||
|
||||
expect(mockDeleteKibanaIndices).not.toHaveBeenCalled();
|
||||
expect(mockdeleteSavedObjectIndices).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes Kibana indices at most once for indices that start with .kibana', async () => {
|
||||
// If we are loading the main Kibana index, we should delete all Kibana indices for backwards compatibility reasons.
|
||||
await doTest('.kibana_7.16.0_001', '.kibana_task_manager_7.16.0_001');
|
||||
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenCalledTimes(1);
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenCalledWith(
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({ onlyTaskManager: true })
|
||||
);
|
||||
});
|
||||
|
|
@ -216,8 +216,8 @@ describe('esArchiver: createCreateIndexStream()', () => {
|
|||
// If we are loading the Kibana task manager index, we should only delete that index, not any other Kibana indices.
|
||||
await doTest('.kibana_task_manager_7.16.0_001', '.kibana_task_manager_7.16.0_002');
|
||||
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenCalledTimes(1);
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenCalledWith(
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ onlyTaskManager: true })
|
||||
);
|
||||
});
|
||||
|
|
@ -227,12 +227,12 @@ describe('esArchiver: createCreateIndexStream()', () => {
|
|||
// So, we first delete only the Kibana task manager indices, then we wind up deleting all Kibana indices.
|
||||
await doTest('.kibana_task_manager_7.16.0_001', '.kibana_7.16.0_001');
|
||||
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenCalledTimes(2);
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenNthCalledWith(
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(2);
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ onlyTaskManager: true })
|
||||
);
|
||||
expect(mockDeleteKibanaIndices).toHaveBeenNthCalledWith(
|
||||
expect(mockdeleteSavedObjectIndices).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.not.objectContaining({ onlyTaskManager: true })
|
||||
);
|
||||
|
|
|
|||
|
|
@ -14,8 +14,12 @@ import type { Client } from '@elastic/elasticsearch';
|
|||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import { Stats } from '../stats';
|
||||
import { deleteKibanaIndices } from './kibana_index';
|
||||
import { deleteSavedObjectIndices } from './kibana_index';
|
||||
import { deleteIndex } from './delete_index';
|
||||
import { deleteDataStream } from './delete_data_stream';
|
||||
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
||||
|
|
@ -96,8 +100,8 @@ export function createCreateIndexStream({
|
|||
|
||||
async function handleIndex(record: DocRecord) {
|
||||
const { index, settings, mappings, aliases } = record.value;
|
||||
const isKibanaTaskManager = index.startsWith('.kibana_task_manager');
|
||||
const isKibana = index.startsWith('.kibana') && !isKibanaTaskManager;
|
||||
const isKibanaTaskManager = index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX);
|
||||
const isKibana = index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !isKibanaTaskManager;
|
||||
|
||||
if (docsOnly) {
|
||||
return;
|
||||
|
|
@ -106,10 +110,10 @@ export function createCreateIndexStream({
|
|||
async function attemptToCreate(attemptNumber = 1) {
|
||||
try {
|
||||
if (isKibana && !kibanaIndexAlreadyDeleted) {
|
||||
await deleteKibanaIndices({ client, stats, log }); // delete all .kibana* indices
|
||||
await deleteSavedObjectIndices({ client, stats, log }); // delete all .kibana* indices
|
||||
kibanaIndexAlreadyDeleted = kibanaTaskManagerIndexAlreadyDeleted = true;
|
||||
} else if (isKibanaTaskManager && !kibanaTaskManagerIndexAlreadyDeleted) {
|
||||
await deleteKibanaIndices({ client, stats, onlyTaskManager: true, log }); // delete only .kibana_task_manager* indices
|
||||
await deleteSavedObjectIndices({ client, stats, onlyTaskManager: true, log }); // delete only .kibana_task_manager* indices
|
||||
kibanaTaskManagerIndexAlreadyDeleted = true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,9 +10,10 @@ import { Transform } from 'stream';
|
|||
import type { Client } from '@elastic/elasticsearch';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { Stats } from '../stats';
|
||||
import { deleteIndex } from './delete_index';
|
||||
import { cleanKibanaIndices } from './kibana_index';
|
||||
import { cleanSavedObjectIndices } from './kibana_index';
|
||||
import { deleteDataStream } from './delete_data_stream';
|
||||
|
||||
export function createDeleteIndexStream(client: Client, stats: Stats, log: ToolingLog) {
|
||||
|
|
@ -28,8 +29,8 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli
|
|||
if (record.type === 'index') {
|
||||
const { index } = record.value;
|
||||
|
||||
if (index.startsWith('.kibana')) {
|
||||
await cleanKibanaIndices({ client, stats, log });
|
||||
if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
|
||||
await cleanSavedObjectIndices({ client, stats, log });
|
||||
} else {
|
||||
await deleteIndex({ client, stats, log, index });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
import type { Client } from '@elastic/elasticsearch';
|
||||
import { Transform } from 'stream';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { Stats } from '../stats';
|
||||
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
||||
import { getIndexTemplate } from '..';
|
||||
|
|
@ -100,7 +101,10 @@ export function createGenerateIndexRecordsStream({
|
|||
value: {
|
||||
// if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that
|
||||
// when it is loaded it can skip migration, if possible
|
||||
index: index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : index,
|
||||
index:
|
||||
index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames
|
||||
? `${MAIN_SAVED_OBJECT_INDEX}_1`
|
||||
: index,
|
||||
settings,
|
||||
mappings,
|
||||
aliases,
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ export { createCreateIndexStream } from './create_index_stream';
|
|||
export { createDeleteIndexStream } from './delete_index_stream';
|
||||
export { createGenerateIndexRecordsStream } from './generate_index_records_stream';
|
||||
export {
|
||||
migrateKibanaIndex,
|
||||
deleteKibanaIndices,
|
||||
cleanKibanaIndices,
|
||||
migrateSavedObjectIndices,
|
||||
deleteSavedObjectIndices,
|
||||
cleanSavedObjectIndices,
|
||||
createDefaultSpace,
|
||||
} from './kibana_index';
|
||||
|
|
|
|||
|
|
@ -11,14 +11,19 @@ import { inspect } from 'util';
|
|||
import type { Client } from '@elastic/elasticsearch';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { KbnClient } from '@kbn/test';
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
ALL_SAVED_OBJECT_INDICES,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import { Stats } from '../stats';
|
||||
import { deleteIndex } from './delete_index';
|
||||
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
||||
|
||||
/**
|
||||
* Deletes all indices that start with `.kibana`, or if onlyTaskManager==true, all indices that start with `.kibana_task_manager`
|
||||
* Deletes all saved object indices, or if onlyTaskManager==true, it deletes task_manager indices
|
||||
*/
|
||||
export async function deleteKibanaIndices({
|
||||
export async function deleteSavedObjectIndices({
|
||||
client,
|
||||
stats,
|
||||
onlyTaskManager = false,
|
||||
|
|
@ -29,8 +34,9 @@ export async function deleteKibanaIndices({
|
|||
onlyTaskManager?: boolean;
|
||||
log: ToolingLog;
|
||||
}) {
|
||||
const indexPattern = onlyTaskManager ? '.kibana_task_manager*' : '.kibana*';
|
||||
const indexNames = await fetchKibanaIndices(client, indexPattern);
|
||||
const indexNames = (await fetchSavedObjectIndices(client)).filter(
|
||||
(indexName) => !onlyTaskManager || indexName.includes(TASK_MANAGER_SAVED_OBJECT_INDEX)
|
||||
);
|
||||
if (!indexNames.length) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -60,27 +66,33 @@ export async function deleteKibanaIndices({
|
|||
* builds up an object that implements just enough of the kbnMigrations interface
|
||||
* as is required by migrations.
|
||||
*/
|
||||
export async function migrateKibanaIndex(kbnClient: KbnClient) {
|
||||
export async function migrateSavedObjectIndices(kbnClient: KbnClient) {
|
||||
await kbnClient.savedObjects.migrate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrations mean that the Kibana index will look something like:
|
||||
* .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting
|
||||
* with .kibana, then filters out any that aren't actually Kibana's core
|
||||
* index (e.g. we don't want to remove .kibana_task_manager or the like).
|
||||
* Check if the given index is a Kibana saved object index.
|
||||
* This includes most .kibana_*
|
||||
* but we must make sure that indices such as '.kibana_security_session_1' are NOT deleted.
|
||||
*
|
||||
* IMPORTANT
|
||||
* Note that we can have more than 2 system indices (different SO types can go to different indices)
|
||||
* ATM we have '.kibana', '.kibana_task_manager', '.kibana_cases'
|
||||
* This method also takes into account legacy indices: .kibana_1, .kibana_task_manager_1.
|
||||
* @param [index] the name of the index to check
|
||||
* @returns boolean 'true' if the index is a Kibana saved object index.
|
||||
*/
|
||||
function isKibanaIndex(index?: string): index is string {
|
||||
return Boolean(
|
||||
index &&
|
||||
(/^\.kibana(:?_\d*)?$/.test(index) ||
|
||||
/^\.kibana(_task_manager)?_(pre)?\d+\.\d+\.\d+/.test(index))
|
||||
);
|
||||
|
||||
const LEGACY_INDICES_REGEXP = new RegExp(`^(${ALL_SAVED_OBJECT_INDICES.join('|')})(:?_\\d*)?$`);
|
||||
const INDICES_REGEXP = new RegExp(`^(${ALL_SAVED_OBJECT_INDICES.join('|')})_(pre)?\\d+.\\d+.\\d+`);
|
||||
|
||||
function isSavedObjectIndex(index?: string): index is string {
|
||||
return Boolean(index && (LEGACY_INDICES_REGEXP.test(index) || INDICES_REGEXP.test(index)));
|
||||
}
|
||||
|
||||
async function fetchKibanaIndices(client: Client, indexPattern: string) {
|
||||
async function fetchSavedObjectIndices(client: Client) {
|
||||
const resp = await client.cat.indices(
|
||||
{ index: indexPattern, format: 'json' },
|
||||
{ index: `${MAIN_SAVED_OBJECT_INDEX}*`, format: 'json' },
|
||||
{
|
||||
headers: ES_CLIENT_HEADERS,
|
||||
}
|
||||
|
|
@ -90,12 +102,12 @@ async function fetchKibanaIndices(client: Client, indexPattern: string) {
|
|||
throw new Error(`expected response to be an array ${inspect(resp)}`);
|
||||
}
|
||||
|
||||
return resp.map((x: { index?: string }) => x.index).filter(isKibanaIndex);
|
||||
return resp.map((x: { index?: string }) => x.index).filter(isSavedObjectIndex);
|
||||
}
|
||||
|
||||
const delay = (delayInMs: number) => new Promise((resolve) => setTimeout(resolve, delayInMs));
|
||||
|
||||
export async function cleanKibanaIndices({
|
||||
export async function cleanSavedObjectIndices({
|
||||
client,
|
||||
stats,
|
||||
log,
|
||||
|
|
@ -107,7 +119,7 @@ export async function cleanKibanaIndices({
|
|||
while (true) {
|
||||
const resp = await client.deleteByQuery(
|
||||
{
|
||||
index: `.kibana,.kibana_task_manager`,
|
||||
index: ALL_SAVED_OBJECT_INDICES,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
|
|
@ -144,7 +156,7 @@ export async function cleanKibanaIndices({
|
|||
`.kibana rather than deleting the whole index`
|
||||
);
|
||||
|
||||
stats.deletedIndex('.kibana');
|
||||
ALL_SAVED_OBJECT_INDICES.forEach((indexPattern) => stats.deletedIndex(indexPattern));
|
||||
}
|
||||
|
||||
export async function createDefaultSpace({ index, client }: { index: string; client: Client }) {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"**/*.ts"
|
||||
],
|
||||
"kbn_references": [
|
||||
"@kbn/core-saved-objects-server",
|
||||
"@kbn/dev-utils",
|
||||
"@kbn/test",
|
||||
"@kbn/tooling-log",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import type { ProvidedType } from '@kbn/test';
|
||||
|
||||
import type { EsArchiverProvider } from '../es_archiver';
|
||||
|
|
@ -13,7 +14,6 @@ import type { RetryService } from '../retry';
|
|||
import type { KibanaServerProvider } from './kibana_server';
|
||||
|
||||
const ES_ARCHIVER_LOAD_METHODS = ['load', 'loadIfNeeded', 'unload', 'emptyKibanaIndex'] as const;
|
||||
const KIBANA_INDEX = '.kibana';
|
||||
|
||||
interface Options {
|
||||
esArchiver: ProvidedType<typeof EsArchiverProvider>;
|
||||
|
|
@ -38,7 +38,7 @@ export function extendEsArchiver({ esArchiver, kibanaServer, retry, defaults }:
|
|||
const statsKeys = Object.keys(stats);
|
||||
const kibanaKeys = statsKeys.filter(
|
||||
// this also matches stats keys like '.kibana_1' and '.kibana_2,.kibana_1'
|
||||
(key) => key.includes(KIBANA_INDEX) && stats[key].created
|
||||
(key) => key.includes(MAIN_SAVED_OBJECT_INDEX) && stats[key].created
|
||||
);
|
||||
|
||||
// if the kibana index was created by the esArchiver then update the uiSettings
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"**/*.ts",
|
||||
],
|
||||
"kbn_references": [
|
||||
"@kbn/core-saved-objects-server",
|
||||
"@kbn/tooling-log",
|
||||
"@kbn/es-archiver",
|
||||
"@kbn/test"
|
||||
|
|
|
|||
|
|
@ -8,9 +8,6 @@
|
|||
|
||||
import path from 'path';
|
||||
import { unlink } from 'fs/promises';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { Env } from '@kbn/config';
|
||||
import { getEnvOptions } from '@kbn/config-mocks';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
|
||||
import { Root } from '@kbn/core-root-server-internal';
|
||||
|
|
@ -19,8 +16,8 @@ import {
|
|||
createRootWithCorePlugins,
|
||||
type TestElasticsearchUtils,
|
||||
} from '@kbn/core-test-helpers-kbn-server';
|
||||
import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
|
||||
|
||||
const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version;
|
||||
const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log');
|
||||
|
||||
async function removeLogFile() {
|
||||
|
|
@ -105,8 +102,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
|
|||
await new Promise((resolve) => setTimeout(resolve, 10000));
|
||||
};
|
||||
|
||||
const migratedIndex = `.kibana_${kibanaVersion}_001`;
|
||||
|
||||
beforeAll(async () => {
|
||||
await removeLogFile();
|
||||
await startServers({
|
||||
|
|
@ -121,7 +116,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => {
|
|||
|
||||
it('copies all the document of the previous index to the new one', async () => {
|
||||
const migratedIndexResponse = await esClient.count({
|
||||
index: migratedIndex,
|
||||
index: ALL_SAVED_OBJECT_INDICES,
|
||||
});
|
||||
const oldIndexResponse = await esClient.count({
|
||||
index: '.kibana_1',
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ import {
|
|||
TestElasticsearchUtils,
|
||||
createTestServers as createkbnServerTestServers,
|
||||
} from '@kbn/core-test-helpers-kbn-server';
|
||||
import {
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
|
||||
const migrationDocLink = getMigrationDocLink().resolveMigrationFailures;
|
||||
const logFilePath = Path.join(__dirname, '7_13_corrupt_transform_failures.log');
|
||||
|
|
@ -114,18 +119,33 @@ describe('migration v2', () => {
|
|||
|
||||
const esClient: ElasticsearchClient = esServer.es.getClient();
|
||||
const docs = await esClient.search({
|
||||
index: '.kibana',
|
||||
index: [
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
TASK_MANAGER_SAVED_OBJECT_INDEX,
|
||||
ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
],
|
||||
_source: false,
|
||||
fields: ['_id'],
|
||||
size: 50,
|
||||
});
|
||||
|
||||
// 23 saved objects + 14 corrupt (discarded) = 37 total in the old index
|
||||
expect((docs.hits.total as SearchTotalHits).value).toEqual(23);
|
||||
// 34 saved objects (11 tasks + 23 misc) + 14 corrupt (discarded) = 48 total in the old indices
|
||||
expect((docs.hits.total as SearchTotalHits).value).toEqual(34);
|
||||
expect(docs.hits.hits.map(({ _id }) => _id).sort()).toEqual([
|
||||
'config:7.13.0',
|
||||
'index-pattern:logs-*',
|
||||
'index-pattern:metrics-*',
|
||||
'task:Actions-actions_telemetry',
|
||||
'task:Actions-cleanup_failed_action_executions',
|
||||
'task:Alerting-alerting_health_check',
|
||||
'task:Alerting-alerting_telemetry',
|
||||
'task:Alerts-alerts_invalidate_api_keys',
|
||||
'task:Lens-lens_telemetry',
|
||||
'task:apm-telemetry-task',
|
||||
'task:data_enhanced_search_sessions_monitor',
|
||||
'task:endpoint:user-artifact-packager:1.0.0',
|
||||
'task:security:endpoint-diagnostics:1.0.0',
|
||||
'task:session_cleanup',
|
||||
'ui-metric:console:DELETE_delete',
|
||||
'ui-metric:console:GET_get',
|
||||
'ui-metric:console:GET_search',
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ describe('migration v2', () => {
|
|||
await root.preboot();
|
||||
await root.setup();
|
||||
await expect(root.start()).rejects.toMatchInlineSnapshot(
|
||||
`[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715264 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
|
||||
`[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715312 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
|
||||
);
|
||||
|
||||
await retryAsync(
|
||||
|
|
@ -131,7 +131,7 @@ describe('migration v2', () => {
|
|||
expect(
|
||||
records.find((rec) =>
|
||||
rec.message.startsWith(
|
||||
`Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715264 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
|
||||
`Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715312 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
|
||||
)
|
||||
)
|
||||
).toBeDefined();
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ describe('migration actions', () => {
|
|||
await bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_with_docs',
|
||||
operations: docs.map(createBulkIndexOperationTuple),
|
||||
operations: docs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})();
|
||||
|
||||
|
|
@ -106,7 +106,7 @@ describe('migration actions', () => {
|
|||
await bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_with_write_block',
|
||||
operations: docs.map(createBulkIndexOperationTuple),
|
||||
operations: docs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})();
|
||||
await setWriteBlock({ client, index: 'existing_index_with_write_block' })();
|
||||
|
|
@ -307,7 +307,7 @@ describe('migration actions', () => {
|
|||
const res = (await bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'new_index_without_write_block',
|
||||
operations: sourceDocs.map(createBulkIndexOperationTuple),
|
||||
operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})()) as Either.Left<unknown>;
|
||||
|
||||
|
|
@ -887,7 +887,7 @@ describe('migration actions', () => {
|
|||
await bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'reindex_target_4',
|
||||
operations: sourceDocs.map(createBulkIndexOperationTuple),
|
||||
operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})();
|
||||
|
||||
|
|
@ -1445,7 +1445,7 @@ describe('migration actions', () => {
|
|||
await bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_without_mappings',
|
||||
operations: sourceDocs.map(createBulkIndexOperationTuple),
|
||||
operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})();
|
||||
|
||||
|
|
@ -1895,7 +1895,7 @@ describe('migration actions', () => {
|
|||
const task = bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_with_docs',
|
||||
operations: newDocs.map(createBulkIndexOperationTuple),
|
||||
operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
});
|
||||
|
||||
|
|
@ -1921,7 +1921,7 @@ describe('migration actions', () => {
|
|||
operations: [
|
||||
...existingDocs,
|
||||
{ _source: { title: 'doc 8' } } as unknown as SavedObjectsRawDoc,
|
||||
].map(createBulkIndexOperationTuple),
|
||||
].map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
});
|
||||
await expect(task()).resolves.toMatchInlineSnapshot(`
|
||||
|
|
@ -1941,7 +1941,7 @@ describe('migration actions', () => {
|
|||
bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_with_write_block',
|
||||
operations: newDocs.map(createBulkIndexOperationTuple),
|
||||
operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
refresh: 'wait_for',
|
||||
})()
|
||||
).resolves.toMatchInlineSnapshot(`
|
||||
|
|
@ -1964,7 +1964,7 @@ describe('migration actions', () => {
|
|||
const task = bulkOverwriteTransformedDocuments({
|
||||
client,
|
||||
index: 'existing_index_with_docs',
|
||||
operations: newDocs.map(createBulkIndexOperationTuple),
|
||||
operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)),
|
||||
});
|
||||
await expect(task()).resolves.toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,386 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
|
||||
import {
|
||||
type ISavedObjectTypeRegistry,
|
||||
type SavedObjectsType,
|
||||
MAIN_SAVED_OBJECT_INDEX,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
readLog,
|
||||
startElasticsearch,
|
||||
getKibanaMigratorTestKit,
|
||||
getCurrentVersionTypeRegistry,
|
||||
overrideTypeRegistry,
|
||||
clearLog,
|
||||
getAggregatedTypesCount,
|
||||
currentVersion,
|
||||
type KibanaMigratorTestKit,
|
||||
} from '../kibana_migrator_test_kit';
|
||||
import { delay } from '../test_utils';
|
||||
|
||||
// define a type => index distribution
|
||||
const RELOCATE_TYPES: Record<string, string> = {
|
||||
dashboard: '.kibana_so_ui',
|
||||
visualization: '.kibana_so_ui',
|
||||
'canvas-workpad': '.kibana_so_ui',
|
||||
search: '.kibana_so_search',
|
||||
task: '.kibana_task_manager',
|
||||
// the remaining types will be forced to go to '.kibana',
|
||||
// overriding `indexPattern: foo` defined in the registry
|
||||
};
|
||||
|
||||
describe('split .kibana index into multiple system indices', () => {
|
||||
let esServer: TestElasticsearchUtils['es'];
|
||||
let typeRegistry: ISavedObjectTypeRegistry;
|
||||
let migratorTestKitFactory: () => Promise<KibanaMigratorTestKit>;
|
||||
|
||||
beforeAll(async () => {
|
||||
typeRegistry = await getCurrentVersionTypeRegistry({ oss: false });
|
||||
|
||||
esServer = await startElasticsearch({
|
||||
dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'),
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await clearLog();
|
||||
});
|
||||
|
||||
describe('when migrating from a legacy version', () => {
|
||||
it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => {
|
||||
const updatedTypeRegistry = overrideTypeRegistry(
|
||||
typeRegistry,
|
||||
(type: SavedObjectsType<any>) => {
|
||||
return {
|
||||
...type,
|
||||
indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX,
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
migratorTestKitFactory = () =>
|
||||
getKibanaMigratorTestKit({
|
||||
types: updatedTypeRegistry.getAllTypes(),
|
||||
kibanaIndex: '.kibana',
|
||||
});
|
||||
|
||||
const { runMigrations, client } = await migratorTestKitFactory();
|
||||
|
||||
// count of types in the legacy index
|
||||
expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({
|
||||
'canvas-workpad': 3,
|
||||
config: 1,
|
||||
dashboard: 3,
|
||||
'index-pattern': 3,
|
||||
map: 3,
|
||||
'maps-telemetry': 1,
|
||||
'sample-data-telemetry': 3,
|
||||
search: 2,
|
||||
telemetry: 1,
|
||||
space: 1,
|
||||
visualization: 39,
|
||||
});
|
||||
|
||||
await runMigrations();
|
||||
|
||||
await client.indices.refresh({
|
||||
index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'],
|
||||
});
|
||||
|
||||
expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({
|
||||
'index-pattern': 3,
|
||||
map: 3,
|
||||
'sample-data-telemetry': 3,
|
||||
config: 1,
|
||||
telemetry: 1,
|
||||
space: 1,
|
||||
});
|
||||
expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({
|
||||
search: 2,
|
||||
});
|
||||
expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({
|
||||
visualization: 39,
|
||||
'canvas-workpad': 3,
|
||||
dashboard: 3,
|
||||
});
|
||||
|
||||
const indicesInfo = await client.indices.get({ index: '.kibana*' });
|
||||
expect(indicesInfo).toEqual(
|
||||
expect.objectContaining({
|
||||
'.kibana_8.8.0_001': {
|
||||
aliases: { '.kibana': expect.any(Object), '.kibana_8.8.0': expect.any(Object) },
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
},
|
||||
settings: { index: expect.any(Object) },
|
||||
},
|
||||
'.kibana_so_search_8.8.0_001': {
|
||||
aliases: {
|
||||
'.kibana_so_search': expect.any(Object),
|
||||
'.kibana_so_search_8.8.0': expect.any(Object),
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
},
|
||||
settings: { index: expect.any(Object) },
|
||||
},
|
||||
'.kibana_so_ui_8.8.0_001': {
|
||||
aliases: {
|
||||
'.kibana_so_ui': expect.any(Object),
|
||||
'.kibana_so_ui_8.8.0': expect.any(Object),
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
},
|
||||
settings: { index: expect.any(Object) },
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
expect(indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap)
|
||||
.toMatchInlineSnapshot(`
|
||||
Object {
|
||||
".kibana": Array [
|
||||
"action",
|
||||
"action_task_params",
|
||||
"alert",
|
||||
"api_key_pending_invalidation",
|
||||
"apm-indices",
|
||||
"apm-server-schema",
|
||||
"apm-service-group",
|
||||
"apm-telemetry",
|
||||
"app_search_telemetry",
|
||||
"application_usage_daily",
|
||||
"application_usage_totals",
|
||||
"canvas-element",
|
||||
"canvas-workpad-template",
|
||||
"cases",
|
||||
"cases-comments",
|
||||
"cases-configure",
|
||||
"cases-connector-mappings",
|
||||
"cases-telemetry",
|
||||
"cases-user-actions",
|
||||
"config",
|
||||
"config-global",
|
||||
"connector_token",
|
||||
"core-usage-stats",
|
||||
"csp-rule-template",
|
||||
"endpoint:user-artifact",
|
||||
"endpoint:user-artifact-manifest",
|
||||
"enterprise_search_telemetry",
|
||||
"epm-packages",
|
||||
"epm-packages-assets",
|
||||
"event_loop_delays_daily",
|
||||
"exception-list",
|
||||
"exception-list-agnostic",
|
||||
"file",
|
||||
"file-upload-usage-collection-telemetry",
|
||||
"fileShare",
|
||||
"fleet-fleet-server-host",
|
||||
"fleet-message-signing-keys",
|
||||
"fleet-preconfiguration-deletion-record",
|
||||
"fleet-proxy",
|
||||
"graph-workspace",
|
||||
"guided-onboarding-guide-state",
|
||||
"guided-onboarding-plugin-state",
|
||||
"index-pattern",
|
||||
"infrastructure-monitoring-log-view",
|
||||
"infrastructure-ui-source",
|
||||
"ingest-agent-policies",
|
||||
"ingest-download-sources",
|
||||
"ingest-outputs",
|
||||
"ingest-package-policies",
|
||||
"ingest_manager_settings",
|
||||
"inventory-view",
|
||||
"kql-telemetry",
|
||||
"legacy-url-alias",
|
||||
"lens",
|
||||
"lens-ui-telemetry",
|
||||
"maintenance-window",
|
||||
"map",
|
||||
"metrics-explorer-view",
|
||||
"ml-job",
|
||||
"ml-module",
|
||||
"ml-trained-model",
|
||||
"monitoring-telemetry",
|
||||
"osquery-manager-usage-metric",
|
||||
"osquery-pack",
|
||||
"osquery-pack-asset",
|
||||
"osquery-saved-query",
|
||||
"query",
|
||||
"rules-settings",
|
||||
"sample-data-telemetry",
|
||||
"search-session",
|
||||
"search-telemetry",
|
||||
"security-rule",
|
||||
"security-solution-signals-migration",
|
||||
"siem-detection-engine-rule-actions",
|
||||
"siem-ui-timeline",
|
||||
"siem-ui-timeline-note",
|
||||
"siem-ui-timeline-pinned-event",
|
||||
"slo",
|
||||
"space",
|
||||
"spaces-usage-stats",
|
||||
"synthetics-monitor",
|
||||
"synthetics-param",
|
||||
"synthetics-privates-locations",
|
||||
"tag",
|
||||
"telemetry",
|
||||
"ui-metric",
|
||||
"upgrade-assistant-ml-upgrade-operation",
|
||||
"upgrade-assistant-reindex-operation",
|
||||
"uptime-dynamic-settings",
|
||||
"uptime-synthetics-api-key",
|
||||
"url",
|
||||
"usage-counters",
|
||||
"workplace_search_telemetry",
|
||||
],
|
||||
".kibana_so_search": Array [
|
||||
"search",
|
||||
],
|
||||
".kibana_so_ui": Array [
|
||||
"canvas-workpad",
|
||||
"dashboard",
|
||||
"visualization",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
}
|
||||
`);
|
||||
|
||||
const logs = await readLog();
|
||||
|
||||
// .kibana_task_manager index exists and has no aliases => LEGACY_* migration path
|
||||
expect(logs).toMatch('[.kibana_task_manager] INIT -> LEGACY_SET_WRITE_BLOCK.');
|
||||
// .kibana_task_manager migrator is NOT involved in relocation, must not sync
|
||||
expect(logs).not.toMatch('[.kibana_task_manager] READY_TO_REINDEX_SYNC');
|
||||
|
||||
// newer indices migrators did not exist, so they all have to reindex (create temp index + sync)
|
||||
['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => {
|
||||
expect(logs).toMatch(`[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`);
|
||||
expect(logs).toMatch(`[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`);
|
||||
// no docs to reindex, as source index did NOT exist
|
||||
expect(logs).toMatch(`[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`);
|
||||
});
|
||||
|
||||
// the .kibana migrator is involved in a relocation, it must also reindex
|
||||
expect(logs).toMatch('[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.');
|
||||
expect(logs).toMatch('[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS.');
|
||||
expect(logs).toMatch('[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.');
|
||||
expect(logs).toMatch('[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.');
|
||||
expect(logs).toMatch('[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.');
|
||||
expect(logs).toMatch('[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.');
|
||||
expect(logs).toMatch('[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.');
|
||||
expect(logs).toMatch(
|
||||
'[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.'
|
||||
);
|
||||
expect(logs).toMatch('[.kibana] Starting to process 59 documents.');
|
||||
expect(logs).toMatch(
|
||||
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.'
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
'[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.'
|
||||
);
|
||||
expect(logs).toMatch('[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.');
|
||||
expect(logs).toMatch(
|
||||
'[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.'
|
||||
);
|
||||
expect(logs).toMatch('[.kibana] Processed 59 documents out of 59.');
|
||||
expect(logs).toMatch(
|
||||
'[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.'
|
||||
);
|
||||
expect(logs).toMatch('[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.');
|
||||
|
||||
// after .kibana migrator is done relocating documents
|
||||
// the 3 migrators share the final part of the flow
|
||||
[
|
||||
['.kibana', 8],
|
||||
['.kibana_so_ui', 45],
|
||||
['.kibana_so_search', 2],
|
||||
].forEach(([index, docCount]) => {
|
||||
expect(logs).toMatch(`[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`);
|
||||
expect(logs).toMatch(`[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`);
|
||||
|
||||
expect(logs).toMatch(`[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`);
|
||||
expect(logs).toMatch(`[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`
|
||||
);
|
||||
expect(logs).toMatch(`[${index}] Starting to process ${docCount} documents.`);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_TRANSFORM.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] OUTDATED_DOCUMENTS_TRANSFORM -> TRANSFORMED_DOCUMENTS_BULK_INDEX.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_REFRESH.`
|
||||
);
|
||||
expect(logs).toMatch(`[${index}] OUTDATED_DOCUMENTS_REFRESH -> CHECK_TARGET_MAPPINGS.`);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`
|
||||
);
|
||||
|
||||
expect(logs).toMatch(
|
||||
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`
|
||||
);
|
||||
expect(logs).toMatch(
|
||||
`[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.`
|
||||
);
|
||||
|
||||
expect(logs).toMatch(`[${index}] MARK_VERSION_INDEX_READY -> DONE.`);
|
||||
expect(logs).toMatch(`[${index}] Migration completed`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// we run the migrator again to ensure that the next time state is loaded everything still works as expected
|
||||
const { runMigrations } = await migratorTestKitFactory();
|
||||
await clearLog();
|
||||
await runMigrations();
|
||||
|
||||
const logs = await readLog();
|
||||
expect(logs).not.toMatch('REINDEX');
|
||||
expect(logs).not.toMatch('CREATE');
|
||||
expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await esServer?.stop();
|
||||
await delay(10);
|
||||
});
|
||||
});
|
||||
|
|
@ -36,7 +36,7 @@ import { type LoggingConfigType, LoggingSystem } from '@kbn/core-logging-server-
|
|||
import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server';
|
||||
import { esTestConfig, kibanaServerTestUser } from '@kbn/test';
|
||||
import type { LoggerFactory } from '@kbn/logging';
|
||||
import { createTestServers } from '@kbn/core-test-helpers-kbn-server';
|
||||
import { createRootWithCorePlugins, createTestServers } from '@kbn/core-test-helpers-kbn-server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { registerServiceConfig } from '@kbn/core-root-server-internal';
|
||||
import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server';
|
||||
|
|
@ -72,7 +72,7 @@ export interface KibanaMigratorTestKitParams {
|
|||
export interface KibanaMigratorTestKit {
|
||||
client: ElasticsearchClient;
|
||||
migrator: IKibanaMigrator;
|
||||
runMigrations: (rerun?: boolean) => Promise<MigrationResult[]>;
|
||||
runMigrations: () => Promise<MigrationResult[]>;
|
||||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
savedObjectsRepository: ISavedObjectsRepository;
|
||||
}
|
||||
|
|
@ -282,6 +282,42 @@ const getMigrator = async (
|
|||
});
|
||||
};
|
||||
|
||||
export const getAggregatedTypesCount = async (client: ElasticsearchClient, index: string) => {
|
||||
await client.indices.refresh();
|
||||
const response = await client.search<unknown, { typesAggregation: { buckets: any[] } }>({
|
||||
index,
|
||||
_source: false,
|
||||
aggs: {
|
||||
typesAggregation: {
|
||||
terms: {
|
||||
// assign type __UNKNOWN__ to those documents that don't define one
|
||||
missing: '__UNKNOWN__',
|
||||
field: 'type',
|
||||
size: 100,
|
||||
},
|
||||
aggs: {
|
||||
docs: {
|
||||
top_hits: {
|
||||
size: 10,
|
||||
_source: {
|
||||
excludes: ['*'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return (response.aggregations!.typesAggregation.buckets as unknown as any).reduce(
|
||||
(acc: any, current: any) => {
|
||||
acc[current.key] = current.doc_count;
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
};
|
||||
|
||||
const registerTypes = (
|
||||
typeRegistry: SavedObjectTypeRegistry,
|
||||
types?: Array<SavedObjectsType<any>>
|
||||
|
|
@ -390,6 +426,28 @@ export const getIncompatibleMappingsMigrator = async ({
|
|||
});
|
||||
};
|
||||
|
||||
export const getCurrentVersionTypeRegistry = async ({
|
||||
oss,
|
||||
}: {
|
||||
oss: boolean;
|
||||
}): Promise<ISavedObjectTypeRegistry> => {
|
||||
const root = createRootWithCorePlugins({}, { oss });
|
||||
await root.preboot();
|
||||
const coreSetup = await root.setup();
|
||||
const typeRegistry = coreSetup.savedObjects.getTypeRegistry();
|
||||
root.shutdown(); // do not await for it, or we might block the tests
|
||||
return typeRegistry;
|
||||
};
|
||||
|
||||
export const overrideTypeRegistry = (
|
||||
typeRegistry: ISavedObjectTypeRegistry,
|
||||
transform: (type: SavedObjectsType<any>) => SavedObjectsType<any>
|
||||
): ISavedObjectTypeRegistry => {
|
||||
const updatedTypeRegistry = new SavedObjectTypeRegistry();
|
||||
typeRegistry.getAllTypes().forEach((type) => updatedTypeRegistry.registerType(transform(type)));
|
||||
return updatedTypeRegistry;
|
||||
};
|
||||
|
||||
export const readLog = async (logFilePath: string = defaultLogFilePath): Promise<string> => {
|
||||
await delay(0.1);
|
||||
return await fs.readFile(logFilePath, 'utf-8');
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
import Hapi from '@hapi/hapi';
|
||||
import h2o2 from '@hapi/h2o2';
|
||||
import { URL } from 'url';
|
||||
import type { SavedObject } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObject, ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
|
||||
import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server';
|
||||
import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal';
|
||||
import { Root } from '@kbn/core-root-server-internal';
|
||||
|
|
@ -18,6 +18,7 @@ import {
|
|||
createTestServers,
|
||||
type TestElasticsearchUtils,
|
||||
} from '@kbn/core-test-helpers-kbn-server';
|
||||
import { kibanaPackageJson as pkg } from '@kbn/repo-info';
|
||||
import {
|
||||
declareGetRoute,
|
||||
declareDeleteRoute,
|
||||
|
|
@ -30,6 +31,7 @@ import {
|
|||
declarePostUpdateByQueryRoute,
|
||||
declarePassthroughRoute,
|
||||
setProxyInterrupt,
|
||||
allCombinationsPermutations,
|
||||
} from './repository_with_proxy_utils';
|
||||
|
||||
let esServer: TestElasticsearchUtils;
|
||||
|
|
@ -98,17 +100,24 @@ describe('404s from proxies', () => {
|
|||
await hapiServer.register(h2o2);
|
||||
// register specific routes to modify the response and a catch-all to relay the request/response as-is
|
||||
|
||||
declareGetRoute(hapiServer, esHostname, esPort);
|
||||
declareDeleteRoute(hapiServer, esHostname, esPort);
|
||||
declarePostUpdateRoute(hapiServer, esHostname, esPort);
|
||||
allCombinationsPermutations(
|
||||
ALL_SAVED_OBJECT_INDICES.map((indexPattern) => `${indexPattern}_${pkg.version}`)
|
||||
)
|
||||
.map((indices) => indices.join(','))
|
||||
.forEach((kbnIndexPath) => {
|
||||
declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
|
||||
declareGetSearchRoute(hapiServer, esHostname, esPort);
|
||||
declarePostSearchRoute(hapiServer, esHostname, esPort);
|
||||
declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath);
|
||||
});
|
||||
|
||||
// register index-agnostic routes
|
||||
declarePostBulkRoute(hapiServer, esHostname, esPort);
|
||||
declarePostMgetRoute(hapiServer, esHostname, esPort);
|
||||
declarePostPitRoute(hapiServer, esHostname, esPort);
|
||||
declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort);
|
||||
|
||||
declarePassthroughRoute(hapiServer, esHostname, esPort);
|
||||
|
||||
await hapiServer.start();
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
*/
|
||||
import Hapi from '@hapi/hapi';
|
||||
import { IncomingMessage } from 'http';
|
||||
import { kibanaPackageJson as pkg } from '@kbn/repo-info';
|
||||
|
||||
// proxy setup
|
||||
const defaultProxyOptions = (hostname: string, port: string) => ({
|
||||
|
|
@ -52,10 +51,13 @@ const proxyOnResponseHandler = async (res: IncomingMessage, h: Hapi.ResponseTool
|
|||
.code(404);
|
||||
};
|
||||
|
||||
const kbnIndex = `.kibana_${pkg.version}`;
|
||||
|
||||
// GET /.kibana_8.0.0/_doc/{type*} route (repository.get calls)
|
||||
export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declareGetRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'GET',
|
||||
path: `/${kbnIndex}/_doc/{type*}`,
|
||||
|
|
@ -70,7 +72,12 @@ export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port:
|
|||
},
|
||||
});
|
||||
// DELETE /.kibana_8.0.0/_doc/{type*} route (repository.delete calls)
|
||||
export const declareDeleteRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declareDeleteRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'DELETE',
|
||||
path: `/${kbnIndex}/_doc/{_id*}`,
|
||||
|
|
@ -133,7 +140,12 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string,
|
|||
},
|
||||
});
|
||||
// GET _search route
|
||||
export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declareGetSearchRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'GET',
|
||||
path: `/${kbnIndex}/_search`,
|
||||
|
|
@ -149,7 +161,12 @@ export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string,
|
|||
},
|
||||
});
|
||||
// POST _search route (`find` calls)
|
||||
export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declarePostSearchRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'POST',
|
||||
path: `/${kbnIndex}/_search`,
|
||||
|
|
@ -168,7 +185,12 @@ export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string
|
|||
},
|
||||
});
|
||||
// POST _update
|
||||
export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declarePostUpdateRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'POST',
|
||||
path: `/${kbnIndex}/_update/{_id*}`,
|
||||
|
|
@ -187,7 +209,12 @@ export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string
|
|||
},
|
||||
});
|
||||
// POST _pit
|
||||
export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, port: string) =>
|
||||
export const declarePostPitRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'POST',
|
||||
path: `/${kbnIndex}/_pit`,
|
||||
|
|
@ -209,7 +236,8 @@ export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, p
|
|||
export const declarePostUpdateByQueryRoute = (
|
||||
hapiServer: Hapi.Server,
|
||||
hostname: string,
|
||||
port: string
|
||||
port: string,
|
||||
kbnIndex: string
|
||||
) =>
|
||||
hapiServer.route({
|
||||
method: 'POST',
|
||||
|
|
@ -244,3 +272,22 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin
|
|||
},
|
||||
},
|
||||
});
|
||||
|
||||
export function allCombinationsPermutations<T>(collection: T[]): T[][] {
|
||||
const recur = (subcollection: T[], size: number): T[][] => {
|
||||
if (size <= 0) {
|
||||
return [[]];
|
||||
}
|
||||
const permutations: T[][] = [];
|
||||
subcollection.forEach((value, index, array) => {
|
||||
array = array.slice();
|
||||
array.splice(index, 1);
|
||||
recur(array, size - 1).forEach((permutation) => {
|
||||
permutation.unshift(value);
|
||||
permutations.push(permutation);
|
||||
});
|
||||
});
|
||||
return permutations;
|
||||
};
|
||||
return collection.map((_, n) => recur(collection, n + 1)).flat();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -239,7 +239,7 @@ import { cmServicesDefinition } from '../../common/content_management/cm_service
|
|||
* that we won't leak any additional fields in our Response, even when the SO client adds new fields to its responses.
|
||||
*/
|
||||
function savedObjectToMapItem(
|
||||
savedObject: SavedObject<MapSavedObjectAttributes>
|
||||
savedObject: SavedObject<MapAttributes>
|
||||
): MapItem {
|
||||
const {
|
||||
id,
|
||||
|
|
@ -293,7 +293,7 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
|
|||
alias_purpose: aliasPurpose,
|
||||
alias_target_id: aliasTargetId,
|
||||
outcome,
|
||||
} = await soClient.resolve<MapSavedObjectAttributes>(SO_TYPE, id);
|
||||
} = await soClient.resolve<MapAttributes>(SO_TYPE, id);
|
||||
|
||||
const response: MapGetOut = {
|
||||
item: savedObjectToMapItem(savedObject),
|
||||
|
|
@ -327,8 +327,8 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
|
|||
|
||||
// Validate input (data & options) & UP transform them to the latest version
|
||||
const { value: dataToLatest, error: dataError } = transforms.create.in.data.up<
|
||||
MapSavedObjectAttributes,
|
||||
MapSavedObjectAttributes
|
||||
MapAttributes,
|
||||
MapAttributes
|
||||
>(data);
|
||||
if (dataError) {
|
||||
throw Boom.badRequest(`Invalid payload. ${dataError.message}`);
|
||||
|
|
@ -345,7 +345,7 @@ export class MapsStorage implements ContentStorage<MapSavedObject, PartialMapSav
|
|||
// - both are on the latest version
|
||||
|
||||
// Save data in DB
|
||||
const savedObject = await soClient.create<MapSavedObjectAttributes>(
|
||||
const savedObject = await soClient.create<MapAttributes>(
|
||||
SO_TYPE,
|
||||
dataToLatest,
|
||||
optionsToLatest
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import {
|
||||
createDashboardSavedObjectTypeMigrations,
|
||||
|
|
@ -18,6 +19,7 @@ export const createDashboardSavedObjectType = ({
|
|||
migrationDeps: DashboardSavedObjectTypeMigrationsDeps;
|
||||
}): SavedObjectsType => ({
|
||||
name: 'dashboard',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
hidden: false,
|
||||
namespaceType: 'multiple-isolated',
|
||||
convertToMultiNamespaceTypeVersion: '8.0.0',
|
||||
|
|
|
|||
|
|
@ -114,12 +114,14 @@ export function dashboardTaskRunner(logger: Logger, core: CoreSetup, embeddable:
|
|||
return dashboardData;
|
||||
};
|
||||
|
||||
const kibanaIndex = core.savedObjects.getKibanaIndex();
|
||||
const dashboardIndex = await core
|
||||
.getStartServices()
|
||||
.then(([coreStart]) => coreStart.savedObjects.getIndexForType('dashboard'));
|
||||
const pageSize = 50;
|
||||
|
||||
const searchParams = {
|
||||
size: pageSize,
|
||||
index: kibanaIndex,
|
||||
index: dashboardIndex,
|
||||
ignore_unavailable: true,
|
||||
filter_path: ['hits.hits', '_scroll_id'],
|
||||
body: { query: { bool: { filter: { term: { type: 'dashboard' } } } } },
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@
|
|||
"@kbn/saved-objects-finder-plugin",
|
||||
"@kbn/saved-objects-management-plugin",
|
||||
"@kbn/shared-ux-button-toolbar",
|
||||
"@kbn/core-saved-objects-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ export class KqlTelemetryService implements Plugin<void> {
|
|||
|
||||
if (usageCollection) {
|
||||
try {
|
||||
makeKQLUsageCollector(usageCollection, savedObjects.getKibanaIndex());
|
||||
const getIndexForType = (type: string) =>
|
||||
getStartServices().then(([coreStart]) => coreStart.savedObjects.getIndexForType(type));
|
||||
makeKQLUsageCollector(usageCollection, getIndexForType);
|
||||
} catch (e) {
|
||||
this.initializerContext.logger
|
||||
.get('kql-telemetry')
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ function setupMockCallCluster(
|
|||
describe('makeKQLUsageCollector', () => {
|
||||
describe('fetch method', () => {
|
||||
beforeEach(() => {
|
||||
fetch = fetchProvider('.kibana');
|
||||
fetch = fetchProvider(() => Promise.resolve('.kibana'));
|
||||
});
|
||||
|
||||
it('should return opt in data from the .kibana/kql-telemetry doc', async () => {
|
||||
|
|
|
|||
|
|
@ -18,19 +18,19 @@ export interface Usage {
|
|||
defaultQueryLanguage: string;
|
||||
}
|
||||
|
||||
export function fetchProvider(index: string) {
|
||||
export function fetchProvider(getIndexForType: (type: string) => Promise<string>) {
|
||||
return async ({ esClient }: CollectorFetchContext): Promise<Usage> => {
|
||||
const [response, config] = await Promise.all([
|
||||
esClient.get(
|
||||
{
|
||||
index,
|
||||
index: await getIndexForType('kql-telemetry'),
|
||||
id: 'kql-telemetry:kql-telemetry',
|
||||
},
|
||||
{ ignore: [404] }
|
||||
),
|
||||
esClient.search(
|
||||
{
|
||||
index,
|
||||
index: await getIndexForType('config'),
|
||||
body: { query: { term: { type: 'config' } } },
|
||||
},
|
||||
{ ignore: [404] }
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
|
|||
describe('makeKQLUsageCollector', () => {
|
||||
let usageCollectionMock: jest.Mocked<UsageCollectionSetup>;
|
||||
|
||||
const getIndexForType = () => Promise.resolve('.kibana');
|
||||
|
||||
beforeEach(() => {
|
||||
usageCollectionMock = {
|
||||
makeUsageCollector: jest.fn(),
|
||||
|
|
@ -20,12 +22,12 @@ describe('makeKQLUsageCollector', () => {
|
|||
});
|
||||
|
||||
it('should call registerCollector', () => {
|
||||
makeKQLUsageCollector(usageCollectionMock as UsageCollectionSetup, '.kibana');
|
||||
makeKQLUsageCollector(usageCollectionMock as UsageCollectionSetup, getIndexForType);
|
||||
expect(usageCollectionMock.registerCollector).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should call makeUsageCollector with type = kql', () => {
|
||||
makeKQLUsageCollector(usageCollectionMock as UsageCollectionSetup, '.kibana');
|
||||
makeKQLUsageCollector(usageCollectionMock as UsageCollectionSetup, getIndexForType);
|
||||
expect(usageCollectionMock.makeUsageCollector).toHaveBeenCalledTimes(1);
|
||||
expect(usageCollectionMock.makeUsageCollector.mock.calls[0][0].type).toBe('kql');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,10 +9,13 @@
|
|||
import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
|
||||
import { fetchProvider, Usage } from './fetch';
|
||||
|
||||
export function makeKQLUsageCollector(usageCollection: UsageCollectionSetup, kibanaIndex: string) {
|
||||
export function makeKQLUsageCollector(
|
||||
usageCollection: UsageCollectionSetup,
|
||||
getIndexForType: (type: string) => Promise<string>
|
||||
) {
|
||||
const kqlUsageCollector = usageCollection.makeUsageCollector<Usage>({
|
||||
type: 'kql',
|
||||
fetch: fetchProvider(kibanaIndex),
|
||||
fetch: fetchProvider(getIndexForType),
|
||||
isReady: () => true,
|
||||
schema: {
|
||||
optInCount: { type: 'long' },
|
||||
|
|
|
|||
|
|
@ -6,11 +6,13 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { SCHEMA_KQL_TELEMETRY_V8_8_0 } from './schemas/kql_telemetry';
|
||||
|
||||
export const kqlTelemetry: SavedObjectsType = {
|
||||
name: 'kql-telemetry',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
namespaceType: 'agnostic',
|
||||
hidden: false,
|
||||
mappings: {
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { savedQueryMigrations } from './migrations/query';
|
||||
import { SCHEMA_QUERY_V8_8_0 } from './schemas/query';
|
||||
|
||||
export const querySavedObjectType: SavedObjectsType = {
|
||||
name: 'query',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
hidden: false,
|
||||
namespaceType: 'multiple-isolated',
|
||||
convertToMultiNamespaceTypeVersion: '8.0.0',
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { migrate712 } from './migrations/to_v7_12_0';
|
||||
import { SCHEMA_SEARCH_TELEMETRY_V8_8_0 } from './schemas/search_telemetry';
|
||||
|
||||
export const searchTelemetry: SavedObjectsType = {
|
||||
name: 'search-telemetry',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
namespaceType: 'agnostic',
|
||||
hidden: false,
|
||||
mappings: {
|
||||
|
|
|
|||
|
|
@ -13,11 +13,12 @@ interface SearchTelemetry {
|
|||
'search-telemetry': CollectedUsage;
|
||||
}
|
||||
|
||||
export function fetchProvider(kibanaIndex: string) {
|
||||
export function fetchProvider(getIndexForType: (type: string) => Promise<string>) {
|
||||
return async ({ esClient }: CollectorFetchContext): Promise<ReportedUsage> => {
|
||||
const searchIndex = await getIndexForType('search-telemetry');
|
||||
const esResponse = await esClient.search<SearchTelemetry>(
|
||||
{
|
||||
index: kibanaIndex,
|
||||
index: searchIndex,
|
||||
body: {
|
||||
query: { term: { type: { value: 'search-telemetry' } } },
|
||||
},
|
||||
|
|
|
|||
|
|
@ -21,12 +21,15 @@ export interface ReportedUsage {
|
|||
averageDuration: number | null;
|
||||
}
|
||||
|
||||
export function registerUsageCollector(usageCollection: UsageCollectionSetup, kibanaIndex: string) {
|
||||
export function registerUsageCollector(
|
||||
usageCollection: UsageCollectionSetup,
|
||||
getIndexForType: (type: string) => Promise<string>
|
||||
) {
|
||||
try {
|
||||
const collector = usageCollection.makeUsageCollector<ReportedUsage>({
|
||||
type: 'search',
|
||||
isReady: () => true,
|
||||
fetch: fetchProvider(kibanaIndex),
|
||||
fetch: fetchProvider(getIndexForType),
|
||||
schema: {
|
||||
successCount: { type: 'long' },
|
||||
errorCount: { type: 'long' },
|
||||
|
|
|
|||
|
|
@ -17,12 +17,13 @@ describe('fetchProvider', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
const kibanaIndex = '123';
|
||||
const getIndexForType = () => Promise.resolve(kibanaIndex);
|
||||
mockLogger = {
|
||||
warn: jest.fn(),
|
||||
debug: jest.fn(),
|
||||
} as any;
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
fetchFn = fetchProvider(kibanaIndex, mockLogger);
|
||||
fetchFn = fetchProvider(getIndexForType, mockLogger);
|
||||
});
|
||||
|
||||
test('returns when ES returns no results', async () => {
|
||||
|
|
|
|||
|
|
@ -17,11 +17,12 @@ interface SessionPersistedTermsBucket {
|
|||
doc_count: number;
|
||||
}
|
||||
|
||||
export function fetchProvider(kibanaIndex: string, logger: Logger) {
|
||||
export function fetchProvider(getIndexForType: (type: string) => Promise<string>, logger: Logger) {
|
||||
return async ({ esClient }: CollectorFetchContext): Promise<ReportedUsage> => {
|
||||
try {
|
||||
const searchSessionIndex = await getIndexForType(SEARCH_SESSION_TYPE);
|
||||
const esResponse = await esClient.search<unknown>({
|
||||
index: kibanaIndex,
|
||||
index: searchSessionIndex,
|
||||
body: {
|
||||
size: 0,
|
||||
aggs: {
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@ export interface ReportedUsage {
|
|||
|
||||
export function registerUsageCollector(
|
||||
usageCollection: UsageCollectionSetup,
|
||||
kibanaIndex: string,
|
||||
getIndexForType: (type: string) => Promise<string>,
|
||||
logger: Logger
|
||||
) {
|
||||
try {
|
||||
const collector = usageCollection.makeUsageCollector<ReportedUsage>({
|
||||
type: 'search-session',
|
||||
isReady: () => true,
|
||||
fetch: fetchProvider(kibanaIndex, logger),
|
||||
fetch: fetchProvider(getIndexForType, logger),
|
||||
schema: {
|
||||
transientCount: { type: 'long' },
|
||||
persistedCount: { type: 'long' },
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@
|
|||
*/
|
||||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { SEARCH_SESSION_TYPE } from '../../../common';
|
||||
import { searchSessionSavedObjectMigrations } from './search_session_migration';
|
||||
|
||||
export const searchSessionSavedObjectType: SavedObjectsType = {
|
||||
name: SEARCH_SESSION_TYPE,
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
namespaceType: 'single',
|
||||
hidden: true,
|
||||
mappings: {
|
||||
|
|
|
|||
|
|
@ -213,12 +213,10 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
|
|||
|
||||
core.savedObjects.registerType(searchTelemetry);
|
||||
if (usageCollection) {
|
||||
registerSearchUsageCollector(usageCollection, core.savedObjects.getKibanaIndex());
|
||||
registerSearchSessionUsageCollector(
|
||||
usageCollection,
|
||||
core.savedObjects.getKibanaIndex(),
|
||||
this.logger
|
||||
);
|
||||
const getIndexForType = (type: string) =>
|
||||
core.getStartServices().then(([coreStart]) => coreStart.savedObjects.getIndexForType(type));
|
||||
registerSearchUsageCollector(usageCollection, getIndexForType);
|
||||
registerSearchSessionUsageCollector(usageCollection, getIndexForType, this.logger);
|
||||
}
|
||||
|
||||
expressions.registerFunction(getEsaggs({ getStartServices: core.getStartServices }));
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@
|
|||
"@kbn/config",
|
||||
"@kbn/config-schema",
|
||||
"@kbn/core-application-browser",
|
||||
"@kbn/core-saved-objects-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@
|
|||
*/
|
||||
|
||||
import type { SavedObjectsType } from '@kbn/core/server';
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { indexPatternSavedObjectTypeMigrations } from './index_pattern_migrations';
|
||||
import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../common';
|
||||
|
||||
export const dataViewSavedObjectType: SavedObjectsType = {
|
||||
name: DATA_VIEW_SAVED_OBJECT_TYPE,
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
hidden: false,
|
||||
namespaceType: 'multiple',
|
||||
convertToMultiNamespaceTypeVersion: '8.0.0',
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
"@kbn/utility-types-jest",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/core-http-server",
|
||||
"@kbn/core-saved-objects-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import { registerSampleDatasetWithIntegration } from './lib/register_with_integr
|
|||
|
||||
export class SampleDataRegistry {
|
||||
constructor(private readonly initContext: PluginInitializerContext) {}
|
||||
|
||||
private readonly sampleDatasets: SampleDatasetSchema[] = [];
|
||||
private readonly appLinksMap = new Map<string, AppLinkData[]>();
|
||||
|
||||
|
|
@ -68,8 +69,9 @@ export class SampleDataRegistry {
|
|||
isDevMode?: boolean
|
||||
) {
|
||||
if (usageCollections) {
|
||||
const kibanaIndex = core.savedObjects.getKibanaIndex();
|
||||
makeSampleDataUsageCollector(usageCollections, kibanaIndex);
|
||||
const getIndexForType = (type: string) =>
|
||||
core.getStartServices().then(([coreStart]) => coreStart.savedObjects.getIndexForType(type));
|
||||
makeSampleDataUsageCollector(usageCollections, getIndexForType);
|
||||
}
|
||||
const usageTracker = usage(
|
||||
core.getStartServices().then(([coreStart]) => coreStart.savedObjects),
|
||||
|
|
@ -176,6 +178,7 @@ export class SampleDataRegistry {
|
|||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export type SampleDataRegistrySetup = ReturnType<SampleDataRegistry['setup']>;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ import { fetchProvider, TelemetryResponse } from './collector_fetch';
|
|||
|
||||
export function makeSampleDataUsageCollector(
|
||||
usageCollection: UsageCollectionSetup,
|
||||
kibanaIndex: string
|
||||
getIndexForType: (type: string) => Promise<string>
|
||||
) {
|
||||
const collector = usageCollection.makeUsageCollector<TelemetryResponse>({
|
||||
type: 'sample-data',
|
||||
fetch: fetchProvider(kibanaIndex),
|
||||
fetch: fetchProvider(getIndexForType),
|
||||
isReady: () => true,
|
||||
schema: {
|
||||
installed: { type: 'array', items: { type: 'keyword' } },
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ const getMockFetchClients = (hits?: unknown[]) => {
|
|||
describe('Sample Data Fetch', () => {
|
||||
let collectorFetchContext: CollectorFetchContext;
|
||||
|
||||
const getIndexForType = (index: string) => (type: string) => Promise.resolve(index);
|
||||
|
||||
test('uninitialized .kibana', async () => {
|
||||
const fetch = fetchProvider('index');
|
||||
const fetch = fetchProvider(getIndexForType('index'));
|
||||
collectorFetchContext = getMockFetchClients();
|
||||
const telemetry = await fetch(collectorFetchContext);
|
||||
|
||||
|
|
@ -28,7 +30,7 @@ describe('Sample Data Fetch', () => {
|
|||
});
|
||||
|
||||
test('installed data set', async () => {
|
||||
const fetch = fetchProvider('index');
|
||||
const fetch = fetchProvider(getIndexForType('index'));
|
||||
collectorFetchContext = getMockFetchClients([
|
||||
{
|
||||
_id: 'sample-data-telemetry:test1',
|
||||
|
|
@ -55,7 +57,7 @@ Object {
|
|||
});
|
||||
|
||||
test('multiple installed data sets', async () => {
|
||||
const fetch = fetchProvider('index');
|
||||
const fetch = fetchProvider(getIndexForType('index'));
|
||||
collectorFetchContext = getMockFetchClients([
|
||||
{
|
||||
_id: 'sample-data-telemetry:test1',
|
||||
|
|
@ -90,7 +92,7 @@ Object {
|
|||
});
|
||||
|
||||
test('installed data set, missing counts', async () => {
|
||||
const fetch = fetchProvider('index');
|
||||
const fetch = fetchProvider(getIndexForType('index'));
|
||||
collectorFetchContext = getMockFetchClients([
|
||||
{
|
||||
_id: 'sample-data-telemetry:test1',
|
||||
|
|
@ -112,7 +114,7 @@ Object {
|
|||
});
|
||||
|
||||
test('installed and uninstalled data sets', async () => {
|
||||
const fetch = fetchProvider('index');
|
||||
const fetch = fetchProvider(getIndexForType('index'));
|
||||
collectorFetchContext = getMockFetchClients([
|
||||
{
|
||||
_id: 'sample-data-telemetry:test0',
|
||||
|
|
|
|||
|
|
@ -33,8 +33,9 @@ export interface TelemetryResponse {
|
|||
|
||||
type ESResponse = SearchResponse<SearchHit>;
|
||||
|
||||
export function fetchProvider(index: string) {
|
||||
export function fetchProvider(getIndexForType: (type: string) => Promise<string>) {
|
||||
return async ({ esClient }: CollectorFetchContext) => {
|
||||
const index = await getIndexForType('sample-data-telemetry');
|
||||
const response = await esClient.search<ESResponse>(
|
||||
{
|
||||
index,
|
||||
|
|
|
|||
|
|
@ -27,8 +27,9 @@ describe('kibana_usage', () => {
|
|||
});
|
||||
|
||||
const kibanaIndex = '.kibana-tests';
|
||||
const getIndicesForTypes = () => Promise.resolve([kibanaIndex]);
|
||||
|
||||
beforeAll(() => registerKibanaUsageCollector(usageCollectionMock, kibanaIndex));
|
||||
beforeAll(() => registerKibanaUsageCollector(usageCollectionMock, getIndicesForTypes));
|
||||
afterAll(() => jest.clearAllTimers());
|
||||
|
||||
afterEach(() => getSavedObjectsCountsMock.mockReset());
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ export async function getKibanaSavedObjectCounts(
|
|||
|
||||
export function registerKibanaUsageCollector(
|
||||
usageCollection: UsageCollectionSetup,
|
||||
kibanaIndex: string
|
||||
getIndicesForTypes: (types: string[]) => Promise<string[]>
|
||||
) {
|
||||
usageCollection.registerCollector(
|
||||
usageCollection.makeUsageCollector<KibanaUsage>({
|
||||
|
|
@ -80,8 +80,9 @@ export function registerKibanaUsageCollector(
|
|||
},
|
||||
},
|
||||
async fetch({ soClient }) {
|
||||
const indices = await getIndicesForTypes(['dashboard', 'visualization', 'search']);
|
||||
return {
|
||||
index: kibanaIndex,
|
||||
index: indices[0],
|
||||
...(await getKibanaSavedObjectCounts(soClient)),
|
||||
};
|
||||
},
|
||||
|
|
|
|||
|
|
@ -123,7 +123,6 @@ export class KibanaUsageCollectionPlugin implements Plugin {
|
|||
pluginStop$: Subject<void>,
|
||||
registerType: SavedObjectsRegisterType
|
||||
) {
|
||||
const kibanaIndex = coreSetup.savedObjects.getKibanaIndex();
|
||||
const getSavedObjectsClient = () => this.savedObjectsClient;
|
||||
const getUiSettingsClient = () => this.uiSettingsClient;
|
||||
const getCoreUsageDataService = () => this.coreUsageData!;
|
||||
|
|
@ -138,7 +137,12 @@ export class KibanaUsageCollectionPlugin implements Plugin {
|
|||
registerUsageCountersUsageCollector(usageCollection);
|
||||
|
||||
registerOpsStatsCollector(usageCollection, metric$);
|
||||
registerKibanaUsageCollector(usageCollection, kibanaIndex);
|
||||
|
||||
const getIndicesForTypes = (types: string[]) =>
|
||||
coreSetup
|
||||
.getStartServices()
|
||||
.then(([coreStart]) => coreStart.savedObjects.getIndicesForTypes(types));
|
||||
registerKibanaUsageCollector(usageCollection, getIndicesForTypes);
|
||||
|
||||
const coreStartPromise = coreSetup.getStartServices().then(([coreStart]) => coreStart);
|
||||
const getAllSavedObjectTypes = async () => {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
*/
|
||||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common';
|
||||
import { VIEW_MODE } from '../../common';
|
||||
|
|
@ -17,6 +18,7 @@ export function getSavedSearchObjectType(
|
|||
): SavedObjectsType {
|
||||
return {
|
||||
name: 'search',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
hidden: false,
|
||||
namespaceType: 'multiple-isolated',
|
||||
convertToMultiNamespaceTypeVersion: '8.0.0',
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
"@kbn/saved-objects-tagging-oss-plugin",
|
||||
"@kbn/i18n",
|
||||
"@kbn/config-schema",
|
||||
"@kbn/core-saved-objects-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ export class UsageCollectionPlugin implements Plugin<UsageCollectionSetup> {
|
|||
|
||||
public setup(core: CoreSetup): UsageCollectionSetup {
|
||||
const config = this.initializerContext.config.get<ConfigType>();
|
||||
const kibanaIndex = core.savedObjects.getKibanaIndex();
|
||||
const kibanaIndex = core.savedObjects.getDefaultIndex();
|
||||
|
||||
const collectorSet = new CollectorSet({
|
||||
logger: this.logger.get('usage-collection', 'collector-set'),
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { SavedObjectsType } from '@kbn/core/server';
|
||||
import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common';
|
||||
import { getAllMigrations } from '../migrations/visualization_saved_object_migrations';
|
||||
|
|
@ -14,6 +15,7 @@ export const getVisualizationSavedObjectType = (
|
|||
getSearchSourceMigrations: () => MigrateFunctionsObject
|
||||
): SavedObjectsType => ({
|
||||
name: 'visualization',
|
||||
indexPattern: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
hidden: false,
|
||||
namespaceType: 'multiple-isolated',
|
||||
convertToMultiNamespaceTypeVersion: '8.0.0',
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@
|
|||
"@kbn/shared-ux-router",
|
||||
"@kbn/saved-objects-management-plugin",
|
||||
"@kbn/saved-objects-finder-plugin",
|
||||
"@kbn/core-saved-objects-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
import { get } from 'lodash';
|
||||
import { ANALYTICS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService }: FtrProviderContext) {
|
||||
|
|
@ -27,7 +28,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
);
|
||||
});
|
||||
|
||||
it('should increment the opt *in* counter in the .kibana/kql-telemetry document', async () => {
|
||||
it('should increment the opt *in* counter in the .kibana_analytics/kql-telemetry document', async () => {
|
||||
await supertest
|
||||
.post('/api/kibana/kql_opt_in_stats')
|
||||
.set('content-type', 'application/json')
|
||||
|
|
@ -36,7 +37,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
|
||||
return es
|
||||
.search({
|
||||
index: '.kibana',
|
||||
index: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
q: 'type:kql-telemetry',
|
||||
})
|
||||
.then((response) => {
|
||||
|
|
@ -45,7 +46,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
});
|
||||
});
|
||||
|
||||
it('should increment the opt *out* counter in the .kibana/kql-telemetry document', async () => {
|
||||
it('should increment the opt *out* counter in the .kibana_analytics/kql-telemetry document', async () => {
|
||||
await supertest
|
||||
.post('/api/kibana/kql_opt_in_stats')
|
||||
.set('content-type', 'application/json')
|
||||
|
|
@ -54,7 +55,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
|
||||
return es
|
||||
.search({
|
||||
index: '.kibana',
|
||||
index: ANALYTICS_SAVED_OBJECT_INDEX,
|
||||
q: 'type:kql-telemetry',
|
||||
})
|
||||
.then((response) => {
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue