Bootstrap ZDT migration algorithm (#151282)

## Summary

Part of https://github.com/elastic/kibana/issues/150309

Purpose of the PR is to create the skeleton of the ZDT algorithm, in
order to make sure we're all aligned on the way we'll be managing our
codebase between the 2 implementation (and to ease with the review of
the follow-up PRs by not having the bootstrap of the algo to review at
the same time)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts

@@ -11,6 +11,9 @@ import { schema, TypeOf } from '@kbn/config-schema';
 import type { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
 
 const migrationSchema = schema.object({
+  algorithm: schema.oneOf([schema.literal('v2'), schema.literal('zdt')], {
+    defaultValue: 'v2',
+  }),
   batchSize: schema.number({ defaultValue: 1_000 }),
   maxBatchSizeBytes: schema.byteSize({ defaultValue: '100mb' }), // 100mb is the default http.max_content_length Elasticsearch config value
   discardUnknownObjects: schema.maybe(

packages/core/saved-objects/core-saved-objects-migration-server-internal/index.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-export { DocumentMigrator, KibanaMigrator, buildActiveMappings, mergeTypes } from './src';
+export { DocumentMigrator, KibanaMigrator, buildActiveMappings, buildTypesMappings } from './src';
 export type { KibanaMigratorOptions } from './src';
 export { getAggregatedTypesDocuments } from './src/actions/check_for_unknown_docs';
 export {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { logActionResponse, logStateTransition, type LogAwareState } from './logs';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { Logger, LogMeta } from '@kbn/logging';
+import { MigrationLog } from '../../types';
+
+export interface LogAwareState {
+  controlState: string;
+  logs: MigrationLog[];
+}
+
+interface StateTransitionLogMeta extends LogMeta {
+  kibana: {
+    migrations: {
+      state: LogAwareState;
+      duration: number;
+    };
+  };
+}
+
+export const logStateTransition = (
+  logger: Logger,
+  logMessagePrefix: string,
+  prevState: LogAwareState,
+  currState: LogAwareState,
+  tookMs: number
+) => {
+  if (currState.logs.length > prevState.logs.length) {
+    currState.logs.slice(prevState.logs.length).forEach(({ message, level }) => {
+      switch (level) {
+        case 'error':
+          return logger.error(logMessagePrefix + message);
+        case 'warning':
+          return logger.warn(logMessagePrefix + message);
+        case 'info':
+          return logger.info(logMessagePrefix + message);
+        default:
+          throw new Error(`unexpected log level ${level}`);
+      }
+    });
+  }
+
+  logger.info(
+    logMessagePrefix + `${prevState.controlState} -> ${currState.controlState}. took: ${tookMs}ms.`
+  );
+  logger.debug<StateTransitionLogMeta>(
+    logMessagePrefix + `${prevState.controlState} -> ${currState.controlState}. took: ${tookMs}ms.`,
+    {
+      kibana: {
+        migrations: {
+          state: currState,
+          duration: tookMs,
+        },
+      },
+    }
+  );
+};
+
+export const logActionResponse = (
+  logger: Logger,
+  logMessagePrefix: string,
+  state: LogAwareState,
+  res: unknown
+) => {
+  logger.debug(logMessagePrefix + `${state.controlState} RESPONSE`, res as LogMeta);
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_types_mappings.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { SavedObjectsType } from '@kbn/core-saved-objects-server';
+import type { SavedObjectsTypeMappingDefinitions } from '@kbn/core-saved-objects-base-server-internal';
+
+/**
+ * Merge mappings from all registered saved object types.
+ */
+export const buildTypesMappings = (
+  types: SavedObjectsType[]
+): SavedObjectsTypeMappingDefinitions => {
+  return types.reduce((acc, { name: type, mappings }) => {
+    const duplicate = acc.hasOwnProperty(type);
+    if (duplicate) {
+      throw new Error(`Type ${type} is already defined.`);
+    }
+    return {
+      ...acc,
+      [type]: mappings,
+    };
+  }, {});
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts

@@ -11,6 +11,8 @@ export type { LogFn } from './migration_logger';
 export { excludeUnusedTypesQuery, REMOVED_TYPES } from './unused_types';
 export { TransformSavedObjectDocumentError } from './transform_saved_object_document_error';
 export { deterministicallyRegenerateObjectId } from './regenerate_object_id';
+export { buildTypesMappings } from './build_types_mappings';
+export { createIndexMap, type IndexMap, type CreateIndexMapOptions } from './build_index_map';
 export type {
   DocumentsTransformFailed,
   DocumentsTransformSuccess,

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/index.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-export { KibanaMigrator, mergeTypes } from './kibana_migrator';
+export { KibanaMigrator } from './kibana_migrator';
 export type { KibanaMigratorOptions } from './kibana_migrator';
-export { buildActiveMappings } from './core';
+export { buildActiveMappings, buildTypesMappings } from './core';
 export { DocumentMigrator } from './document_migrator';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts

@@ -281,6 +281,7 @@ const mockOptions = () => {
     ]),
     kibanaIndex: '.my-index',
     soMigrationsConfig: {
+      algorithm: 'v2',
       batchSize: 20,
       maxBatchSizeBytes: ByteSizeValue.parse('20mb'),
       pollInterval: 20000,

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts

@@ -16,7 +16,6 @@ import Semver from 'semver';
 import type { Logger } from '@kbn/logging';
 import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
-import type { SavedObjectsType } from '@kbn/core-saved-objects-server';
 import type {
   SavedObjectUnsanitizedDoc,
   SavedObjectsRawDoc,
@@ -31,11 +30,12 @@ import {
   type KibanaMigratorStatus,
   type MigrationResult,
 } from '@kbn/core-saved-objects-base-server-internal';
-import { buildActiveMappings } from './core';
+import { buildActiveMappings, buildTypesMappings } from './core';
 import { DocumentMigrator, type VersionedTransformer } from './document_migrator';
 import { createIndexMap } from './core/build_index_map';
 import { runResilientMigrator } from './run_resilient_migrator';
 import { migrateRawDocsSafely } from './core/migrate_raw_docs';
+import { runZeroDowntimeMigration } from './zdt';
 
 // ensure plugins don't try to convert SO namespaceTypes after 8.0.0
 // see https://github.com/elastic/kibana/issues/147344
@@ -91,7 +91,7 @@ export class KibanaMigrator implements IKibanaMigrator {
     this.soMigrationsConfig = soMigrationsConfig;
     this.typeRegistry = typeRegistry;
     this.serializer = new SavedObjectsSerializer(this.typeRegistry);
-    this.mappingProperties = mergeTypes(this.typeRegistry.getAllTypes());
+    this.mappingProperties = buildTypesMappings(this.typeRegistry.getAllTypes());
     this.log = logger;
     this.kibanaVersion = kibanaVersion;
     this.documentMigrator = new DocumentMigrator({
@@ -135,6 +135,28 @@ export class KibanaMigrator implements IKibanaMigrator {
   }
 
   private runMigrationsInternal(): Promise<MigrationResult[]> {
+    const migrationAlgorithm = this.soMigrationsConfig.algorithm;
+    if (migrationAlgorithm === 'zdt') {
+      return this.runMigrationZdt();
+    } else {
+      return this.runMigrationV2();
+    }
+  }
+
+  private runMigrationZdt(): Promise<MigrationResult[]> {
+    return runZeroDowntimeMigration({
+      kibanaIndexPrefix: this.kibanaIndex,
+      typeRegistry: this.typeRegistry,
+      logger: this.log,
+      documentMigrator: this.documentMigrator,
+      migrationConfig: this.soMigrationsConfig,
+      docLinks: this.docLinks,
+      serializer: this.serializer,
+      elasticsearchClient: this.client,
+    });
+  }
+
+  private runMigrationV2(): Promise<MigrationResult[]> {
     const indexMap = createIndexMap({
       kibanaIndexName: this.kibanaIndex,
       indexMap: this.mappingProperties,
@@ -187,20 +209,3 @@ export class KibanaMigrator implements IKibanaMigrator {
     return this.documentMigrator.migrate(doc);
   }
 }
-
-/**
- * Merges savedObjectMappings properties into a single object, verifying that
- * no mappings are redefined.
- */
-export function mergeTypes(types: SavedObjectsType[]): SavedObjectsTypeMappingDefinitions {
-  return types.reduce((acc, { name: type, mappings }) => {
-    const duplicate = acc.hasOwnProperty(type);
-    if (duplicate) {
-      throw new Error(`Type ${type} is already defined.`);
-    }
-    return {
-      ...acc,
-      [type]: mappings,
-    };
-  }, {});
-}

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts

@@ -44,6 +44,7 @@ describe('migrationsStateActionMachine', () => {
     migrationVersionPerType: {},
     indexPrefix: '.my-so-index',
     migrationsConfig: {
+      algorithm: 'v2',
       batchSize: 1000,
       maxBatchSizeBytes: new ByteSizeValue(1e8),
       pollInterval: 0,

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts

@@ -8,7 +8,7 @@
 
 import { errors as EsErrors } from '@elastic/elasticsearch';
 import * as Option from 'fp-ts/lib/Option';
-import type { Logger, LogMeta } from '@kbn/logging';
+import type { Logger } from '@kbn/logging';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import {
   getErrorMessage,
@@ -16,67 +16,12 @@ import {
 } from '@kbn/core-elasticsearch-client-server-internal';
 import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server';
 import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types';
+import { logActionResponse, logStateTransition } from './common/utils/logs';
 import { type Model, type Next, stateActionMachine } from './state_action_machine';
 import { cleanup } from './migrations_state_machine_cleanup';
 import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state';
 import type { BulkOperation } from './model/create_batches';
 
-interface StateTransitionLogMeta extends LogMeta {
-  kibana: {
-    migrations: {
-      state: State;
-      duration: number;
-    };
-  };
-}
-
-const logStateTransition = (
-  logger: Logger,
-  logMessagePrefix: string,
-  prevState: State,
-  currState: State,
-  tookMs: number
-) => {
-  if (currState.logs.length > prevState.logs.length) {
-    currState.logs.slice(prevState.logs.length).forEach(({ message, level }) => {
-      switch (level) {
-        case 'error':
-          return logger.error(logMessagePrefix + message);
-        case 'warning':
-          return logger.warn(logMessagePrefix + message);
-        case 'info':
-          return logger.info(logMessagePrefix + message);
-        default:
-          throw new Error(`unexpected log level ${level}`);
-      }
-    });
-  }
-
-  logger.info(
-    logMessagePrefix + `${prevState.controlState} -> ${currState.controlState}. took: ${tookMs}ms.`
-  );
-  logger.debug<StateTransitionLogMeta>(
-    logMessagePrefix + `${prevState.controlState} -> ${currState.controlState}. took: ${tookMs}ms.`,
-    {
-      kibana: {
-        migrations: {
-          state: currState,
-          duration: tookMs,
-        },
-      },
-    }
-  );
-};
-
-const logActionResponse = (
-  logger: Logger,
-  logMessagePrefix: string,
-  state: State,
-  res: unknown
-) => {
-  logger.debug(logMessagePrefix + `${state.controlState} RESPONSE`, res as LogMeta);
-};
-
 /**
  * A specialized migrations-specific state-action machine that:
  *  - logs messages in state.logs

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_machine_cleanup.ts

@@ -8,10 +8,13 @@
 
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import * as Actions from './actions';
-import type { State } from './state';
 
-export async function cleanup(client: ElasticsearchClient, state?: State) {
-  if (!state) return;
+type CleanableState = { sourceIndexPitId: string } | {};
+
+export async function cleanup(client: ElasticsearchClient, state?: CleanableState) {
+  if (!state) {
+    return;
+  }
   if ('sourceIndexPitId' in state) {
     await Actions.closePit({ client, pitId: state.sourceIndexPitId })();
   }

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts

@@ -15,7 +15,6 @@ import type {
 import * as Either from 'fp-ts/lib/Either';
 import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server';
 import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
-import type { State } from '../state';
 import type { AliasAction, FetchIndexResponse } from '../actions';
 import type { BulkIndexOperationTuple } from './create_batches';
 
@@ -23,15 +22,15 @@ import type { BulkIndexOperationTuple } from './create_batches';
  * A helper function/type for ensuring that all control state's are handled.
  */
 export function throwBadControlState(p: never): never;
-export function throwBadControlState(controlState: any) {
+export function throwBadControlState(controlState: unknown) {
   throw new Error('Unexpected control state: ' + controlState);
 }
 
 /**
  * A helper function/type for ensuring that all response types are handled.
  */
-export function throwBadResponse(state: State, p: never): never;
-export function throwBadResponse(state: State, res: any): never {
+export function throwBadResponse(state: { controlState: string }, p: never): never;
+export function throwBadResponse(state: { controlState: string }, res: unknown): never {
   throw new Error(
     `${state.controlState} received unexpected action response: ` + JSON.stringify(res)
   );

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/retry_state.ts

@@ -6,9 +6,16 @@
  * Side Public License, v 1.
  */
 
-import { State } from '../state';
+import type { MigrationLog } from '../types';
 
-export const delayRetryState = <S extends State>(
+export interface RetryableState {
+  controlState: string;
+  retryCount: number;
+  retryDelay: number;
+  logs: MigrationLog[];
+}
+
+export const delayRetryState = <S extends RetryableState>(
   state: S,
   errorMessage: string,
   /** How many times to retry a step that fails */
@@ -39,7 +46,7 @@ export const delayRetryState = <S extends State>(
     };
   }
 };
-export const resetRetryState = <S extends State>(state: S): S => {
+export const resetRetryState = <S extends RetryableState>(state: S): S => {
   return {
     ...state,
     retryCount: 0,

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type {
+  IncompatibleClusterRoutingAllocation,
+  RetryableEsClientError,
+  WaitForTaskCompletionTimeout,
+  IndexNotFound,
+} from '../../actions';
+
+export {
+  initAction as init,
+  type InitActionParams,
+  type IncompatibleClusterRoutingAllocation,
+  type RetryableEsClientError,
+  type WaitForTaskCompletionTimeout,
+  type IndexNotFound,
+} from '../../actions';
+
+export interface ActionErrorTypeMap {
+  wait_for_task_completion_timeout: WaitForTaskCompletionTimeout;
+  incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation;
+  retryable_es_client_error: RetryableEsClientError;
+  index_not_found_exception: IndexNotFound;
+}
+
+/** Type guard for narrowing the type of a left */
+export function isTypeof<T extends keyof ActionErrorTypeMap>(
+  res: any,
+  typeString: T
+): res is ActionErrorTypeMap[T] {
+  return res.type === typeString;
+}

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { MigratorContext } from './types';
+import type { MigrateIndexOptions } from '../migrate_index';
+
+export type CreateContextOps = Omit<MigrateIndexOptions, 'logger'>;
+
+/**
+ * Create the context object that will be used for this index migration.
+ */
+export const createContext = ({
+  types,
+  docLinks,
+  migrationConfig,
+  elasticsearchClient,
+  indexPrefix,
+  typeRegistry,
+  serializer,
+}: CreateContextOps): MigratorContext => {
+  return {
+    indexPrefix,
+    types,
+    elasticsearchClient,
+    typeRegistry,
+    serializer,
+    maxRetryAttempts: migrationConfig.retryAttempts,
+    migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects,
+  };
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type { MigratorContext } from './types';
+export { createContext, type CreateContextOps } from './create_context';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import type {
+  ISavedObjectTypeRegistry,
+  ISavedObjectsSerializer,
+} from '@kbn/core-saved-objects-server';
+import type { DocLinks } from '@kbn/doc-links';
+
+/**
+ * The set of static, precomputed values and services used by the ZDT migration
+ */
+export interface MigratorContext {
+  /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */
+  readonly indexPrefix: string;
+  /** Name of the types that are living in the index */
+  readonly types: string[];
+  /** The client to use for communications with ES */
+  readonly elasticsearchClient: ElasticsearchClient;
+  /** The maximum number of retries to attempt for a failing action */
+  readonly maxRetryAttempts: number;
+  /** DocLinks for savedObjects. to reference online documentation */
+  readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects'];
+  /** SO serializer to use for migration */
+  readonly serializer: ISavedObjectsSerializer;
+  /** The SO type registry to use for the migration */
+  readonly typeRegistry: ISavedObjectTypeRegistry;
+}

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { runZeroDowntimeMigration, type RunZeroDowntimeMigrationOpts } from './run_zdt_migration';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import {
+  type SavedObjectsMigrationConfigType,
+  type MigrationResult,
+} from '@kbn/core-saved-objects-base-server-internal';
+import type {
+  ISavedObjectTypeRegistry,
+  ISavedObjectsSerializer,
+} from '@kbn/core-saved-objects-server';
+import type { Logger } from '@kbn/logging';
+import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
+import { migrationStateActionMachine } from './migration_state_action_machine';
+import type { VersionedTransformer } from '../document_migrator';
+import { createContext } from './context';
+import { next } from './next';
+import { model } from './model';
+import { createInitialState } from './state';
+
+export interface MigrateIndexOptions {
+  indexPrefix: string;
+  types: string[];
+  /** The SO type registry to use for the migration */
+  typeRegistry: ISavedObjectTypeRegistry;
+  /** Logger to use for migration output */
+  logger: Logger;
+  /** The document migrator to use to convert the document */
+  documentMigrator: VersionedTransformer;
+  /** The migration config to use for the migration */
+  migrationConfig: SavedObjectsMigrationConfigType;
+  /** docLinks contract to use to link to documentation */
+  docLinks: DocLinksServiceStart;
+  /** SO serializer to use for migration */
+  serializer: ISavedObjectsSerializer;
+  /** The client to use for communications with ES */
+  elasticsearchClient: ElasticsearchClient;
+}
+
+export const migrateIndex = async ({
+  logger,
+  ...options
+}: MigrateIndexOptions): Promise<MigrationResult> => {
+  const context = createContext(options);
+  const initialState = createInitialState(context);
+
+  return migrationStateActionMachine({
+    initialState,
+    next: next(context),
+    model,
+    context,
+    logger,
+  });
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { errors as EsErrors } from '@elastic/elasticsearch';
+import type { Logger } from '@kbn/logging';
+import {
+  getErrorMessage,
+  getRequestDebugMeta,
+} from '@kbn/core-elasticsearch-client-server-internal';
+import { logStateTransition, logActionResponse } from '../common/utils';
+import { type Next, stateActionMachine } from '../state_action_machine';
+import { cleanup } from '../migrations_state_machine_cleanup';
+import type { State } from './state';
+import type { MigratorContext } from './context';
+
+/**
+ * A specialized migrations-specific state-action machine that:
+ *  - logs messages in state.logs
+ *  - logs state transitions
+ *  - logs action responses
+ *  - resolves if the final state is DONE
+ *  - rejects if the final state is FATAL
+ *  - catches and logs exceptions and then rejects with a migrations specific error
+ */
+export async function migrationStateActionMachine({
+  initialState,
+  context,
+  next,
+  model,
+  logger,
+}: {
+  initialState: State;
+  context: MigratorContext;
+  next: Next<State>;
+  model: (state: State, res: any, context: MigratorContext) => State;
+  logger: Logger;
+}) {
+  const startTime = Date.now();
+  // Since saved object index names usually start with a `.` and can be
+  // configured by users to include several `.`'s we can't use a logger tag to
+  // indicate which messages come from which index upgrade.
+  const logMessagePrefix = `[${context.indexPrefix}] `;
+  let prevTimestamp = startTime;
+  let lastState: State | undefined;
+  try {
+    const finalState = await stateActionMachine<State>(
+      initialState,
+      (state) => next(state),
+      (state, res) => {
+        lastState = state;
+        logActionResponse(logger, logMessagePrefix, state, res);
+        const newState = model(state, res, context);
+        // Redact the state to reduce the memory consumption and so that we
+        // don't log sensitive information inside documents by only keeping
+        // the _id's of documents
+        const redactedNewState = {
+          ...newState,
+          /* TODO: commented until we have model stages that process outdated docs. (attrs not on model atm)
+          ...{
+            outdatedDocuments: (
+              (newState as ReindexSourceToTempTransform).outdatedDocuments ?? []
+            ).map(
+              (doc) =>
+                ({
+                  _id: doc._id,
+                } as SavedObjectsRawDoc)
+            ),
+          },
+          ...{
+            transformedDocBatches: (
+              (newState as ReindexSourceToTempIndexBulk).transformedDocBatches ?? []
+            ).map((batches) => batches.map((doc) => ({ _id: doc._id }))) as [SavedObjectsRawDoc[]],
+          },
+          */
+        };
+
+        const now = Date.now();
+        logStateTransition(
+          logger,
+          logMessagePrefix,
+          state,
+          redactedNewState as State,
+          now - prevTimestamp
+        );
+        prevTimestamp = now;
+        return newState;
+      }
+    );
+
+    const elapsedMs = Date.now() - startTime;
+    if (finalState.controlState === 'DONE') {
+      logger.info(logMessagePrefix + `Migration completed after ${Math.round(elapsedMs)}ms`);
+      return {
+        status: 'patched' as const,
+        destIndex: context.indexPrefix,
+        elapsedMs,
+      };
+    } else if (finalState.controlState === 'FATAL') {
+      try {
+        await cleanup(context.elasticsearchClient, finalState);
+      } catch (e) {
+        logger.warn('Failed to cleanup after migrations:', e.message);
+      }
+      return Promise.reject(
+        new Error(
+          `Unable to complete saved object migrations for the [${context.indexPrefix}] index: ` +
+            finalState.reason
+        )
+      );
+    } else {
+      throw new Error('Invalid terminating control state');
+    }
+  } catch (e) {
+    try {
+      await cleanup(context.elasticsearchClient, lastState);
+    } catch (err) {
+      logger.warn('Failed to cleanup after migrations:', err.message);
+    }
+    if (e instanceof EsErrors.ResponseError) {
+      // Log the failed request. This is very similar to the
+      // elasticsearch-service's debug logs, but we log everything in single
+      // line until we have sub-ms resolution in our cloud logs. Because this
+      // is error level logs, we're also more careful and don't log the request
+      // body since this can very likely have sensitive saved objects.
+      const req = getRequestDebugMeta(e.meta);
+      const failedRequestMessage = `Unexpected Elasticsearch ResponseError: statusCode: ${
+        req.statusCode
+      }, method: ${req.method}, url: ${req.url} error: ${getErrorMessage(e)},`;
+      logger.error(logMessagePrefix + failedRequestMessage);
+      throw new Error(
+        `Unable to complete saved object migrations for the [${context.indexPrefix}] index. Please check the health of your Elasticsearch cluster and try again. ${failedRequestMessage}`
+      );
+    } else {
+      logger.error(e);
+
+      const newError = new Error(
+        `Unable to complete saved object migrations for the [${context.indexPrefix}] index. ${e}`
+      );
+
+      // restore error stack to point to a source of the problem.
+      newError.stack = `[${e.stack}]`;
+      throw newError;
+    }
+  }
+}

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { model } from './model';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const realStages = jest.requireActual('./stages');
+
+export const StageMocks = Object.keys(realStages).reduce((mocks, key) => {
+  mocks[key] = jest.fn().mockImplementation((state: unknown) => state);
+  return mocks;
+}, {} as Record<string, unknown>);
+
+jest.doMock('./stages', () => {
+  return StageMocks;
+});

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { StageMocks } from './model.test.mocks';
+import * as Either from 'fp-ts/lib/Either';
+import { createContextMock, MockedMigratorContext } from '../test_helpers';
+import type { RetryableEsClientError } from '../../actions';
+import type { State, BaseState, FatalState } from '../state';
+import type { StateActionResponse } from './types';
+import { model } from './model';
+
+describe('model', () => {
+  let context: MockedMigratorContext;
+
+  beforeEach(() => {
+    context = createContextMock();
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  const baseState: BaseState = {
+    controlState: '42',
+    retryCount: 0,
+    retryDelay: 0,
+    logs: [],
+  };
+
+  const retryableError: RetryableEsClientError = {
+    type: 'retryable_es_client_error',
+    message: 'snapshot_in_progress_exception',
+  };
+
+  describe('retry behavior', () => {
+    test('increments retryCount, exponential retryDelay if an action fails with a retryable_es_client_error', () => {
+      let state: State = {
+        ...baseState,
+        controlState: 'INIT',
+      };
+
+      const states = new Array(5).fill(1).map(() => {
+        state = model(state, Either.left(retryableError), context);
+        return state;
+      });
+      const retryState = states.map(({ retryCount, retryDelay }) => ({ retryCount, retryDelay }));
+      expect(retryState).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "retryCount": 1,
+            "retryDelay": 2000,
+          },
+          Object {
+            "retryCount": 2,
+            "retryDelay": 4000,
+          },
+          Object {
+            "retryCount": 3,
+            "retryDelay": 8000,
+          },
+          Object {
+            "retryCount": 4,
+            "retryDelay": 16000,
+          },
+          Object {
+            "retryCount": 5,
+            "retryDelay": 32000,
+          },
+        ]
+      `);
+    });
+
+    test('resets retryCount, retryDelay when an action succeeds', () => {
+      const state: State = {
+        ...baseState,
+        controlState: 'INIT',
+        retryCount: 5,
+        retryDelay: 32000,
+      };
+      const res: StateActionResponse<'INIT'> = Either.right({
+        '.kibana_7.11.0_001': {
+          aliases: {},
+          mappings: { properties: {} },
+          settings: {},
+        },
+      });
+      const newState = model(state, res, context);
+
+      expect(newState.retryCount).toEqual(0);
+      expect(newState.retryDelay).toEqual(0);
+    });
+
+    test('terminates to FATAL after retryAttempts retries', () => {
+      const state: State = {
+        ...baseState,
+        controlState: 'INIT',
+        retryCount: 15,
+        retryDelay: 64000,
+      };
+
+      const newState = model(state, Either.left(retryableError), context) as FatalState;
+
+      expect(newState.controlState).toEqual('FATAL');
+      expect(newState.reason).toMatchInlineSnapshot(
+        `"Unable to complete the INIT step after 15 attempts, terminating. The last failure message was: snapshot_in_progress_exception"`
+      );
+    });
+  });
+
+  describe('dispatching to correct stage', () => {
+    test('dispatching INIT state', () => {
+      const state: State = {
+        ...baseState,
+        controlState: 'INIT',
+      };
+      const res: StateActionResponse<'INIT'> = Either.right({
+        '.kibana_7.11.0_001': {
+          aliases: {},
+          mappings: { properties: {} },
+          settings: {},
+        },
+      });
+      model(state, res, context);
+
+      expect(StageMocks.init).toHaveBeenCalledTimes(1);
+      expect(StageMocks.init).toHaveBeenCalledWith(state, res, context);
+    });
+  });
+});

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Either from 'fp-ts/lib/Either';
+import type { State, AllActionStates } from '../state';
+import type { ResponseType } from '../next';
+import { delayRetryState, resetRetryState } from '../../model/retry_state';
+import { throwBadControlState } from '../../model/helpers';
+import { isTypeof } from '../actions';
+import { MigratorContext } from '../context';
+import * as Stages from './stages';
+import { StateActionResponse } from './types';
+
+export const model = (
+  current: State,
+  response: ResponseType<AllActionStates>,
+  context: MigratorContext
+): State => {
+  if (Either.isLeft<unknown, unknown>(response)) {
+    if (isTypeof(response.left, 'retryable_es_client_error')) {
+      return delayRetryState(current, response.left.message, context.maxRetryAttempts);
+    }
+  } else {
+    current = resetRetryState(current);
+  }
+
+  switch (current.controlState) {
+    case 'INIT':
+      return Stages.init(current, response as StateActionResponse<'INIT'>, context);
+    case 'DONE':
+    case 'FATAL':
+      // The state-action machine will never call the model in the terminating states
+      return throwBadControlState(current as never);
+    default:
+      return throwBadControlState(current);
+  }
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { init } from './init';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Either from 'fp-ts/lib/Either';
+import { createContextMock, MockedMigratorContext } from '../../test_helpers';
+import type { InitState } from '../../state';
+import type { StateActionResponse } from '../types';
+import { init } from './init';
+
+describe('Action: init', () => {
+  let context: MockedMigratorContext;
+
+  const createState = (parts: Partial<InitState> = {}): InitState => ({
+    controlState: 'INIT',
+    retryDelay: 0,
+    retryCount: 0,
+    logs: [],
+    ...parts,
+  });
+
+  beforeEach(() => {
+    context = createContextMock();
+  });
+
+  test('INIT -> DONE because its not implemented yet', () => {
+    const state = createState();
+    const res: StateActionResponse<'INIT'> = Either.right({
+      '.kibana_8.7.0_001': {
+        aliases: {
+          '.kibana': {},
+          '.kibana_8.7.0': {},
+        },
+        mappings: { properties: {} },
+        settings: {},
+      },
+    });
+
+    const newState = init(state, res, context);
+
+    expect(newState.controlState).toEqual('DONE');
+  });
+
+  test('INIT -> INIT when cluster routing allocation is incompatible', () => {
+    const state = createState();
+    const res: StateActionResponse<'INIT'> = Either.left({
+      type: 'incompatible_cluster_routing_allocation',
+    });
+
+    const newState = init(state, res, context);
+
+    expect(newState.controlState).toEqual('INIT');
+    expect(newState.retryCount).toEqual(1);
+    expect(newState.retryDelay).toEqual(2000);
+    expect(newState.logs).toHaveLength(1);
+  });
+});

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Either from 'fp-ts/lib/Either';
+import { delayRetryState } from '../../../model/retry_state';
+import { throwBadResponse } from '../../../model/helpers';
+import { isTypeof } from '../../actions';
+import type { State } from '../../state';
+import type { ModelStage } from '../types';
+
+export const init: ModelStage<'INIT', 'DONE' | 'FATAL'> = (state, res, context): State => {
+  if (Either.isLeft(res)) {
+    const left = res.left;
+    if (isTypeof(left, 'incompatible_cluster_routing_allocation')) {
+      const retryErrorMessage = `[${left.type}] Incompatible Elasticsearch cluster settings detected. Remove the persistent and transient Elasticsearch cluster setting 'cluster.routing.allocation.enable' or set it to a value of 'all' to allow migrations to proceed. Refer to ${context.migrationDocLinks.routingAllocationDisabled} for more information on how to resolve the issue.`;
+      return delayRetryState(state, retryErrorMessage, context.maxRetryAttempts);
+    } else {
+      return throwBadResponse(state, left);
+    }
+  }
+
+  // nothing implemented yet, just going to 'DONE'
+  return {
+    ...state,
+    controlState: 'DONE',
+  };
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/types.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ExcludeRetryableEsError } from '../../model/types';
+import type { MigratorContext } from '../context';
+import type {
+  AllActionStates,
+  AllControlStates,
+  StateFromActionState,
+  StateFromControlState,
+} from '../state';
+import type { ResponseType } from '../next';
+
+/**
+ * Utility type used to define the input of stage functions
+ */
+export type StateActionResponse<T extends AllActionStates> = ExcludeRetryableEsError<
+  ResponseType<T>
+>;
+
+/**
+ * Defines a stage delegation function for the model
+ */
+export type ModelStage<T extends AllActionStates, R extends AllControlStates> = (
+  state: StateFromActionState<T>,
+  res: StateActionResponse<T>,
+  context: MigratorContext
+) => StateFromControlState<T | R>;

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { AllActionStates, InitState, State } from './state';
+import type { MigratorContext } from './context';
+import * as Actions from './actions';
+
+export type ActionMap = ReturnType<typeof nextActionMap>;
+
+/**
+ * The response type of the provided control state's action.
+ *
+ * E.g. given 'INIT', provides the response type of the action triggered by
+ * `next` in the 'INIT' control state.
+ */
+export type ResponseType<ControlState extends AllActionStates> = Awaited<
+  ReturnType<ReturnType<ActionMap[ControlState]>>
+>;
+
+export const nextActionMap = (context: MigratorContext) => {
+  return {
+    INIT: (state: InitState) =>
+      Actions.init({ client: context.elasticsearchClient, indices: [context.indexPrefix] }),
+  };
+};
+
+export const next = (context: MigratorContext) => {
+  const map = nextActionMap(context);
+
+  return (state: State) => {
+    const delay = <F extends (...args: any) => any>(fn: F): (() => ReturnType<F>) => {
+      return () => {
+        return state.retryDelay > 0
+          ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn)
+          : fn();
+      };
+    };
+
+    if (state.controlState === 'DONE' || state.controlState === 'FATAL') {
+      // Return null if we're in one of the terminating states
+      return null;
+    } else {
+      // Otherwise return the delayed action
+      // We use an explicit cast as otherwise TS infers `(state: never) => ...`
+      // here because state is inferred to be the intersection of all states
+      // instead of the union.
+      const nextAction = map[state.controlState] as (
+        state: State
+      ) => ReturnType<typeof map[AllActionStates]>;
+      return delay(nextAction(state));
+    }
+  };
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { Logger } from '@kbn/logging';
+import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
+import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import type {
+  ISavedObjectTypeRegistry,
+  ISavedObjectsSerializer,
+} from '@kbn/core-saved-objects-server';
+import {
+  type SavedObjectsMigrationConfigType,
+  type MigrationResult,
+} from '@kbn/core-saved-objects-base-server-internal';
+import type { VersionedTransformer } from '../document_migrator';
+import { buildMigratorConfigs } from './utils';
+import { migrateIndex } from './migrate_index';
+
+export interface RunZeroDowntimeMigrationOpts {
+  /** The kibana system index prefix. e.g `.kibana` */
+  kibanaIndexPrefix: string;
+  /** The SO type registry to use for the migration */
+  typeRegistry: ISavedObjectTypeRegistry;
+  /** Logger to use for migration output */
+  logger: Logger;
+  /** The document migrator to use to convert the document */
+  documentMigrator: VersionedTransformer;
+  /** The migration config to use for the migration */
+  migrationConfig: SavedObjectsMigrationConfigType;
+  /** docLinks contract to use to link to documentation */
+  docLinks: DocLinksServiceStart;
+  /** SO serializer to use for migration */
+  serializer: ISavedObjectsSerializer;
+  /** The client to use for communications with ES */
+  elasticsearchClient: ElasticsearchClient;
+}
+
+export const runZeroDowntimeMigration = async (
+  options: RunZeroDowntimeMigrationOpts
+): Promise<MigrationResult[]> => {
+  const migratorConfigs = buildMigratorConfigs({
+    kibanaIndexPrefix: options.kibanaIndexPrefix,
+    typeRegistry: options.typeRegistry,
+  });
+
+  return await Promise.all(
+    migratorConfigs.map((migratorConfig) => {
+      return migrateIndex({
+        ...options,
+        ...migratorConfig,
+      });
+    })
+  );
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { InitState, State } from './types';
+import type { MigratorContext } from '../context';
+
+export const createInitialState = (context: MigratorContext): State => {
+  const initialState: InitState = {
+    controlState: 'INIT',
+    logs: [],
+    retryCount: 0,
+    retryDelay: 0,
+  };
+  return initialState;
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type {
+  BaseState,
+  InitState,
+  DoneState,
+  FatalState,
+  State,
+  AllActionStates,
+  AllControlStates,
+  StateFromActionState,
+  StateFromControlState,
+} from './types';
+export { createInitialState } from './create_initial_state';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { MigrationLog } from '../../types';
+import type { ControlState } from '../../state_action_machine';
+
+export interface BaseState extends ControlState {
+  readonly retryCount: number;
+  readonly retryDelay: number;
+  readonly logs: MigrationLog[];
+}
+
+export interface InitState extends BaseState {
+  readonly controlState: 'INIT';
+}
+
+/** Migration completed successfully */
+export interface DoneState extends BaseState {
+  readonly controlState: 'DONE';
+}
+
+/** Migration terminated with a failure */
+export interface FatalState extends BaseState {
+  readonly controlState: 'FATAL';
+  /** The reason the migration was terminated */
+  readonly reason: string;
+}
+
+export type State = InitState | DoneState | FatalState;
+
+export type AllControlStates = State['controlState'];
+
+export type AllActionStates = Exclude<AllControlStates, 'FATAL' | 'DONE'>;
+
+/**
+ * Manually maintained reverse-lookup map used by `StateFromAction`
+ */
+export interface ControlStateMap {
+  INIT: InitState;
+  FATAL: FatalState;
+  DONE: DoneState;
+}
+
+/**
+ * Utility type to reverse lookup an `AllControlStates` to it's corresponding State subtype.
+ */
+export type StateFromControlState<T extends AllControlStates> = ControlStateMap[T];
+
+/**
+ * Utility type to reverse lookup an `AllActionStates` to it's corresponding State subtype.
+ */
+export type StateFromActionState<T extends AllActionStates> = StateFromControlState<T>;

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  ElasticsearchClientMock,
+  elasticsearchClientMock,
+} from '@kbn/core-elasticsearch-client-server-mocks';
+import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-internal';
+import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks';
+import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
+import type { MigratorContext } from '../context';
+
+export type MockedMigratorContext = Omit<MigratorContext, 'elasticsearchClient'> & {
+  elasticsearchClient: ElasticsearchClientMock;
+};
+
+export const createContextMock = (
+  parts: Partial<MockedMigratorContext> = {}
+): MockedMigratorContext => {
+  const typeRegistry = new SavedObjectTypeRegistry();
+
+  return {
+    indexPrefix: '.kibana',
+    types: ['foo', 'bar'],
+    elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(),
+    maxRetryAttempts: 15,
+    migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects,
+    typeRegistry,
+    serializer: serializerMock.create(),
+    ...parts,
+  };
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { createContextMock, type MockedMigratorContext } from './context';

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_migrator_configs.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
+
+export interface MigratorConfig {
+  /** The index prefix for this migrator. e.g '.kibana' */
+  indexPrefix: string;
+  /** The id of the types this migrator is in charge of */
+  types: string[];
+}
+
+export const buildMigratorConfigs = ({
+  typeRegistry,
+  kibanaIndexPrefix,
+}: {
+  typeRegistry: ISavedObjectTypeRegistry;
+  kibanaIndexPrefix: string;
+}): MigratorConfig[] => {
+  const configMap = new Map<string, MigratorConfig>();
+  typeRegistry.getAllTypes().forEach((type) => {
+    const typeIndexPrefix = type.indexPattern ?? kibanaIndexPrefix;
+    if (!configMap.has(typeIndexPrefix)) {
+      configMap.set(typeIndexPrefix, {
+        indexPrefix: typeIndexPrefix,
+        types: [],
+      });
+    }
+    const migratorConfig = configMap.get(typeIndexPrefix)!;
+    migratorConfig.types.push(type.name);
+  });
+  return [...configMap.values()];
+};

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { buildMigratorConfigs, type MigratorConfig } from './get_migrator_configs';

packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json

@@ -30,6 +30,7 @@
     "@kbn/doc-links",
     "@kbn/safer-lodash-set",
     "@kbn/logging-mocks",
+    "@kbn/core-saved-objects-base-server-mocks",
   ],
   "exclude": [
     "target/**/*",

packages/core/saved-objects/core-saved-objects-migration-server-mocks/src/kibana_migrator.mock.ts

@@ -12,7 +12,10 @@ import type {
   IKibanaMigrator,
   KibanaMigratorStatus,
 } from '@kbn/core-saved-objects-base-server-internal';
-import { buildActiveMappings, mergeTypes } from '@kbn/core-saved-objects-migration-server-internal';
+import {
+  buildActiveMappings,
+  buildTypesMappings,
+} from '@kbn/core-saved-objects-migration-server-internal';
 
 const defaultSavedObjectTypes: SavedObjectsType[] = [
   {
@@ -57,7 +60,7 @@ const createMigrator = (
     ),
   };
 
-  mockMigrator.getActiveMappings.mockReturnValue(buildActiveMappings(mergeTypes(types)));
+  mockMigrator.getActiveMappings.mockReturnValue(buildActiveMappings(buildTypesMappings(types)));
   mockMigrator.migrateDocument.mockImplementation((doc) => doc);
   return mockMigrator;
 };