[7.17] Migrations: Don't auto-create temp index (#158182) (#188646)

# Backport This will backport the following commits from `main` to `7.17`: - [Migrations: Don't auto-create temp index (#158182)](https://github.com/elastic/kibana/pull/158182)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Rudolf Meijering <skaapgif@gmail.com>"}},{"branch":"8.16","label":"v8.16.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT-->
2025-04-24 01:38:56 -04:00 · 2024-07-22 15:28:18 +02:00 · 2024-07-22 15:28:18 +02:00 · 086906199c
commit 086906199c
parent 584b29a0c5
11 changed files with 67 additions and 11 deletions
--- a/src/core/server/saved_objects/migrationsv2/snapshots/migrations_state_action_machine.test.ts.snap
+++ b/src/core/server/saved_objects/migrationsv2/snapshots/migrations_state_action_machine.test.ts.snap
@ -44,6 +44,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
@ -181,6 +182,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
@ -322,6 +324,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
@ -467,6 +470,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
@ -643,6 +647,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
@ -791,6 +796,7 @@ Object {
                "properties": Object {},
              },
              "tempIndex": ".my-so-index_7.11.0_reindex_temp",
+              "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias",
              "tempIndexMappings": Object {
                "dynamic": false,
                "properties": Object {
--- a/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts
+++ b/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts
@ -50,6 +50,12 @@ export interface BulkOverwriteTransformedDocumentsParams {
  index: string;
  transformedDocs: SavedObjectsRawDoc[];
  refresh?: estypes.Refresh;
+  /**
+   * If true, we prevent Elasticsearch from auto-creating the index if it
+   * doesn't exist. We use the ES paramater require_alias: true so `index`
+   * must be an alias, otherwise the bulk index will fail.
+   */
+  useAliasToPreventAutoCreate?: boolean;
 }

 /**
@ -62,6 +68,7 @@ export const bulkOverwriteTransformedDocuments =
    index,
    transformedDocs,
    refresh = false,
+    useAliasToPreventAutoCreate = false,
  }: BulkOverwriteTransformedDocumentsParams): TaskEither.TaskEither<
    | RetryableEsClientError
    | TargetIndexHadWriteBlock
@ -83,7 +90,7 @@ export const bulkOverwriteTransformedDocuments =
        // mappings. Such tampering could lead to many other problems and is
        // probably unlikely so for now we'll accept this risk and wait till
        // system indices puts in place a hard control.
-        require_alias: false,
+        require_alias: useAliasToPreventAutoCreate,
        wait_for_active_shards: WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE,
        refresh,
        filter_path: ['items.*.error'],
--- a/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts
@ -70,6 +70,7 @@ describe('migration actions', () => {
    await createIndex({
      client,
      indexName: 'existing_index_with_docs',
+      aliases: ['existing_index_with_docs_alias'],
      mappings: {
        dynamic: true,
        properties: {},
@ -133,7 +134,9 @@ describe('migration actions', () => {
      expect(res.right).toEqual(
        expect.objectContaining({
          existing_index_with_docs: {
-            aliases: {},
+            aliases: {
+              existing_index_with_docs_alias: {},
+            },
            mappings: expect.anything(),
            settings: expect.anything(),
          },
@ -1519,6 +1522,30 @@ describe('migration actions', () => {
                }
              `);
    });
+    it('resolves left index_not_found_exception if the index does not exist and useAliasToPreventAutoCreate=true', async () => {
+      const newDocs = [
+        { _source: { title: 'doc 5' } },
+        { _source: { title: 'doc 6' } },
+        { _source: { title: 'doc 7' } },
+      ] as unknown as SavedObjectsRawDoc[];
+      await expect(
+        bulkOverwriteTransformedDocuments({
+          client,
+          index: 'existing_index_with_docs_alias_that_does_not_exist',
+          useAliasToPreventAutoCreate: true,
+          transformedDocs: newDocs,
+          refresh: 'wait_for',
+        })()
+      ).resolves.toMatchInlineSnapshot(`
+          Object {
+            "_tag": "Left",
+            "left": Object {
+              "index": "existing_index_with_docs_alias_that_does_not_exist",
+              "type": "index_not_found_exception",
+            },
+          }
+      `);
+    });
    it('resolves left target_index_had_write_block if there are write_block errors', async () => {
      const newDocs = [
        { _source: { title: 'doc 5' } },
--- a/src/core/server/saved_objects/migrationsv2/initial_state.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/initial_state.test.ts
@ -72,6 +72,7 @@ describe('createInitialState', () => {
        },
      },
      tempIndex: '.kibana_task_manager_8.1.0_reindex_temp',
+      tempIndexAlias: '.kibana_task_manager_8.1.0_reindex_temp_alias',
      tempIndexMappings: {
        dynamic: false,
        properties: {
--- a/src/core/server/saved_objects/migrationsv2/initial_state.ts
+++ b/src/core/server/saved_objects/migrationsv2/initial_state.ts
@ -73,6 +73,7 @@ export const createInitialState = ({
    versionAlias: `${indexPrefix}_${kibanaVersion}`,
    versionIndex: `${indexPrefix}_${kibanaVersion}_001`,
    tempIndex: `${indexPrefix}_${kibanaVersion}_reindex_temp`,
+    tempIndexAlias: `${indexPrefix}_${kibanaVersion}_reindex_temp_alias`,
    kibanaVersion,
    preMigrationScript: Option.fromNullable(preMigrationScript),
    targetIndexMappings: targetMappings,
--- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts
@ -43,7 +43,7 @@ describe('migration v2', function () {
        es: {
          license: 'basic',
          dataArchive: Path.join(__dirname, 'archives', '7.14.0_xpack_sample_saved_objects.zip'),
-          esArgs: ['http.max_content_length=1715277b'],
+          esArgs: ['http.max_content_length=1715329b'],
        },
      },
    }));
@ -61,7 +61,7 @@ describe('migration v2', function () {
  });

  it('completes the migration even when a full batch would exceed ES http.max_content_length', async () => {
-    root = createRoot({ maxBatchSizeBytes: 1715277 });
+    root = createRoot({ maxBatchSizeBytes: 1715329 });
    esServer = await startES();
    await root.preboot();
    await root.setup();
@ -90,7 +90,7 @@ describe('migration v2', function () {
    await root.preboot();
    await root.setup();
    await expect(root.start()).rejects.toMatchInlineSnapshot(
-      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
+      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715283 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
    );

    await retryAsync(
@ -103,7 +103,7 @@ describe('migration v2', function () {
        expect(
          records.find((rec) =>
            rec.message.startsWith(
-              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
+              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715283 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
            )
          )
        ).toBeDefined();
--- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts
@ -54,7 +54,7 @@ describe('migration v2', () => {
  });

  it('fails with a descriptive message when maxBatchSizeBytes exceeds ES http.max_content_length', async () => {
-    root = createRoot({ maxBatchSizeBytes: 1715277 });
+    root = createRoot({ maxBatchSizeBytes: 1715329 });
    esServer = await startES();
    await root.preboot();
    await root.setup();
--- a/src/core/server/saved_objects/migrationsv2/model/model.test.ts
+++ b/src/core/server/saved_objects/migrationsv2/model/model.test.ts
@ -81,6 +81,7 @@ describe('migrations v2 model', () => {
    versionAlias: '.kibana_7.11.0',
    versionIndex: '.kibana_7.11.0_001',
    tempIndex: '.kibana_7.11.0_reindex_temp',
+    tempIndexAlias: '.kibana_7.11.0_reindex_temp_alias',
    unusedTypesQuery: {
      bool: {
        must_not: [
--- a/src/core/server/saved_objects/migrationsv2/model/model.ts
+++ b/src/core/server/saved_objects/migrationsv2/model/model.ts
@ -504,7 +504,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
      if (stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) {
        const batches = createBatches(
          res.right.processedDocs,
-          stateP.tempIndex,
+          stateP.tempIndexAlias,
          stateP.maxBatchSizeBytes
        );
        if (Either.isRight(batches)) {
--- a/src/core/server/saved_objects/migrationsv2/next.ts
+++ b/src/core/server/saved_objects/migrationsv2/next.ts
@ -87,6 +87,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
      Actions.createIndex({
        client,
        indexName: state.tempIndex,
+        aliases: [state.tempIndexAlias],
        mappings: state.tempIndexMappings,
      }),
    REINDEX_SOURCE_TO_TEMP_OPEN_PIT: (state: ReindexSourceToTempOpenPit) =>
@ -110,8 +111,14 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
    REINDEX_SOURCE_TO_TEMP_INDEX_BULK: (state: ReindexSourceToTempIndexBulk) =>
      Actions.bulkOverwriteTransformedDocuments({
        client,
-        index: state.tempIndex,
+        index: state.tempIndexAlias,
        transformedDocs: state.transformedDocBatches[state.currentBatch],
+        /*
+         * Since other nodes can delete the temp index while we're busy writing
+         * to it, we use the alias to prevent the auto-creation of the index if
+         * it doesn't exist.
+         */
+        useAliasToPreventAutoCreate: true,
        /**
         * Since we don't run a search against the target index, we disable "refresh" to speed up
         * the migration process.
--- a/src/core/server/saved_objects/migrationsv2/types.ts
+++ b/src/core/server/saved_objects/migrationsv2/types.ts
@ -117,10 +117,16 @@ export interface BaseState extends ControlState {
   */
  readonly versionIndex: string;
  /**
-   * An alias on the target index used as part of an "reindex block" that
-   * prevents lost deletes e.g. `.kibana_7.11.0_reindex`.
+   * A temporary index used as part of an "reindex block" that
+   * prevents lost deletes e.g. `.kibana_7.11.0_reindex_temp`.
   */
  readonly tempIndex: string;
+  /**
+   * An alias to the tempIndex used to prevent ES from auto-creating the temp
+   * index if one node deletes it while another writes to it
+   * e.g. `.kibana_7.11.0_reindex_temp_alias`.
+   */
+  readonly tempIndexAlias: string;
  /**
   * When reindexing we use a source query to exclude saved objects types which
   * are no longer used. These saved objects will still be kept in the outdated