[8.8] [esArchiver] Automatically cleanup SO indices when SO documents are found in data.json (#159582) (#159910)

# Backport This will backport the following commits from `main` to `8.8`: - [[esArchiver] Automatically cleanup SO indices when SO documents are found in data.json (#159582)](https://github.com/elastic/kibana/pull/159582)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Gerard Soldevila <gerard.soldevila@elastic.co>
2025-04-23 17:28:26 -04:00 · 2023-06-19 08:50:56 -04:00 · 2023-06-19 08:50:56 -04:00 · d2683cba7b
commit d2683cba7b
parent 9e48a57755
6 changed files with 133 additions and 27 deletions
--- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.mock.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.mock.ts
@ -6,12 +6,17 @@
 * Side Public License, v 1.
 */

-import type { deleteSavedObjectIndices } from './kibana_index';
+import type { cleanSavedObjectIndices, deleteSavedObjectIndices } from './kibana_index';

-export const mockdeleteSavedObjectIndices = jest.fn() as jest.MockedFunction<
+export const mockCleanSavedObjectIndices = jest.fn() as jest.MockedFunction<
+  typeof cleanSavedObjectIndices
+>;
+
+export const mockDeleteSavedObjectIndices = jest.fn() as jest.MockedFunction<
  typeof deleteSavedObjectIndices
 >;

 jest.mock('./kibana_index', () => ({
-  deleteSavedObjectIndices: mockdeleteSavedObjectIndices,
+  cleanSavedObjectIndices: mockCleanSavedObjectIndices,
+  deleteSavedObjectIndices: mockDeleteSavedObjectIndices,
 }));
--- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.ts
@ -6,7 +6,10 @@
 * Side Public License, v 1.
 */

-import { mockdeleteSavedObjectIndices } from './create_index_stream.test.mock';
+import {
+  mockCleanSavedObjectIndices,
+  mockDeleteSavedObjectIndices,
+} from './create_index_stream.test.mock';

 import sinon from 'sinon';
 import Chance from 'chance';
@ -28,7 +31,8 @@ const chance = new Chance();
 const log = createStubLogger();

 beforeEach(() => {
-  mockdeleteSavedObjectIndices.mockClear();
+  mockCleanSavedObjectIndices.mockClear();
+  mockDeleteSavedObjectIndices.mockClear();
 });

 describe('esArchiver: createCreateIndexStream()', () => {
@ -199,25 +203,25 @@ describe('esArchiver: createCreateIndexStream()', () => {
    it('does not delete Kibana indices for indexes that do not start with .kibana', async () => {
      await doTest('.foo');

-      expect(mockdeleteSavedObjectIndices).not.toHaveBeenCalled();
+      expect(mockDeleteSavedObjectIndices).not.toHaveBeenCalled();
    });

    it('deletes Kibana indices at most once for indices that start with .kibana', async () => {
      // If we are loading the main Kibana index, we should delete all Kibana indices for backwards compatibility reasons.
      await doTest('.kibana_7.16.0_001', '.kibana_task_manager_7.16.0_001');

-      expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
-      expect(mockdeleteSavedObjectIndices).toHaveBeenCalledWith(
-        expect.not.objectContaining({ onlyTaskManager: true })
+      expect(mockDeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
+      expect(mockDeleteSavedObjectIndices).toHaveBeenCalledWith(
+        expect.not.objectContaining({ index: '.kibana_task_manager_7.16.0_001' })
      );
    });

-    it('deletes Kibana task manager index at most once, using onlyTaskManager: true', async () => {
+    it('deletes Kibana task manager index at most once', async () => {
      // If we are loading the Kibana task manager index, we should only delete that index, not any other Kibana indices.
      await doTest('.kibana_task_manager_7.16.0_001', '.kibana_task_manager_7.16.0_002');

-      expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
-      expect(mockdeleteSavedObjectIndices).toHaveBeenCalledWith(
+      expect(mockDeleteSavedObjectIndices).toHaveBeenCalledTimes(1);
+      expect(mockDeleteSavedObjectIndices).toHaveBeenCalledWith(
        expect.objectContaining({ onlyTaskManager: true })
      );
    });
@ -227,18 +231,63 @@ describe('esArchiver: createCreateIndexStream()', () => {
      // So, we first delete only the Kibana task manager indices, then we wind up deleting all Kibana indices.
      await doTest('.kibana_task_manager_7.16.0_001', '.kibana_7.16.0_001');

-      expect(mockdeleteSavedObjectIndices).toHaveBeenCalledTimes(2);
-      expect(mockdeleteSavedObjectIndices).toHaveBeenNthCalledWith(
+      expect(mockDeleteSavedObjectIndices).toHaveBeenCalledTimes(2);
+      expect(mockDeleteSavedObjectIndices).toHaveBeenNthCalledWith(
        1,
        expect.objectContaining({ onlyTaskManager: true })
      );
-      expect(mockdeleteSavedObjectIndices).toHaveBeenNthCalledWith(
+      expect(mockDeleteSavedObjectIndices).toHaveBeenNthCalledWith(
        2,
-        expect.not.objectContaining({ onlyTaskManager: true })
+        expect.not.objectContaining({ index: expect.any(String) })
      );
    });
  });

+  describe('saved object cleanup', () => {
+    describe('when saved object documents are found', () => {
+      it('cleans the corresponding saved object indices', async () => {
+        const client = createStubClient();
+        const stats = createStubStats();
+        await createPromiseFromStreams([
+          createListStream([
+            createStubDocRecord('.kibana_task_manager', 1),
+            createStubDocRecord('.kibana_alerting_cases', 2),
+            createStubDocRecord('.kibana', 3),
+          ]),
+          createCreateIndexStream({ client, stats, log }),
+        ]);
+
+        expect(mockCleanSavedObjectIndices).toHaveBeenCalledTimes(2);
+
+        expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
+          1,
+          expect.objectContaining({ index: '.kibana_task_manager' })
+        );
+        expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
+          2,
+          expect.not.objectContaining({ index: expect.any(String) })
+        );
+      });
+    });
+
+    describe('when saved object documents are not found', () => {
+      it('does not clean any indices', async () => {
+        const client = createStubClient();
+        const stats = createStubStats();
+        await createPromiseFromStreams([
+          createListStream([
+            createStubDocRecord('.foo', 1),
+            createStubDocRecord('.bar', 2),
+            createStubDocRecord('.baz', 3),
+          ]),
+          createCreateIndexStream({ client, stats, log }),
+        ]);
+
+        expect(mockCleanSavedObjectIndices).not.toHaveBeenCalled();
+      });
+    });
+  });
+
  describe('docsOnly = true', () => {
    it('passes through "hit" records without attempting to create indices', async () => {
      const client = createStubClient();
--- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts
@ -19,7 +19,7 @@ import {
  TASK_MANAGER_SAVED_OBJECT_INDEX,
 } from '@kbn/core-saved-objects-server';
 import { Stats } from '../stats';
-import { deleteSavedObjectIndices } from './kibana_index';
+import { cleanSavedObjectIndices, deleteSavedObjectIndices } from './kibana_index';
 import { deleteIndex } from './delete_index';
 import { deleteDataStream } from './delete_data_stream';
 import { ES_CLIENT_HEADERS } from '../../client_headers';
@ -50,14 +50,36 @@ export function createCreateIndexStream({
  // If we're trying to import Kibana index docs, we need to ensure that
  // previous indices are removed so we're starting w/ a clean slate for
  // migrations. This only needs to be done once per archive load operation.
-  let kibanaIndexAlreadyDeleted = false;
+  let kibanaIndicesAlreadyDeleted = false;
  let kibanaTaskManagerIndexAlreadyDeleted = false;

+  // if we detect saved object documents defined in the data.json, we will cleanup their indices
+  let kibanaIndicesAlreadyCleaned = false;
+  let kibanaTaskManagerIndexAlreadyCleaned = false;
+
  async function handleDoc(stream: Readable, record: DocRecord) {
-    if (skipDocsFromIndices.has(record.value.index)) {
+    const index = record.value.index;
+
+    if (skipDocsFromIndices.has(index)) {
      return;
    }

+    if (!skipExisting) {
+      if (index?.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX)) {
+        if (!kibanaTaskManagerIndexAlreadyDeleted && !kibanaTaskManagerIndexAlreadyCleaned) {
+          await cleanSavedObjectIndices({ client, stats, log, index });
+          kibanaTaskManagerIndexAlreadyCleaned = true;
+          log.debug(`Cleaned saved object index [${index}]`);
+        }
+      } else if (index?.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
+        if (!kibanaIndicesAlreadyDeleted && !kibanaIndicesAlreadyCleaned) {
+          await cleanSavedObjectIndices({ client, stats, log });
+          kibanaIndicesAlreadyCleaned = kibanaTaskManagerIndexAlreadyCleaned = true;
+          log.debug(`Cleaned all saved object indices`);
+        }
+      }
+    }
+
    stream.push(record);
  }

@ -109,12 +131,14 @@ export function createCreateIndexStream({

    async function attemptToCreate(attemptNumber = 1) {
      try {
-        if (isKibana && !kibanaIndexAlreadyDeleted) {
+        if (isKibana && !kibanaIndicesAlreadyDeleted) {
          await deleteSavedObjectIndices({ client, stats, log }); // delete all .kibana* indices
-          kibanaIndexAlreadyDeleted = kibanaTaskManagerIndexAlreadyDeleted = true;
+          kibanaIndicesAlreadyDeleted = kibanaTaskManagerIndexAlreadyDeleted = true;
+          log.debug(`Deleted all saved object indices`);
        } else if (isKibanaTaskManager && !kibanaTaskManagerIndexAlreadyDeleted) {
          await deleteSavedObjectIndices({ client, stats, onlyTaskManager: true, log }); // delete only .kibana_task_manager* indices
          kibanaTaskManagerIndexAlreadyDeleted = true;
+          log.debug(`Deleted saved object index [${index}]`);
        }

        await client.indices.create(
@ -137,7 +161,11 @@ export function createCreateIndexStream({
          err?.body?.error?.reason?.includes('index exists with the same name as the alias') &&
          attemptNumber < 3
        ) {
-          kibanaIndexAlreadyDeleted = false;
+          kibanaTaskManagerIndexAlreadyDeleted = false;
+          if (isKibana) {
+            kibanaIndicesAlreadyDeleted = false;
+          }
+
          const aliasStr = inspect(aliases);
          log.info(
            `failed to create aliases [${aliasStr}] because ES indicated an index/alias already exists, trying again`
--- a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.test.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.test.ts
@ -6,6 +6,8 @@
 * Side Public License, v 1.
 */

+import { mockCleanSavedObjectIndices } from './create_index_stream.test.mock';
+
 import sinon from 'sinon';

 import { createListStream, createPromiseFromStreams } from '@kbn/utils';
@ -22,6 +24,10 @@ import {

 const log = createStubLogger();

+beforeEach(() => {
+  mockCleanSavedObjectIndices.mockClear();
+});
+
 describe('esArchiver: createDeleteIndexStream()', () => {
  it('deletes the index without checking if it exists', async () => {
    const stats = createStubStats();
--- a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts
@ -10,13 +10,20 @@ import { Transform } from 'stream';
 import type { Client } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/tooling-log';

-import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
+import {
+  MAIN_SAVED_OBJECT_INDEX,
+  TASK_MANAGER_SAVED_OBJECT_INDEX,
+} from '@kbn/core-saved-objects-server';
 import { Stats } from '../stats';
 import { deleteIndex } from './delete_index';
 import { cleanSavedObjectIndices } from './kibana_index';
 import { deleteDataStream } from './delete_data_stream';

 export function createDeleteIndexStream(client: Client, stats: Stats, log: ToolingLog) {
+  // if we detect saved object documents defined in the data.json, we will cleanup their indices
+  let kibanaIndicesAlreadyCleaned = false;
+  let kibanaTaskManagerIndexAlreadyCleaned = false;
+
  return new Transform({
    readableObjectMode: true,
    writableObjectMode: true,
@ -29,8 +36,18 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli
        if (record.type === 'index') {
          const { index } = record.value;

-          if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
-            await cleanSavedObjectIndices({ client, stats, log });
+          if (index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX)) {
+            if (!kibanaTaskManagerIndexAlreadyCleaned) {
+              await cleanSavedObjectIndices({ client, stats, index, log });
+              kibanaTaskManagerIndexAlreadyCleaned = true;
+              log.debug(`Cleaned saved object index [${index}]`);
+            }
+          } else if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
+            if (!kibanaIndicesAlreadyCleaned) {
+              await cleanSavedObjectIndices({ client, stats, log });
+              kibanaIndicesAlreadyCleaned = kibanaTaskManagerIndexAlreadyCleaned = true;
+              log.debug(`Cleaned all saved object indices`);
+            }
          } else {
            await deleteIndex({ client, stats, log, index });
          }
--- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts
+++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts
@ -50,7 +50,6 @@ export async function deleteSavedObjectIndices({
      headers: ES_CLIENT_HEADERS,
    }
  );
-
  await deleteIndex({
    client,
    stats,
@ -111,15 +110,17 @@ export async function cleanSavedObjectIndices({
  client,
  stats,
  log,
+  index = ALL_SAVED_OBJECT_INDICES,
 }: {
  client: Client;
  stats: Stats;
  log: ToolingLog;
+  index?: string | string[];
 }) {
  while (true) {
    const resp = await client.deleteByQuery(
      {
-        index: ALL_SAVED_OBJECT_INDICES,
+        index,
        body: {
          query: {
            bool: {