🌊 Fix MKI tests (#207397)

Closes https://github.com/elastic/kibana/issues/207310

The deployment agnostic tests were not running properly against MKI
because they directly mess with system indices.

This PR fixes this by removing these parts of the streams tests as they
are anyway tested already by the separate storage adapter tests.

It also extends the behavior of the "disable" streams API endpoint to
also wipe the asset links and stream definitions for classic streams to
leave a clean state. To do this, I extended the storage adapter by a
"clean" function, which deletes the index templates and all backing
indices.

x-pack/solutions/observability/packages/utils_server/es/storage/index.ts

@@ -67,6 +67,11 @@ export interface StorageClientDeleteResponse {
   result: Extract<Result, 'deleted' | 'not_found'>;
 }
 
+export interface StorageClientCleanResponse {
+  acknowledged: boolean;
+  result: Extract<Result, 'deleted' | 'noop'>;
+}
+
 export type StorageClientIndexRequest<TDocument = unknown> = Omit<
   IndexRequest<Omit<TDocument, '_id'>>,
   'index'
@@ -96,6 +101,8 @@ export type StorageClientDelete = (
   request: StorageClientDeleteRequest
 ) => Promise<StorageClientDeleteResponse>;
 
+export type StorageClientClean = () => Promise<StorageClientCleanResponse>;
+
 export type StorageClientGet<TStorageSettings extends StorageSettings = never> = (
   request: StorageClientGetRequest
 ) => Promise<StorageClientGetResponse<StorageDocumentOf<TStorageSettings>>>;
@@ -107,6 +114,7 @@ export interface IStorageClient<TStorageSettings extends StorageSettings = never
   bulk: StorageClientBulk<TStorageSettings>;
   index: StorageClientIndex<TStorageSettings>;
   delete: StorageClientDelete;
+  clean: StorageClientClean;
   get: StorageClientGet<TStorageSettings>;
   existsIndex: StorageClientExistsIndex;
 }

x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.ts

@@ -32,6 +32,8 @@ import {
   StorageClientExistsIndex,
   StorageDocumentOf,
   StorageClientSearchResponse,
+  StorageClientClean,
+  StorageClientCleanResponse,
 } from '..';
 import { getSchemaVersion } from '../get_schema_version';
 import { StorageMappingProperty } from '../types';
@@ -446,6 +448,36 @@ export class StorageIndexAdapter<TStorageSettings extends IndexStorageSettings>
     });
   };
 
+  private clean: StorageClientClean = async (): Promise<StorageClientCleanResponse> => {
+    const allIndices = await this.getExistingIndices();
+    const hasIndices = Object.keys(allIndices).length > 0;
+    // Delete all indices
+    await Promise.all(
+      Object.keys(allIndices).map((index) =>
+        wrapEsCall(
+          this.esClient.indices.delete({
+            index,
+          })
+        )
+      )
+    );
+    // Delete the index template
+    const template = await this.getExistingIndexTemplate();
+    const hasTemplate = !!template;
+    if (template) {
+      await wrapEsCall(
+        this.esClient.indices.deleteIndexTemplate({
+          name: getIndexTemplateName(this.storage.name),
+        })
+      );
+    }
+
+    return {
+      acknowledged: true,
+      result: hasIndices || hasTemplate ? 'deleted' : 'noop',
+    };
+  };
+
   private delete: StorageClientDelete = async ({
     id,
     refresh = 'wait_for',
@@ -546,6 +578,7 @@ export class StorageIndexAdapter<TStorageSettings extends IndexStorageSettings>
     return {
       bulk: this.bulk,
       delete: this.delete,
+      clean: this.clean,
       index: this.index,
       search: this.search,
       get: this.get,

x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/integration_tests/index.test.ts

@@ -178,6 +178,10 @@ describe('StorageIndexAdapter', () => {
       });
     });
 
+    it('deletes the document', async () => {
+      await verifyClean();
+    });
+
     // FLAKY: https://github.com/elastic/kibana/issues/206482
     // FLAKY: https://github.com/elastic/kibana/issues/206483
     describe.skip('after rolling over the index manually and indexing the same document', () => {
@@ -316,6 +320,10 @@ describe('StorageIndexAdapter', () => {
       it('deletes the document from the rolled over index', async () => {
         await verifyDocumentDeletedInRolledOverIndex();
       });
+
+      it('deletes the documents', async () => {
+        await verifyClean();
+      });
     });
   });
 
@@ -349,6 +357,10 @@ describe('StorageIndexAdapter', () => {
 
       expect(getIndicesResponse[writeIndexName].mappings?._meta?.version).toEqual('next_version');
     });
+
+    it('deletes the documents', async () => {
+      await verifyClean();
+    });
   });
 
   describe('when writing/bootstrapping with an existing, incompatible index', () => {
@@ -387,6 +399,10 @@ describe('StorageIndexAdapter', () => {
         		illegal_argument_exception: mapper [foo] cannot be changed from type [keyword] to [text]"
       `);
     });
+
+    it('deletes the documents', async () => {
+      await verifyClean();
+    });
   });
 
   function createStorageIndexAdapter<TStorageSettings extends StorageSettings>(
@@ -567,4 +583,28 @@ describe('StorageIndexAdapter', () => {
       },
     });
   }
+
+  async function verifyClean() {
+    await client.clean();
+
+    // verify that the index template is removed
+    const templates = await esClient.indices
+      .getIndexTemplate({
+        name: TEST_INDEX_NAME,
+      })
+      .catch((error) => {
+        if (isResponseError(error) && error.statusCode === 404) {
+          return { index_templates: [] };
+        }
+        throw error;
+      });
+
+    expect(templates.index_templates).toEqual([]);
+
+    // verify that the backing indices are removed
+    const indices = await esClient.indices.get({
+      index: `${TEST_INDEX_NAME}*`,
+    });
+    expect(Object.keys(indices)).toEqual([]);
+  }
 });

x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts

@@ -177,6 +177,10 @@ export class AssetClient {
     await this.clients.storageClient.delete({ id });
   }
 
+  async clean() {
+    await this.clients.storageClient.clean();
+  }
+
   async getAssetIds({
     entityId,
     entityType,

x-pack/solutions/observability/plugins/streams/server/lib/streams/client.ts

@@ -143,6 +143,9 @@ export class StreamsClient {
 
     await this.deleteStreamFromDefinition(definition);
 
+    const { assetClient, storageClient } = this.dependencies;
+    await Promise.all([assetClient.clean(), storageClient.clean()]);
+
     return { acknowledged: true, result: 'deleted' };
   }
 

x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/assets/dashboard.ts

@@ -96,21 +96,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     expect(response.status).to.be(200);
   }
 
-  async function deleteAssetIndices() {
-    const concreteIndices = await esClient.indices.resolveIndex({
-      name: '.kibana_streams_assets*',
-    });
-
-    if (concreteIndices.indices.length) {
-      await esClient.indices.delete({
-        index: concreteIndices.indices.map((index) => index.name),
-      });
-    }
-  }
-
   describe('Asset links', function () {
-    // see details: https://github.com/elastic/kibana/issues/207310
-    this.tags(['failsOnMKI']);
     before(async () => {
       apiClient = await createStreamsRepositoryAdminClient(roleScopedSupertest);
       await enableStreams(apiClient);
@@ -123,28 +109,6 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
 
     after(async () => {
       await disableStreams(apiClient);
-
-      await deleteAssetIndices();
-    });
-
-    describe('without writing', () => {
-      it('creates no indices initially', async () => {
-        const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' });
-
-        expect(exists).to.eql(false);
-      });
-
-      it('creates no indices after reading the assets', async () => {
-        const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', {
-          params: { path: { id: 'logs' } },
-        });
-
-        expect(response.status).to.be(200);
-
-        const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' });
-
-        expect(exists).to.eql(false);
-      });
     });
 
     describe('after linking a dashboard', () => {
@@ -159,12 +123,6 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
         await unlinkDashboard(SEARCH_DASHBOARD_ID);
       });
 
-      it('creates the index', async () => {
-        const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' });
-
-        expect(exists).to.be(true);
-      });
-
       it('lists the dashboard in the stream response', async () => {
         const response = await apiClient.fetch('GET /api/streams/{id}', {
           params: { path: { id: 'logs' } },
@@ -185,54 +143,27 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
         expect(response.body.dashboards.length).to.eql(1);
       });
 
-      describe('after manually rolling over the index and relinking the dashboard', () => {
+      describe('after disabling', () => {
         before(async () => {
-          await esClient.indices.updateAliases({
-            actions: [
-              {
-                add: {
-                  index: `.kibana_streams_assets-000001`,
-                  alias: `.kibana_streams_assets`,
-                  is_write_index: false,
-                },
-              },
-            ],
-          });
-
-          await esClient.indices.create({
-            index: `.kibana_streams_assets-000002`,
-          });
-
-          await unlinkDashboard(SEARCH_DASHBOARD_ID);
-          await linkDashboard(SEARCH_DASHBOARD_ID);
+          // disabling and re-enabling streams wipes the asset links
+          await disableStreams(apiClient);
+          await enableStreams(apiClient);
         });
 
-        it('there are no duplicates', async () => {
+        it('dropped all dashboards', async () => {
           const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', {
             params: { path: { id: 'logs' } },
           });
 
           expect(response.status).to.eql(200);
 
-          expect(response.body.dashboards.length).to.eql(1);
-
-          const esResponse = await esClient.search({
-            index: `.kibana_streams_assets`,
-          });
-
-          expect(esResponse.hits.hits.length).to.eql(1);
-        });
-      });
-
-      describe('after deleting the indices and relinking the dashboard', () => {
-        before(async () => {
-          await deleteAssetIndices();
-
-          await unlinkDashboard(SEARCH_DASHBOARD_ID);
-          await linkDashboard(SEARCH_DASHBOARD_ID);
+          expect(response.body.dashboards.length).to.eql(0);
         });
 
         it('recovers on write and lists the linked dashboard ', async () => {
+          await unlinkDashboard(SEARCH_DASHBOARD_ID);
+          await linkDashboard(SEARCH_DASHBOARD_ID);
+
           const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', {
             params: { path: { id: 'logs' } },
           });