[ML] Add tests for anomaly embeddables migrations (#116520) (#117429)

* [ML] Add tests for anomaly charts embeddable migrations

* [ML] Broaden tests for anomaly swimlane as well

* [ML] Fix function rename

* [ML] Update tests to use bulk api

* [ML] Remove override

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Quynh Nguyen <43350163+qn895@users.noreply.github.com>

x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts

@@ -95,7 +95,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         it('can open job selection flyout', async () => {
           await PageObjects.dashboard.clickCreateDashboardPrompt();
           await ml.dashboardEmbeddables.assertDashboardIsEmpty();
-          await ml.dashboardEmbeddables.openJobSelectionFlyout();
+          await ml.dashboardEmbeddables.openAnomalyJobSelectionFlyout('ml_anomaly_charts');
           await a11y.testAppSnapshot();
         });
 

x-pack/test/functional/apps/ml/embeddables/anomaly_charts_dashboard_embeddables.ts

@@ -6,42 +6,16 @@
  */
 
 import { FtrProviderContext } from '../../../ftr_provider_context';
-import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';
-
-// @ts-expect-error not full interface
-const JOB_CONFIG: Job = {
-  job_id: `fq_multi_1_ae`,
-  description:
-    'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
-  groups: ['farequote', 'automated', 'multi-metric'],
-  analysis_config: {
-    bucket_span: '1h',
-    influencers: ['airline'],
-    detectors: [
-      { function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
-      { function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
-      { function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
-    ],
-  },
-  data_description: { time_field: '@timestamp' },
-  analysis_limits: { model_memory_limit: '20mb' },
-  model_plot_config: { enabled: true },
-};
-
-// @ts-expect-error not full interface
-const DATAFEED_CONFIG: Datafeed = {
-  datafeed_id: 'datafeed-fq_multi_1_ae',
-  indices: ['ft_farequote'],
-  job_id: 'fq_multi_1_ae',
-  query: { bool: { must: [{ match_all: {} }] } },
-};
+import { JOB_CONFIG, DATAFEED_CONFIG, ML_EMBEDDABLE_TYPES } from './constants';
 
 const testDataList = [
   {
+    type: 'testData',
     suiteSuffix: 'with multi metric job',
     panelTitle: `ML anomaly charts for ${JOB_CONFIG.job_id}`,
     jobConfig: JOB_CONFIG,
     datafeedConfig: DATAFEED_CONFIG,
+    dashboardTitle: `ML anomaly charts for fq_multi_1_ae ${Date.now()}`,
     expected: {
       influencers: [
         {
@@ -59,7 +33,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const ml = getService('ml');
   const PageObjects = getPageObjects(['common', 'timePicker', 'dashboard']);
 
-  describe('anomaly charts', function () {
+  describe('anomaly charts in dashboard', function () {
     this.tags(['mlqa']);
 
     before(async () => {
@@ -69,6 +43,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await ml.securityUI.loginAsMlPowerUser();
     });
 
+    after(async () => {
+      await ml.api.cleanMlIndices();
+    });
+
     for (const testData of testDataList) {
       describe(testData.suiteSuffix, function () {
         before(async () => {
@@ -79,14 +57,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           await PageObjects.common.navigateToApp('dashboard');
         });
 
-        after(async () => {
-          await ml.api.cleanMlIndices();
-        });
-
         it('can open job selection flyout', async () => {
-          await PageObjects.dashboard.clickCreateDashboardPrompt();
+          await PageObjects.dashboard.clickNewDashboard();
           await ml.dashboardEmbeddables.assertDashboardIsEmpty();
-          await ml.dashboardEmbeddables.openJobSelectionFlyout();
+          await ml.dashboardEmbeddables.openAnomalyJobSelectionFlyout(
+            ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS
+          );
         });
 
         it('can select jobs', async () => {
@@ -109,6 +85,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           await PageObjects.timePicker.pauseAutoRefresh();
           await ml.dashboardEmbeddables.assertAnomalyChartsSeverityThresholdControlExists();
           await ml.dashboardEmbeddables.assertAnomalyChartsExists();
+          await PageObjects.dashboard.saveDashboard(testData.dashboardTitle);
         });
       });
     }

x-pack/test/functional/apps/ml/embeddables/anomaly_embeddables_migration.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+import { JOB_CONFIG, DATAFEED_CONFIG, ML_EMBEDDABLE_TYPES } from './constants';
+
+const testDataList = [
+  {
+    type: ML_EMBEDDABLE_TYPES.ANOMALY_SWIMLANE,
+    panelTitle: 'ML anomaly swim lane',
+    dashboardSavedObject: {
+      type: 'dashboard',
+      attributes: {
+        title: `7.15.2 ML anomaly swimlane dashboard ${Date.now()}`,
+        description: '',
+        panelsJSON: `[{"version":"7.15.2","type":"ml_anomaly_swimlane","gridData":{"x":0,"y":0,"w":24,"h":15,"i":"c177ed0a-dea0-40f8-8980-cfb0c6bc13a8"},"panelIndex":"c177ed0a-dea0-40f8-8980-cfb0c6bc13a8","embeddableConfig":{"jobIds":["fq_multi_1_ae"],"swimlaneType":"viewBy","viewBy":"airline","enhancements":{}},"title":"ML anomaly swim lane"}]`,
+        optionsJSON: '{"useMargins":true,"syncColors":false,"hidePanelTitles":false}',
+        timeRestore: true,
+        timeTo: '2016-02-11T00:00:00.000Z',
+        timeFrom: '2016-02-07T00:00:00.000Z',
+        refreshInterval: {
+          pause: true,
+          value: 0,
+        },
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: '{"query":{"query":"","language":"kuery"},"filter":[]}',
+        },
+      },
+      coreMigrationVersion: '7.15.2',
+    },
+  },
+  {
+    type: ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS,
+    panelTitle: 'ML anomaly charts',
+    dashboardSavedObject: {
+      type: 'dashboard',
+      attributes: {
+        title: `7.15.2 ML anomaly charts dashboard ${Date.now()}`,
+        description: '',
+        panelsJSON:
+          '[{"version":"7.15.2","type":"ml_anomaly_charts","gridData":{"x":0,"y":0,"w":38,"h":21,"i":"1155890b-c19c-4d98-8153-50e6434612f1"},"panelIndex":"1155890b-c19c-4d98-8153-50e6434612f1","embeddableConfig":{"jobIds":["fq_multi_1_ae"],"maxSeriesToPlot":6,"severityThreshold":0,"enhancements":{}},"title":"ML anomaly charts"}]',
+        optionsJSON: '{"useMargins":true,"syncColors":false,"hidePanelTitles":false}',
+        timeRestore: true,
+        timeTo: '2016-02-11T00:00:00.000Z',
+        timeFrom: '2016-02-07T00:00:00.000Z',
+        refreshInterval: {
+          pause: true,
+          value: 0,
+        },
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: '{"query":{"query":"","language":"kuery"},"filter":[]}',
+        },
+      },
+      coreMigrationVersion: '7.15.2',
+    },
+  },
+];
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const ml = getService('ml');
+  const PageObjects = getPageObjects(['common', 'timePicker', 'dashboard']);
+
+  describe('anomaly embeddables migration in Dashboard', function () {
+    this.tags(['mlqa']);
+
+    before(async () => {
+      await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
+      await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');
+      await ml.testResources.setKibanaTimeZoneToUTC();
+      await ml.securityUI.loginAsMlPowerUser();
+
+      await ml.api.createAndRunAnomalyDetectionLookbackJob(JOB_CONFIG, DATAFEED_CONFIG);
+      // Using bulk API because create API might return 400 for conflict errors
+      await ml.testResources.createBulkSavedObjects(
+        testDataList.map((d) => d.dashboardSavedObject)
+      );
+
+      await PageObjects.common.navigateToApp('dashboard');
+    });
+
+    after(async () => {
+      await ml.api.cleanMlIndices();
+    });
+
+    for (const testData of testDataList) {
+      const { dashboardSavedObject, panelTitle, type } = testData;
+      describe(`for ${panelTitle}`, function () {
+        before(async () => {
+          await PageObjects.common.navigateToApp('dashboard');
+        });
+
+        after(async () => {
+          await ml.testResources.deleteDashboardByTitle(dashboardSavedObject.attributes.title);
+        });
+
+        it(`loads saved dashboard from version ${dashboardSavedObject.coreMigrationVersion}`, async () => {
+          await PageObjects.dashboard.loadSavedDashboard(dashboardSavedObject.attributes.title);
+
+          await ml.dashboardEmbeddables.assertDashboardPanelExists(panelTitle);
+
+          if (type === ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS) {
+            await ml.dashboardEmbeddables.assertAnomalyChartsSeverityThresholdControlExists();
+            await ml.dashboardEmbeddables.assertAnomalyChartsExists();
+          }
+
+          if (type === ML_EMBEDDABLE_TYPES.ANOMALY_SWIMLANE) {
+            await ml.dashboardEmbeddables.assertAnomalySwimlaneExists();
+          }
+        });
+      });
+    }
+  });
+}

x-pack/test/functional/apps/ml/embeddables/constants.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Datafeed, Job } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';
+
+// @ts-expect-error not full interface
+export const JOB_CONFIG: Job = {
+  job_id: `fq_multi_1_ae`,
+  description:
+    'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
+  groups: ['farequote', 'automated', 'multi-metric'],
+  analysis_config: {
+    bucket_span: '1h',
+    influencers: ['airline'],
+    detectors: [
+      { function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
+      { function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
+      { function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
+    ],
+  },
+  data_description: { time_field: '@timestamp' },
+  analysis_limits: { model_memory_limit: '20mb' },
+  model_plot_config: { enabled: true },
+};
+
+// @ts-expect-error not full interface
+export const DATAFEED_CONFIG: Datafeed = {
+  datafeed_id: 'datafeed-fq_multi_1_ae',
+  indices: ['ft_farequote'],
+  job_id: 'fq_multi_1_ae',
+  query: { bool: { must: [{ match_all: {} }] } },
+};
+
+export const ML_EMBEDDABLE_TYPES = {
+  ANOMALY_SWIMLANE: 'ml_anomaly_swimlane',
+  ANOMALY_CHARTS: 'ml_anomaly_charts',
+} as const;

x-pack/test/functional/apps/ml/embeddables/index.ts

@@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
   describe('embeddables', function () {
     this.tags(['skipFirefox']);
     loadTestFile(require.resolve('./anomaly_charts_dashboard_embeddables'));
+    loadTestFile(require.resolve('./anomaly_embeddables_migration'));
   });
 }

x-pack/test/functional/services/ml/dashboard_embeddables.ts

@@ -93,13 +93,21 @@ export function MachineLearningDashboardEmbeddablesProvider(
       });
     },
 
-    async openJobSelectionFlyout() {
+    async assertAnomalySwimlaneExists() {
+      await retry.tryForTime(60 * 1000, async () => {
+        await testSubjects.existOrFail(`mlAnomalySwimlaneEmbeddableWrapper`);
+      });
+    },
+
+    async openAnomalyJobSelectionFlyout(
+      mlEmbeddableType: 'ml_anomaly_swimlane' | 'ml_anomaly_charts'
+    ) {
       await retry.tryForTime(60 * 1000, async () => {
         await dashboardAddPanel.clickEditorMenuButton();
         await testSubjects.existOrFail('dashboardEditorContextMenu', { timeout: 2000 });
 
         await dashboardAddPanel.clickEmbeddableFactoryGroupButton('ml');
-        await dashboardAddPanel.clickAddNewEmbeddableLink('ml_anomaly_charts');
+        await dashboardAddPanel.clickAddNewEmbeddableLink(mlEmbeddableType);
 
         await mlDashboardJobSelectionTable.assertJobSelectionTableExists();
       });

x-pack/test/functional/services/ml/test_resources.ts

@@ -128,6 +128,20 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider
       return createResponse.id;
     },
 
+    async createBulkSavedObjects(body: object[]): Promise<string> {
+      log.debug(`Creating bulk saved objects'`);
+
+      const createResponse = await supertest
+        .post(`/api/saved_objects/_bulk_create`)
+        .set(COMMON_REQUEST_HEADERS)
+        .send(body)
+        .expect(200)
+        .then((res: any) => res.body);
+
+      log.debug(` > Created bulk saved objects'`);
+      return createResponse;
+    },
+
     async createIndexPatternIfNeeded(title: string, timeFieldName?: string): Promise<string> {
       const indexPatternId = await this.getIndexPatternId(title);
       if (indexPatternId !== undefined) {