[ML] Data Frame Analytics functional tests: adds custom urls creation tests (#160822)

## Summary

Adds basic custom urls functional tests.

flaky test runner
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2545

Part of https://github.com/elastic/kibana/issues/157980

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/ml/public/application/components/custom_urls/custom_url_editor/interval_time_range_selector.tsx

@@ -27,6 +27,7 @@ export const IntervalTimerangeSelector: FC<IntervalTimerangeSelectorProps> = ({
     <>
       <EuiSpacer size="xs" />
       <EuiSwitch
+        data-test-subj="mlJobCustomUrlIntervalTimeRangeSwitch"
         disabled={disabled}
         showLabel={true}
         label={i18n.translate('xpack.ml.customUrlsEditor.addIntervalTimeRangeSwitchLabel', {

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_edit/edit_action_flyout.tsx

@@ -357,7 +357,12 @@ export const EditActionFlyout: FC<Required<EditAction>> = ({ closeFlyout, item }
       <EuiFlyoutFooter>
         <EuiFlexGroup justifyContent="spaceBetween">
           <EuiFlexItem grow={false}>
-            <EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
+            <EuiButtonEmpty
+              data-test-subj="mlAnalyticsEditFlyoutCancelButton"
+              iconType="cross"
+              onClick={closeFlyout}
+              flush="left"
+            >
               {i18n.translate('xpack.ml.dataframe.analyticsList.editFlyoutCancelButtonText', {
                 defaultMessage: 'Cancel',
               })}

x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts

@@ -5,9 +5,34 @@
  * 2.0.
  */
 
+import { TIME_RANGE_TYPE } from '@kbn/ml-plugin/public/application/components/custom_urls/custom_url_editor/constants';
 import type { AnalyticsTableRowDetails } from '../../../services/ml/data_frame_analytics_table';
 import type { FtrProviderContext } from '../../../ftr_provider_context';
 import type { FieldStatsType } from '../common/types';
+import {
+  type DiscoverUrlConfig,
+  type DashboardUrlConfig,
+  type OtherUrlConfig,
+} from '../../../services/ml/data_frame_analytics_edit';
+
+const testDiscoverCustomUrl: DiscoverUrlConfig = {
+  label: 'Show data',
+  indexPattern: 'ft_bank_marketing',
+  queryEntityFieldNames: ['day'],
+  timeRange: TIME_RANGE_TYPE.AUTO,
+};
+
+const testDashboardCustomUrl: DashboardUrlConfig = {
+  label: 'Show dashboard',
+  dashboardName: 'ML Test',
+  queryEntityFieldNames: ['day'],
+  timeRange: TIME_RANGE_TYPE.AUTO,
+};
+
+const testOtherCustomUrl: OtherUrlConfig = {
+  label: 'elastic.co',
+  url: 'https://www.elastic.co/',
+};
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
@@ -15,10 +40,12 @@ export default function ({ getService }: FtrProviderContext) {
   const editedDescription = 'Edited description';
 
   describe('classification creation', function () {
+    let testDashboardId: string | null = null;
     before(async () => {
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification');
       await ml.testResources.createIndexPatternIfNeeded('ft_bank_marketing');
       await ml.testResources.setKibanaTimeZoneToUTC();
+      testDashboardId = await ml.testResources.createMLTestDashboardIfNeeded();
 
       await ml.securityUI.loginAsMlPowerUser();
     });
@@ -345,6 +372,40 @@ export default function ({ getService }: FtrProviderContext) {
           );
         });
 
+        it('adds discover custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for discover url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds discover custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDiscoverCustomUrl(
+            testData.jobId,
+            testDiscoverCustomUrl
+          );
+        });
+
+        it('adds dashboard custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for dashboard url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds dashboard custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDashboardCustomUrl(
+            testData.jobId,
+            testDashboardCustomUrl,
+            {
+              index: 1,
+              url: `dashboards#/view/${testDashboardId}?_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:kuery,query:'day:\"$day$\"'))`,
+            }
+          );
+        });
+
+        it('adds other custom url type to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for other url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds other type custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addOtherTypeCustomUrl(testData.jobId, testOtherCustomUrl);
+        });
+
         it('edits the analytics job and displays it correctly in the job list', async () => {
           await ml.testExecution.logTestStep(
             'should open the edit form for the created job in the analytics table'

x-pack/test/functional/apps/ml/data_frame_analytics/custom_urls.ts

@@ -0,0 +1,197 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { TIME_RANGE_TYPE } from '@kbn/ml-plugin/public/application/components/custom_urls/custom_url_editor/constants';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+import {
+  type DiscoverUrlConfig,
+  type DashboardUrlConfig,
+  type OtherUrlConfig,
+} from '../../../services/ml/data_frame_analytics_edit';
+
+const testDiscoverCustomUrl: DiscoverUrlConfig = {
+  label: 'Show data',
+  indexPattern: 'ft_farequote',
+  queryEntityFieldNames: ['airline'],
+  timeRange: TIME_RANGE_TYPE.AUTO,
+};
+
+const testDashboardCustomUrl: DashboardUrlConfig = {
+  label: 'Show dashboard',
+  dashboardName: 'ML Test',
+  queryEntityFieldNames: ['airline'],
+  timeRange: TIME_RANGE_TYPE.INTERVAL,
+  timeRangeInterval: '1h',
+};
+
+const testOtherCustomUrl: OtherUrlConfig = {
+  label: 'elastic.co',
+  url: 'https://www.elastic.co/',
+};
+
+export default function ({ getService }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const ml = getService('ml');
+  const browser = getService('browser');
+
+  describe('custom urls', function () {
+    const dfaJobId = `fq_regression_${Date.now()}`;
+    const generateDestinationIndex = (analyticsId: string) => `user-${analyticsId}`;
+    let testDashboardId: string | null = null;
+    const dfaJobConfig = {
+      id: dfaJobId,
+      description: 'Regression job based on farequote dataset',
+      source: {
+        index: ['ft_farequote'],
+        query: {
+          match_all: {},
+        },
+      },
+      dest: {
+        index: generateDestinationIndex(dfaJobId),
+        results_field: 'ml',
+      },
+      analysis: {
+        regression: {
+          dependent_variable: 'responsetime',
+          training_percent: 50,
+        },
+      },
+      analyzed_fields: {
+        includes: [],
+        excludes: [],
+      },
+      model_memory_limit: '20mb',
+    };
+
+    before(async () => {
+      await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
+      await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');
+      await ml.testResources.setKibanaTimeZoneToUTC();
+      await ml.securityUI.loginAsMlPowerUser();
+      await ml.api.createAndRunDFAJob(dfaJobConfig);
+    });
+
+    after(async () => {
+      await ml.api.cleanMlIndices();
+      await ml.api.deleteIndices('user-farequote_small');
+      await ml.testResources.deleteIndexPatternByTitle('ft_farequote');
+    });
+
+    describe('run custom urls', function () {
+      before(async () => {
+        testDashboardId = await ml.testResources.createMLTestDashboardIfNeeded();
+      });
+
+      it('opens the custom URLs tab in the edit job flyout', async () => {
+        await ml.testExecution.logTestStep('load the analytics management page');
+        await ml.navigation.navigateToMl();
+        await ml.navigation.navigateToDataFrameAnalytics();
+        await ml.dataFrameAnalyticsTable.waitForAnalyticsToLoad();
+
+        await ml.testExecution.logTestStep('open the custom URLs tab in the edit job flyout');
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.openEditCustomUrlsForJobTab(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.closeEditJobFlyout();
+      });
+
+      it('adds a custom URL with query entities to Discover in the edit job flyout', async () => {
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.addDiscoverCustomUrl(dfaJobId, testDiscoverCustomUrl);
+      });
+
+      it('adds a custom URL to Dashboard in the edit job flyout', async () => {
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.addDashboardCustomUrl(dfaJobId, testDashboardCustomUrl, {
+          index: 1,
+          url: `dashboards#/view/${testDashboardId}?_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:kuery,query:'airline:\"$airline$\"'))`,
+        });
+      });
+
+      it('adds a custom URL to an external page in the edit job flyout', async () => {
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.addOtherTypeCustomUrl(dfaJobId, testOtherCustomUrl);
+      });
+
+      it('tests other type custom URL', async () => {
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.openEditCustomUrlsForJobTab(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.testOtherTypeCustomUrlAction(2, testOtherCustomUrl.url);
+      });
+
+      it('edits other type custom URL', async () => {
+        const edit = {
+          label: `${testOtherCustomUrl.url} edited`,
+          url: `${testOtherCustomUrl.url}guide/index.html`,
+        };
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.openEditCustomUrlsForJobTab(dfaJobId);
+        await ml.testExecution.logTestStep('edit the custom URL in the edit job flyout');
+        await ml.dataFrameAnalyticsEdit.editCustomUrl(2, edit);
+
+        await ml.testExecution.logTestStep('tests custom URL edit has been applied');
+        await ml.dataFrameAnalyticsEdit.testOtherTypeCustomUrlAction(2, edit.url);
+      });
+
+      it('deletes a custom URL', async () => {
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.openEditCustomUrlsForJobTab(dfaJobId);
+        const beforeCustomUrls = await ml.dataFrameAnalyticsEdit.deleteCustomUrl(dfaJobId, 2);
+        // Save the edit and check the custom URL has been deleted.
+        await ml.dataFrameAnalyticsEdit.updateAnalyticsJob();
+        await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.openEditCustomUrlsForJobTab(dfaJobId);
+        await ml.dataFrameAnalyticsEdit.assertCustomUrlsLength(beforeCustomUrls - 1);
+      });
+
+      // wrapping into own describe to make sure new tab is cleaned up even if test failed
+      // see: https://github.com/elastic/kibana/pull/67280#discussion_r430528122
+      describe('tests Discover type custom URL', () => {
+        let tabsCount = 1;
+
+        it('opens Discover page from test link in the edit job flyout', async () => {
+          await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+          await ml.dataFrameAnalyticsEdit.openTestCustomUrl(dfaJobId, 0);
+          await browser.switchTab(1);
+          tabsCount++;
+          await ml.dataFrameAnalyticsEdit.testDiscoverCustomUrlAction(); // Discover has no content for last 15m.
+        });
+
+        after(async () => {
+          if (tabsCount > 1) {
+            await browser.closeCurrentWindow();
+            await browser.switchTab(0);
+            await ml.dataFrameAnalyticsEdit.closeEditJobFlyout();
+          }
+        });
+      });
+
+      // wrapping into own describe to make sure new tab is cleaned up even if test failed
+      // see: https://github.com/elastic/kibana/pull/67280#discussion_r430528122
+      describe('tests Dashboard type custom URL', () => {
+        let tabsCount = 1;
+        const testDashboardPanelCount = 0; // ML Test dashboard has no content.
+
+        it('opens Dashboard page from test link in the edit job flyout', async () => {
+          await ml.dataFrameAnalyticsTable.openEditFlyout(dfaJobId);
+          await ml.dataFrameAnalyticsEdit.openTestCustomUrl(dfaJobId, 1);
+          await browser.switchTab(1);
+          tabsCount++;
+          await ml.dataFrameAnalyticsEdit.testDashboardCustomUrlAction(testDashboardPanelCount);
+        });
+
+        after(async () => {
+          if (tabsCount > 1) {
+            await browser.closeCurrentWindow();
+            await browser.switchTab(0);
+            await ml.dataFrameAnalyticsEdit.closeEditJobFlyout();
+          }
+        });
+      });
+    });
+  });
+}

x-pack/test/functional/apps/ml/data_frame_analytics/index.ts

@@ -27,6 +27,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
       await ml.securityCommon.cleanMlRoles();
 
       await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote_small');
+      await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote');
       await esArchiver.unload('x-pack/test/functional/es_archives/ml/bm_classification');
       await esArchiver.unload('x-pack/test/functional/es_archives/ml/ihp_outlier');
       await esArchiver.unload('x-pack/test/functional/es_archives/ml/egs_regression');
@@ -42,5 +43,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./regression_creation_saved_search'));
     loadTestFile(require.resolve('./classification_creation_saved_search'));
     loadTestFile(require.resolve('./outlier_detection_creation_saved_search'));
+    loadTestFile(require.resolve('./custom_urls'));
   });
 }

x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts

@@ -5,9 +5,34 @@
  * 2.0.
  */
 
+import { TIME_RANGE_TYPE } from '@kbn/ml-plugin/public/application/components/custom_urls/custom_url_editor/constants';
 import type { FtrProviderContext } from '../../../ftr_provider_context';
 import type { AnalyticsTableRowDetails } from '../../../services/ml/data_frame_analytics_table';
 import type { FieldStatsType } from '../common/types';
+import {
+  type DiscoverUrlConfig,
+  type DashboardUrlConfig,
+  type OtherUrlConfig,
+} from '../../../services/ml/data_frame_analytics_edit';
+
+const testDiscoverCustomUrl: DiscoverUrlConfig = {
+  label: 'Show data',
+  indexPattern: 'ft_ihp_outlier',
+  queryEntityFieldNames: ['SaleType'],
+  timeRange: TIME_RANGE_TYPE.AUTO,
+};
+
+const testDashboardCustomUrl: DashboardUrlConfig = {
+  label: 'Show dashboard',
+  dashboardName: 'ML Test',
+  queryEntityFieldNames: ['SaleType'],
+  timeRange: TIME_RANGE_TYPE.AUTO,
+};
+
+const testOtherCustomUrl: OtherUrlConfig = {
+  label: 'elastic.co',
+  url: 'https://www.elastic.co/',
+};
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
@@ -15,9 +40,12 @@ export default function ({ getService }: FtrProviderContext) {
   const editedDescription = 'Edited description';
 
   describe('outlier detection creation', function () {
+    let testDashboardId: string | null = null;
+
     before(async () => {
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ihp_outlier');
       await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier');
+      testDashboardId = await ml.testResources.createMLTestDashboardIfNeeded();
       await ml.testResources.setKibanaTimeZoneToUTC();
 
       await ml.securityUI.loginAsMlPowerUser();
@@ -336,6 +364,40 @@ export default function ({ getService }: FtrProviderContext) {
           );
         });
 
+        it('adds discover custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for discover url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds discover custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDiscoverCustomUrl(
+            testData.jobId,
+            testDiscoverCustomUrl
+          );
+        });
+
+        it('adds dashboard custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for dashboard url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds dashboard custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDashboardCustomUrl(
+            testData.jobId,
+            testDashboardCustomUrl,
+            {
+              index: 1,
+              url: `dashboards#/view/${testDashboardId}?_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:kuery,query:'SaleType:\"$SaleType$\"'))`,
+            }
+          );
+        });
+
+        it('adds other custom url type to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for other url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('add other type custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addOtherTypeCustomUrl(testData.jobId, testOtherCustomUrl);
+        });
+
         it('edits the analytics job and displays it correctly in the job list', async () => {
           await ml.testExecution.logTestStep(
             'should open the edit form for the created job in the analytics table'

x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts

@@ -5,9 +5,15 @@
  * 2.0.
  */
 
+import { TIME_RANGE_TYPE } from '@kbn/ml-plugin/public/application/components/custom_urls/custom_url_editor/constants';
 import type { FtrProviderContext } from '../../../ftr_provider_context';
 import type { AnalyticsTableRowDetails } from '../../../services/ml/data_frame_analytics_table';
 import type { FieldStatsType } from '../common/types';
+import {
+  type DiscoverUrlConfig,
+  type DashboardUrlConfig,
+  type OtherUrlConfig,
+} from '../../../services/ml/data_frame_analytics_edit';
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
@@ -28,11 +34,32 @@ export default function ({ getService }: FtrProviderContext) {
     },
   ];
 
+  const testDiscoverCustomUrl: DiscoverUrlConfig = {
+    label: 'Show data',
+    indexPattern: 'ft_egs_regression',
+    queryEntityFieldNames: ['stabf'],
+    timeRange: TIME_RANGE_TYPE.AUTO,
+  };
+
+  const testDashboardCustomUrl: DashboardUrlConfig = {
+    label: 'Show dashboard',
+    dashboardName: 'ML Test',
+    queryEntityFieldNames: ['stabf'],
+    timeRange: TIME_RANGE_TYPE.AUTO,
+  };
+
+  const testOtherCustomUrl: OtherUrlConfig = {
+    label: 'elastic.co',
+    url: 'https://www.elastic.co/',
+  };
+
   describe('regression creation', function () {
+    let testDashboardId: string | null = null;
     before(async () => {
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/egs_regression');
       await ml.testResources.createIndexPatternIfNeeded('ft_egs_regression');
       await ml.testResources.setKibanaTimeZoneToUTC();
+      testDashboardId = await ml.testResources.createMLTestDashboardIfNeeded();
 
       await ml.securityUI.loginAsMlPowerUser();
     });
@@ -345,6 +372,40 @@ export default function ({ getService }: FtrProviderContext) {
           );
         });
 
+        it('adds discover custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for discover url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds discover custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDiscoverCustomUrl(
+            testData.jobId,
+            testDiscoverCustomUrl
+          );
+        });
+
+        it('adds dashboard custom url to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for dashboard url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds dashboard custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addDashboardCustomUrl(
+            testData.jobId,
+            testDashboardCustomUrl,
+            {
+              index: 1,
+              url: `dashboards#/view/${testDashboardId}?_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:kuery,query:'stabf:\"$stabf$\"'))`,
+            }
+          );
+        });
+
+        it('adds other custom url type to the analytics job', async () => {
+          await ml.testExecution.logTestStep('opens edit flyout for other url');
+          await ml.dataFrameAnalyticsTable.openEditFlyout(testData.jobId);
+
+          await ml.testExecution.logTestStep('adds other type custom url for the analytics job');
+          await ml.dataFrameAnalyticsEdit.addOtherTypeCustomUrl(testData.jobId, testOtherCustomUrl);
+        });
+
         it('edits the analytics job and displays it correctly in the job list', async () => {
           await ml.testExecution.logTestStep(
             'should open the edit form for the created job in the analytics table'

x-pack/test/functional/services/ml/custom_urls.ts

@@ -168,7 +168,7 @@ export function MachineLearningCustomUrlsProvider({
       await PageObjects.header.waitUntilLoadingHasFinished();
     },
 
-    async assertDiscoverCustomUrlAction(expectedHitCountFormatted: string) {
+    async assertDiscoverCustomUrlAction(expectedHitCountFormatted?: string) {
       await PageObjects.discover.waitForDiscoverAppOnScreen();
       await PageObjects.header.waitUntilLoadingHasFinished();
 
@@ -178,14 +178,15 @@ export function MachineLearningCustomUrlsProvider({
       // During cloud tests, the small browser width might cause hit count to be invisible
       // so temporarily collapsing the sidebar ensures the count shows
       await PageObjects.discover.closeSidebar();
-
-      await retry.tryForTime(10 * 1000, async () => {
-        const hitCount = await PageObjects.discover.getHitCount();
-        expect(hitCount).to.eql(
-          expectedHitCountFormatted,
-          `Expected Discover hit count to be '${expectedHitCountFormatted}' (got '${hitCount}')`
-        );
-      });
+      if (expectedHitCountFormatted) {
+        await retry.tryForTime(10 * 1000, async () => {
+          const hitCount = await PageObjects.discover.getHitCount();
+          expect(hitCount).to.eql(
+            expectedHitCountFormatted,
+            `Expected Discover hit count to be '${expectedHitCountFormatted}' (got '${hitCount}')`
+          );
+        });
+      }
     },
 
     async assertDashboardCustomUrlAction(expectedPanelCount: number) {

x-pack/test/functional/services/ml/data_frame_analytics_edit.ts

@@ -7,17 +7,218 @@
 
 import expect from '@kbn/expect';
 
+import {
+  TimeRangeType,
+  TIME_RANGE_TYPE,
+  URL_TYPE,
+} from '@kbn/ml-plugin/public/application/components/custom_urls/custom_url_editor/constants';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { MlCommonUI } from './common_ui';
+import { MlCustomUrls } from './custom_urls';
+
+export interface DiscoverUrlConfig {
+  label: string;
+  indexPattern: string;
+  queryEntityFieldNames: string[];
+  timeRange: TimeRangeType;
+  timeRangeInterval?: string;
+}
+
+export interface DashboardUrlConfig {
+  label: string;
+  dashboardName: string;
+  queryEntityFieldNames: string[];
+  timeRange: TimeRangeType;
+  timeRangeInterval?: string;
+}
+
+export interface OtherUrlConfig {
+  label: string;
+  url: string;
+}
 
 export function MachineLearningDataFrameAnalyticsEditProvider(
-  { getService }: FtrProviderContext,
-  mlCommonUI: MlCommonUI
+  { getPageObject, getService }: FtrProviderContext,
+  mlCommonUI: MlCommonUI,
+  customUrls: MlCustomUrls
 ) {
   const testSubjects = getService('testSubjects');
   const retry = getService('retry');
+  const headerPage = getPageObject('header');
 
   return {
+    async openEditCustomUrlsForJobTab(analyticsId: string) {
+      // click Custom URLs tab
+      await testSubjects.click('mlEditAnalyticsJobFlyout-customUrls');
+      await this.ensureEditCustomUrlTabOpen();
+      await headerPage.waitUntilLoadingHasFinished();
+    },
+
+    async ensureEditCustomUrlTabOpen() {
+      await testSubjects.existOrFail('mlJobOpenCustomUrlFormButton', { timeout: 5000 });
+    },
+
+    async clickOpenCustomUrlEditor() {
+      await this.ensureEditCustomUrlTabOpen();
+      await testSubjects.click('mlJobOpenCustomUrlFormButton');
+      await testSubjects.existOrFail('mlJobCustomUrlForm');
+    },
+
+    async clickIntervalTimeRangeSwitch() {
+      await retry.tryForTime(10 * 1000, async () => {
+        await testSubjects.clickWhenNotDisabledWithoutRetry(
+          'mlJobCustomUrlIntervalTimeRangeSwitch'
+        );
+      });
+      await testSubjects.existOrFail('mlJobCustomUrlTimeRangeIntervalInput');
+    },
+
+    async getExistingCustomUrlCount(): Promise<number> {
+      const existingCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel');
+      return existingCustomUrls.length;
+    },
+
+    async saveCustomUrl(expectedLabel: string, expectedIndex: number, expectedValue?: string) {
+      await retry.tryForTime(5000, async () => {
+        await testSubjects.click('mlJobAddCustomUrl');
+        await customUrls.assertCustomUrlLabel(expectedIndex, expectedLabel);
+      });
+
+      if (expectedValue !== undefined) {
+        await customUrls.assertCustomUrlUrlValue(expectedIndex, expectedValue);
+      }
+    },
+
+    async fillInDiscoverUrlForm(customUrl: DiscoverUrlConfig, addTimerange: boolean = false) {
+      await this.clickOpenCustomUrlEditor();
+      await customUrls.setCustomUrlLabel(customUrl.label);
+      await mlCommonUI.selectRadioGroupValue(
+        `mlJobCustomUrlLinkToTypeInput`,
+        URL_TYPE.KIBANA_DISCOVER
+      );
+      await mlCommonUI.selectSelectValueByVisibleText(
+        'mlJobCustomUrlDiscoverIndexPatternInput',
+        customUrl.indexPattern
+      );
+      await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames);
+      if (addTimerange) {
+        await mlCommonUI.selectSelectValueByVisibleText(
+          'mlJobCustomUrlTimeRangeInput',
+          customUrl.timeRange
+        );
+      }
+      if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) {
+        await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!);
+      }
+    },
+
+    async fillInDashboardUrlForm(customUrl: DashboardUrlConfig) {
+      await this.clickOpenCustomUrlEditor();
+      await customUrls.setCustomUrlLabel(customUrl.label);
+      await mlCommonUI.selectRadioGroupValue(
+        'mlJobCustomUrlLinkToTypeInput',
+        URL_TYPE.KIBANA_DASHBOARD
+      );
+      await mlCommonUI.selectSelectValueByVisibleText(
+        'mlJobCustomUrlDashboardNameInput',
+        customUrl.dashboardName
+      );
+
+      await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames);
+      if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) {
+        await this.clickIntervalTimeRangeSwitch();
+        await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!);
+      }
+    },
+
+    async fillInOtherUrlForm(customUrl: OtherUrlConfig) {
+      await this.clickOpenCustomUrlEditor();
+      await customUrls.setCustomUrlLabel(customUrl.label);
+      await mlCommonUI.selectRadioGroupValue('mlJobCustomUrlLinkToTypeInput', URL_TYPE.OTHER);
+      await customUrls.setCustomUrlOtherTypeUrl(customUrl.url);
+    },
+
+    async addDashboardCustomUrl(
+      jobId: string,
+      customUrl: DashboardUrlConfig,
+      expectedResult: { index: number; url: string }
+    ) {
+      await this.openEditCustomUrlsForJobTab(jobId);
+
+      await retry.tryForTime(30 * 1000, async () => {
+        await this.fillInDashboardUrlForm(customUrl);
+      });
+      await this.saveCustomUrl(customUrl.label, expectedResult.index, expectedResult.url);
+      await this.updateAnalyticsJob();
+    },
+
+    async addDiscoverCustomUrl(jobId: string, customUrl: DiscoverUrlConfig) {
+      await this.openEditCustomUrlsForJobTab(jobId);
+
+      await retry.tryForTime(30 * 1000, async () => {
+        const existingCustomUrlCount = await this.getExistingCustomUrlCount();
+
+        await this.fillInDiscoverUrlForm(customUrl);
+        await this.saveCustomUrl(customUrl.label, existingCustomUrlCount);
+      });
+
+      await this.updateAnalyticsJob();
+    },
+
+    async addOtherTypeCustomUrl(jobId: string, customUrl: OtherUrlConfig) {
+      await this.openEditCustomUrlsForJobTab(jobId);
+
+      await retry.tryForTime(30 * 1000, async () => {
+        const existingCustomUrlCount = await this.getExistingCustomUrlCount();
+
+        await this.fillInOtherUrlForm(customUrl);
+        await this.saveCustomUrl(customUrl.label, existingCustomUrlCount);
+      });
+
+      await this.updateAnalyticsJob();
+    },
+
+    async testOtherTypeCustomUrlAction(indexInList: number, expectedUrl: string) {
+      await customUrls.assertCustomUrlUrlValue(indexInList, expectedUrl);
+      await this.closeEditJobFlyout();
+    },
+
+    async editCustomUrl(indexInList: number, customUrl: { label: string; url: string }) {
+      await customUrls.editCustomUrlLabel(indexInList, customUrl.label);
+      await customUrls.editCustomUrlUrlValue(indexInList, customUrl.url);
+    },
+
+    async deleteCustomUrl(jobId: string, indexInList: number) {
+      const beforeCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel');
+      await customUrls.deleteCustomUrl(indexInList);
+      return beforeCustomUrls.length;
+    },
+
+    async assertCustomUrlsLength(expectedLength: number) {
+      await customUrls.assertCustomUrlsLength(expectedLength);
+      await this.closeEditJobFlyout();
+    },
+
+    async openTestCustomUrl(jobId: string, indexInList: number) {
+      await this.openEditCustomUrlsForJobTab(jobId);
+      await customUrls.clickTestCustomUrl(indexInList);
+    },
+
+    async testDiscoverCustomUrlAction(expectedHitCountFormatted?: string) {
+      await customUrls.assertDiscoverCustomUrlAction(expectedHitCountFormatted);
+    },
+
+    async testDashboardCustomUrlAction(expectedPanelCount: number) {
+      await customUrls.assertDashboardCustomUrlAction(expectedPanelCount);
+    },
+
+    async closeEditJobFlyout() {
+      if (await testSubjects.exists('mlAnalyticsEditFlyoutCancelButton')) {
+        await testSubjects.click('mlAnalyticsEditFlyoutCancelButton');
+        await this.assertAnalyticsEditFlyoutMissing();
+      }
+    },
+
     async assertJobDescriptionEditInputExists() {
       await testSubjects.existOrFail('mlAnalyticsEditFlyoutDescriptionInput');
     },

x-pack/test/functional/services/ml/index.ts

@@ -90,7 +90,11 @@ export function MachineLearningProvider(context: FtrProviderContext) {
     api,
     commonFieldStatsFlyout
   );
-  const dataFrameAnalyticsEdit = MachineLearningDataFrameAnalyticsEditProvider(context, commonUI);
+  const dataFrameAnalyticsEdit = MachineLearningDataFrameAnalyticsEditProvider(
+    context,
+    commonUI,
+    customUrls
+  );
   const dataFrameAnalyticsResults = MachineLearningDataFrameAnalyticsResultsProvider(
     context,
     commonUI,