[lens] split x-pack/test/functional/apps/lens/group3/config.ts into smaller groups (#155860)

## Summary


Re-grouping lens functional tests into 6 groups since 2 groups often
cross the 35+ min runtime. Also, we have retry in place so pipeline
might have +40 minutes run if any of test fails.

<img width="1354" alt="image"
src="https://user-images.githubusercontent.com/10977896/234843379-aec47a81-22dc-4ecf-aee5-b1d489b3b31f.png">

x-pack/test/functional/apps/lens/group1/config.ts 18m 42s
x-pack/test/functional/apps/lens/group2/config.ts 19m 26s
x-pack/test/functional/apps/lens/group3/config.ts 18m 26s
x-pack/test/functional/apps/lens/group4/config.ts 18m 40s
x-pack/test/functional/apps/lens/group5/config.ts 17m 55s
x-pack/test/functional/apps/lens/group6/config.ts 19m 24s

---------

Co-authored-by: Jon <jon@budzenski.me>

.buildkite/ftr_configs.yml

@@ -252,6 +252,9 @@ enabled:
   - x-pack/test/functional/apps/lens/group1/config.ts
   - x-pack/test/functional/apps/lens/group2/config.ts
   - x-pack/test/functional/apps/lens/group3/config.ts
+  - x-pack/test/functional/apps/lens/group4/config.ts
+  - x-pack/test/functional/apps/lens/group5/config.ts
+  - x-pack/test/functional/apps/lens/group6/config.ts
   - x-pack/test/functional/apps/lens/open_in_lens/tsvb/config.ts
   - x-pack/test/functional/apps/lens/open_in_lens/agg_based/config.ts
   - x-pack/test/functional/apps/lens/open_in_lens/dashboard/config.ts

x-pack/plugins/lens/readme.md

@@ -229,7 +229,8 @@ Run all tests from the `x-pack` root directory
   - Run `node scripts/functional_tests_server`
   - Run `node ../scripts/functional_test_runner.js --config ./test/functional/apps/lens/group1/config.ts`
   - Run `node ../scripts/functional_test_runner.js --config ./test/functional/apps/lens/group2/config.ts`
-  - Run `node ../scripts/functional_test_runner.js --config ./test/functional/apps/lens/group3/config.ts`
+  - ...
+  - Run `node ../scripts/functional_test_runner.js --config ./test/functional/apps/lens/group6/config.ts`
 - API Functional tests:
   - Run `node scripts/functional_tests_server`
   - Run `node ../scripts/functional_test_runner.js --config ./test/api_integration/config.ts --grep=Lens`

x-pack/test/functional/apps/lens/group1/index.ts

@@ -37,7 +37,7 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
     };
     let indexPatternString: string;
     before(async () => {
-      await log.debug('Starting lens before method');
+      log.debug('Starting lens before method');
       await browser.setWindowSize(1280, 1200);
       await kibanaServer.savedObjects.cleanStandardList();
       try {
@@ -74,16 +74,9 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
     if (config.get('esTestCluster.ccs')) {
       loadTestFile(require.resolve('./smokescreen'));
     } else {
-      loadTestFile(require.resolve('./smokescreen'));
-      loadTestFile(require.resolve('./ad_hoc_data_view'));
-      loadTestFile(require.resolve('./partition'));
-      loadTestFile(require.resolve('./persistent_context'));
-      loadTestFile(require.resolve('./table_dashboard'));
-      loadTestFile(require.resolve('./table'));
-      loadTestFile(require.resolve('./text_based_languages'));
-      loadTestFile(require.resolve('./fields_list'));
-      loadTestFile(require.resolve('./layer_actions'));
-      loadTestFile(require.resolve('./field_formatters'));
+      // total run time ~16 min
+      loadTestFile(require.resolve('./smokescreen')); // 12m 12s
+      loadTestFile(require.resolve('./ad_hoc_data_view')); // 3m 40s
     }
   });
 };

x-pack/test/functional/apps/lens/group2/index.ts

@@ -14,22 +14,14 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
   const PageObjects = getPageObjects(['timePicker']);
-  const config = getService('config');
-  let remoteEsArchiver;
 
   describe('lens app - group 2', () => {
     const esArchive = 'x-pack/test/functional/es_archives/logstash_functional';
     const localIndexPatternString = 'logstash-*';
-    const remoteIndexPatternString = 'ftr-remote:logstash-*';
     const localFixtures = {
       lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json',
       lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/default',
     };
-
-    const remoteFixtures = {
-      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/lens_basic.json',
-      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/default',
-    };
     let esNode: EsArchiver;
     let fixtureDirs: {
       lensBasic: string;
@@ -37,21 +29,12 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
     };
     let indexPatternString: string;
     before(async () => {
-      await log.debug('Starting lens before method');
+      log.debug('Starting lens before method');
       await browser.setWindowSize(1280, 1200);
       await kibanaServer.savedObjects.cleanStandardList();
-      try {
-        config.get('esTestCluster.ccs');
-        remoteEsArchiver = getService('remoteEsArchiver' as 'esArchiver');
-        esNode = remoteEsArchiver;
-        fixtureDirs = remoteFixtures;
-        indexPatternString = remoteIndexPatternString;
-      } catch (error) {
-        esNode = esArchiver;
-        fixtureDirs = localFixtures;
-        indexPatternString = localIndexPatternString;
-      }
-
+      esNode = esArchiver;
+      fixtureDirs = localFixtures;
+      indexPatternString = localIndexPatternString;
       await esNode.load(esArchive);
       // changing the timepicker default here saves us from having to set it in Discover (~8s)
       await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
@@ -64,21 +47,21 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
     });
 
     after(async () => {
-      await esArchiver.unload(esArchive);
+      await esNode.unload(esArchive);
       await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings();
       await kibanaServer.importExport.unload(fixtureDirs.lensBasic);
       await kibanaServer.importExport.unload(fixtureDirs.lensDefault);
       await kibanaServer.savedObjects.cleanStandardList();
     });
 
-    loadTestFile(require.resolve('./add_to_dashboard'));
-    loadTestFile(require.resolve('./runtime_fields'));
-    loadTestFile(require.resolve('./dashboard'));
-    loadTestFile(require.resolve('./terms'));
-    loadTestFile(require.resolve('./epoch_millis'));
-    loadTestFile(require.resolve('./show_underlying_data'));
-    loadTestFile(require.resolve('./show_underlying_data_dashboard'));
-    loadTestFile(require.resolve('./share'));
-    loadTestFile(require.resolve('./tsdb'));
+    // total run time ~ 16m 20s
+    loadTestFile(require.resolve('./partition')); // 1m 40s
+    loadTestFile(require.resolve('./persistent_context')); // 1m
+    loadTestFile(require.resolve('./table_dashboard')); // 3m 10s
+    loadTestFile(require.resolve('./table')); // 1m 40s
+    loadTestFile(require.resolve('./text_based_languages')); // 3m 40s
+    loadTestFile(require.resolve('./fields_list')); // 2m 7s
+    loadTestFile(require.resolve('./layer_actions')); // 1m 45s
+    loadTestFile(require.resolve('./field_formatters')); // 1m 30s
   });
 };

x-pack/test/functional/apps/lens/group3/index.ts

@@ -71,25 +71,10 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
       await kibanaServer.savedObjects.cleanStandardList();
     });
 
-    loadTestFile(require.resolve('./colors'));
-    loadTestFile(require.resolve('./chart_data'));
-    loadTestFile(require.resolve('./time_shift'));
-    loadTestFile(require.resolve('./drag_and_drop'));
-    loadTestFile(require.resolve('./disable_auto_apply'));
-    loadTestFile(require.resolve('./geo_field'));
-    loadTestFile(require.resolve('./formula'));
-    loadTestFile(require.resolve('./heatmap'));
-    loadTestFile(require.resolve('./gauge'));
-    loadTestFile(require.resolve('./metric'));
-    loadTestFile(require.resolve('./legacy_metric'));
-    loadTestFile(require.resolve('./reference_lines'));
-    loadTestFile(require.resolve('./annotations'));
-    loadTestFile(require.resolve('./inspector'));
-    loadTestFile(require.resolve('./error_handling'));
-    loadTestFile(require.resolve('./lens_tagging'));
-    loadTestFile(require.resolve('./lens_reporting'));
-    // keep these two last in the group in this order because they are messing with the default saved objects
-    loadTestFile(require.resolve('./rollup'));
-    loadTestFile(require.resolve('./no_data'));
+    // total run time ~16 min
+    loadTestFile(require.resolve('./add_to_dashboard')); // 12m 50s
+    loadTestFile(require.resolve('./runtime_fields')); // 1m
+    loadTestFile(require.resolve('./terms')); // 1m 35s
+    loadTestFile(require.resolve('./epoch_millis')); // 30s
   });
 };

x-pack/test/functional/apps/lens/group4/config.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
+
+  return {
+    ...functionalConfig.getAll(),
+    testFiles: [require.resolve('.')],
+  };
+}

x-pack/test/functional/apps/lens/group4/index.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EsArchiver } from '@kbn/es-archiver';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext) => {
+  const browser = getService('browser');
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const PageObjects = getPageObjects(['timePicker']);
+  const config = getService('config');
+  let remoteEsArchiver;
+
+  describe('lens app - group 4', () => {
+    const esArchive = 'x-pack/test/functional/es_archives/logstash_functional';
+    const localIndexPatternString = 'logstash-*';
+    const remoteIndexPatternString = 'ftr-remote:logstash-*';
+    const localFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/default',
+    };
+
+    const remoteFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/default',
+    };
+    let esNode: EsArchiver;
+    let fixtureDirs: {
+      lensBasic: string;
+      lensDefault: string;
+    };
+    let indexPatternString: string;
+    before(async () => {
+      await log.debug('Starting lens before method');
+      await browser.setWindowSize(1280, 1200);
+      await kibanaServer.savedObjects.cleanStandardList();
+      try {
+        config.get('esTestCluster.ccs');
+        remoteEsArchiver = getService('remoteEsArchiver' as 'esArchiver');
+        esNode = remoteEsArchiver;
+        fixtureDirs = remoteFixtures;
+        indexPatternString = remoteIndexPatternString;
+      } catch (error) {
+        esNode = esArchiver;
+        fixtureDirs = localFixtures;
+        indexPatternString = localIndexPatternString;
+      }
+
+      await esNode.load(esArchive);
+      // changing the timepicker default here saves us from having to set it in Discover (~8s)
+      await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.uiSettings.update({
+        defaultIndex: indexPatternString,
+        'dateFormat:tz': 'UTC',
+      });
+      await kibanaServer.importExport.load(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.load(fixtureDirs.lensDefault);
+    });
+
+    after(async () => {
+      await esArchiver.unload(esArchive);
+      await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.importExport.unload(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.unload(fixtureDirs.lensDefault);
+      await kibanaServer.savedObjects.cleanStandardList();
+    });
+
+    // total run time ~16m 30s
+    loadTestFile(require.resolve('./colors')); // 1m 2s
+    loadTestFile(require.resolve('./chart_data')); // 1m 10s
+    loadTestFile(require.resolve('./time_shift')); // 1m
+    loadTestFile(require.resolve('./dashboard')); // 6m 45s
+    loadTestFile(require.resolve('./show_underlying_data')); // 2m
+    loadTestFile(require.resolve('./show_underlying_data_dashboard')); // 2m 10s
+    loadTestFile(require.resolve('./share')); // 1m 20s
+    // keep it last in the group
+    loadTestFile(require.resolve('./tsdb')); // 1m
+  });
+};

x-pack/test/functional/apps/lens/group5/config.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
+
+  return {
+    ...functionalConfig.getAll(),
+    testFiles: [require.resolve('.')],
+  };
+}

x-pack/test/functional/apps/lens/group5/index.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EsArchiver } from '@kbn/es-archiver';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext) => {
+  const browser = getService('browser');
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const PageObjects = getPageObjects(['timePicker']);
+  const config = getService('config');
+  let remoteEsArchiver;
+
+  describe('lens app - group 5', () => {
+    const esArchive = 'x-pack/test/functional/es_archives/logstash_functional';
+    const localIndexPatternString = 'logstash-*';
+    const remoteIndexPatternString = 'ftr-remote:logstash-*';
+    const localFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/default',
+    };
+
+    const remoteFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/default',
+    };
+    let esNode: EsArchiver;
+    let fixtureDirs: {
+      lensBasic: string;
+      lensDefault: string;
+    };
+    let indexPatternString: string;
+    before(async () => {
+      log.debug('Starting lens before method');
+      await browser.setWindowSize(1280, 1200);
+      await kibanaServer.savedObjects.cleanStandardList();
+      try {
+        config.get('esTestCluster.ccs');
+        remoteEsArchiver = getService('remoteEsArchiver' as 'esArchiver');
+        esNode = remoteEsArchiver;
+        fixtureDirs = remoteFixtures;
+        indexPatternString = remoteIndexPatternString;
+      } catch (error) {
+        esNode = esArchiver;
+        fixtureDirs = localFixtures;
+        indexPatternString = localIndexPatternString;
+      }
+
+      await esNode.load(esArchive);
+      // changing the timepicker default here saves us from having to set it in Discover (~8s)
+      await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.uiSettings.update({
+        defaultIndex: indexPatternString,
+        'dateFormat:tz': 'UTC',
+      });
+      await kibanaServer.importExport.load(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.load(fixtureDirs.lensDefault);
+    });
+
+    after(async () => {
+      await esArchiver.unload(esArchive);
+      await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.importExport.unload(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.unload(fixtureDirs.lensDefault);
+      await kibanaServer.savedObjects.cleanStandardList();
+    });
+
+    // total run time ~ 16m
+    loadTestFile(require.resolve('./drag_and_drop')); // 7m 40s
+    loadTestFile(require.resolve('./geo_field')); // 26s
+    loadTestFile(require.resolve('./formula')); // 5m 52s
+    loadTestFile(require.resolve('./heatmap')); // 51s
+    loadTestFile(require.resolve('./gauge')); // 1m 17s
+  });
+};

x-pack/test/functional/apps/lens/group6/config.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
+
+  return {
+    ...functionalConfig.getAll(),
+    testFiles: [require.resolve('.')],
+  };
+}

x-pack/test/functional/apps/lens/group6/index.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EsArchiver } from '@kbn/es-archiver';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext) => {
+  const browser = getService('browser');
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const PageObjects = getPageObjects(['timePicker']);
+  const config = getService('config');
+  let remoteEsArchiver;
+
+  describe('lens app - group 6', () => {
+    const esArchive = 'x-pack/test/functional/es_archives/logstash_functional';
+    const localIndexPatternString = 'logstash-*';
+    const remoteIndexPatternString = 'ftr-remote:logstash-*';
+    const localFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/default',
+    };
+
+    const remoteFixtures = {
+      lensBasic: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/lens_basic.json',
+      lensDefault: 'x-pack/test/functional/fixtures/kbn_archiver/lens/ccs/default',
+    };
+    let esNode: EsArchiver;
+    let fixtureDirs: {
+      lensBasic: string;
+      lensDefault: string;
+    };
+    let indexPatternString: string;
+    before(async () => {
+      log.debug('Starting lens before method');
+      await browser.setWindowSize(1280, 1200);
+      await kibanaServer.savedObjects.cleanStandardList();
+      try {
+        config.get('esTestCluster.ccs');
+        remoteEsArchiver = getService('remoteEsArchiver' as 'esArchiver');
+        esNode = remoteEsArchiver;
+        fixtureDirs = remoteFixtures;
+        indexPatternString = remoteIndexPatternString;
+      } catch (error) {
+        esNode = esArchiver;
+        fixtureDirs = localFixtures;
+        indexPatternString = localIndexPatternString;
+      }
+
+      await esNode.load(esArchive);
+      // changing the timepicker default here saves us from having to set it in Discover (~8s)
+      await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.uiSettings.update({
+        defaultIndex: indexPatternString,
+        'dateFormat:tz': 'UTC',
+      });
+      await kibanaServer.importExport.load(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.load(fixtureDirs.lensDefault);
+    });
+
+    after(async () => {
+      await esArchiver.unload(esArchive);
+      await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings();
+      await kibanaServer.importExport.unload(fixtureDirs.lensBasic);
+      await kibanaServer.importExport.unload(fixtureDirs.lensDefault);
+      await kibanaServer.savedObjects.cleanStandardList();
+    });
+
+    // total run time ~16m
+    loadTestFile(require.resolve('./metric')); // 4m 7s
+    loadTestFile(require.resolve('./legacy_metric')); // 29s
+    loadTestFile(require.resolve('./disable_auto_apply')); // 1m 6s
+    loadTestFile(require.resolve('./reference_lines')); // 1m
+    loadTestFile(require.resolve('./annotations')); // 1m
+    loadTestFile(require.resolve('./inspector')); // 1m 19s
+    loadTestFile(require.resolve('./error_handling')); // 1m 8s
+    loadTestFile(require.resolve('./lens_tagging')); // 1m 9s
+    loadTestFile(require.resolve('./lens_reporting')); // 3m
+    // keep these two last in the group in this order because they are messing with the default saved objects
+    loadTestFile(require.resolve('./rollup')); // 1m 30s
+    loadTestFile(require.resolve('./no_data')); // 36s
+  });
+};