[ci][ftr][kbn/test] split up CI jobs (#25838)

* [ci][ftr][kbn/test] split up CI jobs

* [ci] run --assert-none-excluded in ci groups, before builds

* [ftr] improve error message when excluded tests found

* [ci] document other places the ciGroups live

.ci/jobs.yml

@@ -1,7 +1,26 @@
 JOB:
-  - selenium
-  - intake
-  - x-pack
+  - kibana-intake
+  - x-pack-intake
+  # make sure all kibana-ciGRoups are listed in tasks/function_test_groups.js
+  - kibana-ciGroup1
+  - kibana-ciGroup2
+  - kibana-ciGroup3
+  - kibana-ciGroup4
+  - kibana-ciGroup5
+  - kibana-ciGroup6
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
+  - x-pack-ciGroup1
+  - x-pack-ciGroup2
+  - x-pack-ciGroup3
+  - x-pack-ciGroup4
+  - x-pack-ciGroup5
+  - x-pack-ciGroup6
 
 # `~` is yaml for `null`
 exclude: ~

.ci/run.sh

@@ -6,15 +6,20 @@ set -e
 cd "$(dirname "$0")/.."
 
 case "$JOB" in
-"selenium")
-  ./test/scripts/jenkins_selenium.sh
-  ;;
-"intake")
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
-"x-pack")
+kibana-ciGroup*)
+  export CI_GROUP="${JOB##kibana-ciGroup}"
+  ./test/scripts/jenkins_ci_group.sh
+  ;;
+x-pack-intake)
   ./test/scripts/jenkins_xpack.sh
   ;;
+x-pack-ciGroup*)
+  export CI_GROUP="${JOB##x-pack-ciGroup}"
+  ./test/scripts/jenkins_xpack_ci_group.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

packages/kbn-test/src/functional_tests/lib/index.js

@@ -19,6 +19,6 @@
 
 export { runKibanaServer } from './run_kibana_server';
 export { runElasticsearch } from './run_elasticsearch';
-export { runFtr, hasTests, assertNoneExcluded } from './fun_ftr';
+export { runFtr, hasTests, assertNoneExcluded } from './run_ftr';
 export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths';
 export { runCli } from './run_cli';

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -39,14 +39,15 @@ export async function assertNoneExcluded({ configPath, options }) {
   const ftr = createFtr({ configPath, options });
 
   const stats = await ftr.getTestStats();
-  if (stats.excludedTests > 0) {
+  if (stats.excludedTests.length > 0) {
     throw new CliError(`
-      ${stats.excludedTests} tests in the ${configPath} config
+      ${stats.excludedTests.length} tests in the ${configPath} config
       are excluded when filtering by the tags run on CI. Make sure that all suites are
       tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
 
-      ${JSON.stringify(options.suiteTags)}
+      tags: ${JSON.stringify(options.suiteTags)}
 
+      - ${stats.excludedTests.join('\n      - ')}
     `);
   }
 }
@@ -65,5 +66,5 @@ export async function runFtr({ configPath, options }) {
 export async function hasTests({ configPath, options }) {
   const ftr = createFtr({ configPath, options });
   const stats = await ftr.getTestStats();
-  return stats.tests > 0;
+  return stats.testCount > 0;
 }

src/dev/jest/junit_reporter.js

@@ -49,7 +49,7 @@ export default class JestJUnitReporter {
    * @return {undefined}
    */
   onRunComplete(contexts, results) {
-    if (!process.env.CI) {
+    if (!process.env.CI || !results.testResults.length) {
       return;
     }
 

src/dev/mocha/junit_report_generation.js

@@ -27,6 +27,8 @@ import xmlBuilder from 'xmlbuilder';
 import { getSnapshotOfRunnableLogs } from './log_cache';
 import { escapeCdata } from '../xml';
 
+const dateNow = Date.now.bind(Date);
+
 export function setupJUnitReportGeneration(runner, options = {}) {
   const {
     reportName = 'Unnamed Mocha Tests',
@@ -47,11 +49,11 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   );
 
   const setStartTime = (node) => {
-    node.startTime = Date.now();
+    node.startTime = dateNow();
   };
 
   const setEndTime = node => {
-    node.endTime = Date.now();
+    node.endTime = dateNow();
   };
 
   const getFullTitle = node => {
@@ -85,6 +87,9 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   runner.on('end', () => {
     // crawl the test graph to collect all defined tests
     const allTests = findAllTests(runner.suite);
+    if (!allTests.length) {
+      return;
+    }
 
     // filter out just the failures
     const failures = results.filter(result => result.failed);

src/functional_test_runner/functional_test_runner.js

@@ -89,8 +89,8 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
         );
 
         return {
-          tests: countTests(mocha.suite),
-          excludedTests: mocha.excludedTests.length
+          testCount: countTests(mocha.suite),
+          excludedTests: mocha.excludedTests.map(t => t.fullTitle())
         };
       });
     }

src/functional_test_runner/lib/config/read_config_file.js

@@ -22,18 +22,17 @@ import { defaultsDeep } from 'lodash';
 import { Config } from './config';
 import { transformDeprecations } from './transform_deprecations';
 
-async function getSettingsFromFile(log, path, settingOverrides) {
-  log.debug('Loading config file from %j', path);
+const cache = new WeakMap();
 
+async function getSettingsFromFile(log, path, settingOverrides) {
   const configModule = require(path);
   const configProvider = configModule.__esModule
     ? configModule.default
     : configModule;
 
-  const settingsWithDefaults = defaultsDeep(
-    {},
-    settingOverrides,
-    await configProvider({
+  if (!cache.has(configProvider)) {
+    log.debug('Loading config file from %j', path);
+    cache.set(configProvider, configProvider({
       log,
       async readConfigFile(...args) {
         return new Config({
@@ -42,7 +41,13 @@ async function getSettingsFromFile(log, path, settingOverrides) {
           path,
         });
       }
-    })
+    }));
+  }
+
+  const settingsWithDefaults = defaultsDeep(
+    {},
+    settingOverrides,
+    await cache.get(configProvider)
   );
 
   const logDeprecation = (...args) => log.error(...args);

tasks/config/run.js

@@ -18,6 +18,7 @@
  */
 
 import { resolve } from 'path';
+import { getFunctionalTestGroupRunConfigs } from '../function_test_groups';
 
 const { version } = require('../../package.json');
 const KIBANA_INSTALL_DIR = `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`;
@@ -220,18 +221,9 @@ module.exports = function (grunt) {
       ],
     },
 
-    functionalTestsRelease: {
-      cmd: process.execPath,
-      args: [
-        'scripts/functional_tests',
-        '--config', 'test/functional/config.js',
-        '--esFrom', esFrom,
-        '--bail',
-        '--debug',
-        '--kibana-install-dir', KIBANA_INSTALL_DIR,
-        '--',
-        '--server.maxPayloadBytes=1648576',
-      ],
-    },
+    ...getFunctionalTestGroupRunConfigs({
+      esFrom,
+      kibanaInstallDir: KIBANA_INSTALL_DIR
+    })
   };
 };

tasks/function_test_groups.js

@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import execa from 'execa';
+import grunt from 'grunt';
+
+/**
+ * The list of tags that we use in the functional tests, if we add a new group we need to add it to this list
+ * and to the list of jobs in .ci/jobs.yml
+ */
+const TEST_TAGS = [
+  'ciGroup1',
+  'ciGroup2',
+  'ciGroup3',
+  'ciGroup4',
+  'ciGroup5',
+  'ciGroup6',
+  'ciGroup7',
+  'ciGroup8',
+  'ciGroup9',
+  'ciGroup10',
+  'ciGroup11',
+  'ciGroup12'
+];
+
+export function getFunctionalTestGroupRunConfigs({ esFrom, kibanaInstallDir } = {}) {
+  return {
+    // include a run task for each test group
+    ...TEST_TAGS.reduce((acc, tag) => ({
+      ...acc,
+      [`functionalTests_${tag}`]: {
+        cmd: process.execPath,
+        args: [
+          'scripts/functional_tests',
+          '--include-tag', tag,
+          '--config', 'test/functional/config.js',
+          '--esFrom', esFrom,
+          '--bail',
+          '--debug',
+          '--kibana-install-dir', kibanaInstallDir,
+          '--',
+          '--server.maxPayloadBytes=1648576',
+        ],
+      }
+    }), {}),
+  };
+}
+
+grunt.registerTask(
+  'functionalTests:ensureAllTestsInCiGroup',
+  'Check that all of the functional tests are in a CI group',
+  async function () {
+    const done = this.async();
+
+    try {
+      const stats = JSON.parse(await execa.stderr(process.execPath, [
+        'scripts/functional_test_runner',
+        ...TEST_TAGS.map(tag => `--include-tag=${tag}`),
+        '--config', 'test/functional/config.js',
+        '--test-stats'
+      ]));
+
+      if (stats.excludedTests.length > 0) {
+        grunt.fail.fatal(`
+          ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that
+          all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js"
+          knows about the tag that you are using.
+
+          tags: ${JSON.stringify({ include: TEST_TAGS })}
+
+          - ${stats.excludedTests.join('\n          - ')}
+        `);
+        return;
+      }
+
+      done();
+    } catch (error) {
+      grunt.fail.fatal(error.stack);
+    }
+  }
+);

tasks/jenkins.js

@@ -41,12 +41,6 @@ module.exports = function (grunt) {
     'run:apiIntegrationTests',
   ]);
 
-  grunt.registerTask('jenkins:selenium', [
-    'checkPlugins',
-    'run:functionalTestsRelease',
-    'run:pluginFunctionalTestsRelease',
-  ]);
-
   grunt.registerTask(
     'jenkins:report',
     'Reports failed tests found in junit xml files to Github issues',

test/functional/apps/console/index.js

@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('console app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1300, 1100);
     });

test/functional/apps/context/index.js

@@ -24,6 +24,8 @@ export default function ({ getService, getPageObjects, loadTestFile }) {
   const kibanaServer = getService('kibanaServer');
 
   describe('context app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       await remote.setWindowSize(1200, 800);
       await esArchiver.loadIfNeeded('logstash_functional');

test/functional/apps/dashboard/index.js

@@ -22,26 +22,30 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
   const esArchiver = getService('esArchiver');
   const PageObjects = getPageObjects(['dashboard']);
 
+  async function loadCurrentData() {
+    await remote.setWindowSize(1300, 900);
+    await PageObjects.dashboard.initTests({
+      kibanaIndex: 'dashboard/current/kibana',
+      dataIndex: 'dashboard/current/data',
+      defaultIndex: 'logstash-*',
+    });
+    await PageObjects.dashboard.preserveCrossAppState();
+  }
+
+  async function unloadCurrentData() {
+    await PageObjects.dashboard.clearSavedObjectsFromAppLinks();
+    await esArchiver.unload('dashboard/current/kibana');
+    await esArchiver.unload('dashboard/current/data');
+  }
+
   describe('dashboard app', function () {
+    // This has to be first since the other tests create some embeddables as side affects and our counting assumes
+    // a fresh index.
     describe('using current data', function () {
-      before(async () => {
-        await remote.setWindowSize(1300, 900);
-        await PageObjects.dashboard.initTests({
-          kibanaIndex: 'dashboard/current/kibana',
-          dataIndex: 'dashboard/current/data',
-          defaultIndex: 'logstash-*',
-        });
-        await PageObjects.dashboard.preserveCrossAppState();
-      });
+      this.tags('ciGroup2');
+      before(loadCurrentData);
+      after(unloadCurrentData);
 
-      after(async function () {
-        await PageObjects.dashboard.clearSavedObjectsFromAppLinks();
-        await esArchiver.unload('dashboard/current/kibana');
-        await esArchiver.unload('dashboard/current/data');
-      });
-
-      // This has to be first since the other tests create some embeddables as side affects and our counting assumes
-      // a fresh index.
       loadTestFile(require.resolve('./_empty_dashboard'));
       loadTestFile(require.resolve('./_dark_theme'));
       loadTestFile(require.resolve('./_embeddable_rendering'));
@@ -50,6 +54,13 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
       loadTestFile(require.resolve('./_dashboard_options'));
       loadTestFile(require.resolve('./_data_shared_attributes'));
       loadTestFile(require.resolve('./_embed_mode'));
+    });
+
+    describe('using current data', function () {
+      this.tags('ciGroup3');
+      before(loadCurrentData);
+      after(unloadCurrentData);
+
       loadTestFile(require.resolve('./_full_screen_mode'));
       loadTestFile(require.resolve('./_dashboard_filter_bar'));
       loadTestFile(require.resolve('./_dashboard_filtering'));
@@ -63,12 +74,19 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
     // the data once to save on time. Eventually, all of these tests should just use current data and we can reserve
     // legacy data only for specifically testing BWC situations.
     describe('using legacy data', function () {
+      this.tags('ciGroup4');
       before(() => remote.setWindowSize(1200, 900));
 
       loadTestFile(require.resolve('./_dashboard_time_picker'));
       loadTestFile(require.resolve('./_bwc_shared_urls'));
       loadTestFile(require.resolve('./_panel_controls'));
       loadTestFile(require.resolve('./_dashboard_state'));
+    });
+
+    describe('using legacy data', function () {
+      this.tags('ciGroup5');
+      before(() => remote.setWindowSize(1200, 900));
+
       loadTestFile(require.resolve('./_dashboard_save'));
       loadTestFile(require.resolve('./_dashboard_time'));
       loadTestFile(require.resolve('./_dashboard_listing'));

test/functional/apps/discover/index.js

@@ -22,6 +22,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('discover app', function () {
+    this.tags('ciGroup6');
+
     before(function () {
       return remote.setWindowSize(1200, 800);
     });

test/functional/apps/getting_started/index.js

@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('Getting Started ', function () {
+    this.tags('ciGroup6');
+
     before(async function () {
       await remote.setWindowSize(1200, 800);
     });

test/functional/apps/home/index.js

@@ -21,6 +21,8 @@ export default function ({ getService, loadTestFile }) {
   const remote = getService('remote');
 
   describe('homepage app', function () {
+    this.tags('ciGroup6');
+
     before(function () {
       return remote.setWindowSize(1200, 800);
     });

test/functional/apps/management/index.js

@@ -21,33 +21,38 @@ export default function ({ getService, loadTestFile }) {
   const esArchiver = getService('esArchiver');
 
   describe('management', function () {
-    // on setup, we create an settingsPage instance
-    // that we will use for all the tests
-    before(async function () {
+    before(async () => {
       await esArchiver.unload('logstash_functional');
       await esArchiver.load('empty_kibana');
       await esArchiver.loadIfNeeded('makelogs');
     });
 
-    after(async function () {
+    after(async () => {
       await esArchiver.unload('makelogs');
       await esArchiver.unload('empty_kibana');
     });
 
-    loadTestFile(require.resolve('./_create_index_pattern_wizard'));
-    loadTestFile(require.resolve('./_index_pattern_create_delete'));
-    loadTestFile(require.resolve('./_index_pattern_results_sort'));
-    loadTestFile(require.resolve('./_index_pattern_popularity'));
-    loadTestFile(require.resolve('./_kibana_settings'));
-    loadTestFile(require.resolve('./_scripted_fields'));
-    loadTestFile(require.resolve('./_scripted_fields_preview'));
-    loadTestFile(require.resolve('./_index_pattern_filter'));
-    loadTestFile(require.resolve('./_scripted_fields_filter'));
-    loadTestFile(require.resolve('./_import_objects'));
-    loadTestFile(require.resolve('./_mgmt_import_saved_objects'));
-    loadTestFile(require.resolve('./_test_huge_fields'));
-    loadTestFile(require.resolve('./_handle_alias'));
-    loadTestFile(require.resolve('./_handle_version_conflict'));
-  });
+    describe('', function () {
+      this.tags('ciGroup7');
 
+      loadTestFile(require.resolve('./_create_index_pattern_wizard'));
+      loadTestFile(require.resolve('./_index_pattern_create_delete'));
+      loadTestFile(require.resolve('./_index_pattern_results_sort'));
+      loadTestFile(require.resolve('./_index_pattern_popularity'));
+      loadTestFile(require.resolve('./_kibana_settings'));
+      loadTestFile(require.resolve('./_scripted_fields'));
+      loadTestFile(require.resolve('./_scripted_fields_preview'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup8');
+
+      loadTestFile(require.resolve('./_index_pattern_filter'));
+      loadTestFile(require.resolve('./_scripted_fields_filter'));
+      loadTestFile(require.resolve('./_import_objects'));
+      loadTestFile(require.resolve('./_test_huge_fields'));
+      loadTestFile(require.resolve('./_handle_alias'));
+      loadTestFile(require.resolve('./_handle_version_conflict'));
+    });
+  });
 }

test/functional/apps/status_page/index.js

@@ -25,6 +25,8 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common']);
 
   describe('status page', function () {
+    this.tags('ciGroup1');
+
     beforeEach(async () => {
       await PageObjects.common.navigateToApp('status_page');
     });

test/functional/apps/timelion/index.js

@@ -24,6 +24,8 @@ export default function ({ getService, loadTestFile }) {
   const kibanaServer = getService('kibanaServer');
 
   describe('timelion app', function () {
+    this.tags('ciGroup1');
+
     before(async function () {
       log.debug('Starting timelion before method');
       remote.setWindowSize(1280, 800);

test/functional/apps/visualize/index.js

@@ -23,8 +23,9 @@ export default function ({ getService, loadTestFile }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
+
   describe('visualize app', function () {
-    before(async function () {
+    before(async ()=> {
       log.debug('Starting visualize before method');
       remote.setWindowSize(1280, 800);
       await esArchiver.loadIfNeeded('logstash_functional');
@@ -32,30 +33,49 @@ export default function ({ getService, loadTestFile }) {
       await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'UTC', 'defaultIndex': 'logstash-*' });
     });
 
-    loadTestFile(require.resolve('./_embedding_chart'));
-    loadTestFile(require.resolve('./_inspector'));
-    loadTestFile(require.resolve('./_chart_types'));
-    loadTestFile(require.resolve('./_experimental_vis'));
-    loadTestFile(require.resolve('./_gauge_chart'));
-    loadTestFile(require.resolve('./_area_chart'));
-    loadTestFile(require.resolve('./_line_chart'));
-    loadTestFile(require.resolve('./_data_table'));
-    loadTestFile(require.resolve('./_data_table_nontimeindex'));
-    loadTestFile(require.resolve('./_pie_chart'));
-    loadTestFile(require.resolve('./_tag_cloud'));
-    loadTestFile(require.resolve('./_tile_map'));
-    loadTestFile(require.resolve('./_region_map'));
-    loadTestFile(require.resolve('./_vertical_bar_chart'));
-    loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex'));
-    loadTestFile(require.resolve('./_heatmap_chart'));
-    loadTestFile(require.resolve('./_point_series_options'));
-    loadTestFile(require.resolve('./_markdown_vis'));
-    loadTestFile(require.resolve('./_tsvb_chart'));
-    loadTestFile(require.resolve('./_shared_item'));
-    loadTestFile(require.resolve('./_input_control_vis'));
-    loadTestFile(require.resolve('./_histogram_request_start'));
-    loadTestFile(require.resolve('./_vega_chart'));
-    loadTestFile(require.resolve('./_lab_mode'));
-    loadTestFile(require.resolve('./_linked_saved_searches.js'));
+    describe('', function () {
+      this.tags('ciGroup9');
+
+      loadTestFile(require.resolve('./_embedding_chart'));
+      loadTestFile(require.resolve('./_chart_types'));
+      loadTestFile(require.resolve('./_area_chart'));
+      loadTestFile(require.resolve('./_data_table'));
+      loadTestFile(require.resolve('./_data_table_nontimeindex'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup10');
+
+      loadTestFile(require.resolve('./_inspector'));
+      loadTestFile(require.resolve('./_experimental_vis'));
+      loadTestFile(require.resolve('./_gauge_chart'));
+      loadTestFile(require.resolve('./_heatmap_chart'));
+      loadTestFile(require.resolve('./_input_control_vis'));
+      loadTestFile(require.resolve('./_histogram_request_start'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup11');
+
+      loadTestFile(require.resolve('./_line_chart'));
+      loadTestFile(require.resolve('./_pie_chart'));
+      loadTestFile(require.resolve('./_region_map'));
+      loadTestFile(require.resolve('./_point_series_options'));
+      loadTestFile(require.resolve('./_markdown_vis'));
+      loadTestFile(require.resolve('./_shared_item'));
+      loadTestFile(require.resolve('./_lab_mode'));
+      loadTestFile(require.resolve('./_linked_saved_searches'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup12');
+
+      loadTestFile(require.resolve('./_tag_cloud'));
+      loadTestFile(require.resolve('./_tile_map'));
+      loadTestFile(require.resolve('./_vertical_bar_chart'));
+      loadTestFile(require.resolve('./_vertical_bar_chart_nontimeindex'));
+      loadTestFile(require.resolve('./_tsvb_chart'));
+      loadTestFile(require.resolve('./_vega_chart'));
+    });
   });
 }

test/scripts/jenkins_ci_group.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+
+"$(FORCE_COLOR=0 yarn bin)/grunt" functionalTests:ensureAllTestsInCiGroup;
+
+node scripts/build --debug --oss;
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_ES_FROM=${TEST_ES_FROM:-source}
+
+"$(FORCE_COLOR=0 yarn bin)/grunt" "run:functionalTests_ciGroup${CI_GROUP}" --from=source;
+
+if [ "$CI_GROUP" == "1" ]; then
+  "$(FORCE_COLOR=0 yarn bin)/grunt" run:pluginFunctionalTestsRelease --from=source;
+fi

test/scripts/jenkins_selenium.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/java_setup.sh"
-
-node scripts/build --debug --oss;
-
-export TEST_BROWSER_HEADLESS=1
-export TEST_ES_FROM=${TEST_ES_FROM:-source}
-"$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:selenium --from=source;

test/scripts/jenkins_xpack.sh

@@ -24,19 +24,3 @@ cd "$XPACK_DIR"
 node scripts/jest --ci --no-cache --verbose
 echo ""
 echo ""
-
-
-echo " -> building and extracting default Kibana distributable for use in functional tests"
-cd "$KIBANA_DIR"
-node scripts/build --debug --no-oss
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
-
-export TEST_ES_FROM=${TEST_ES_FROM:-source}
-echo " -> Running functional and api tests"
-cd "$XPACK_DIR"
-node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir"
-echo ""
-echo ""

test/scripts/jenkins_xpack_ci_group.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
+source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+
+export TEST_BROWSER_HEADLESS=1
+export XPACK_DIR="$(cd "$(dirname "$0")/../../x-pack"; pwd)"
+echo "-> XPACK_DIR ${XPACK_DIR}"
+
+echo " -> Ensuring all functional tests are in a ciGroup"
+cd "$XPACK_DIR"
+node scripts/functional_tests --assert-none-excluded \
+  --include-tag ciGroup1 \
+  --include-tag ciGroup2 \
+  --include-tag ciGroup3 \
+  --include-tag ciGroup4 \
+  --include-tag ciGroup5 \
+  --include-tag ciGroup6
+
+echo " -> building and extracting default Kibana distributable for use in functional tests"
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+export TEST_ES_FROM=${TEST_ES_FROM:-source}
+echo " -> Running functional and api tests"
+cd "$XPACK_DIR"
+node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir" --include-tag "ciGroup$CI_GROUP"
+echo ""
+echo ""

x-pack/test/api_integration/apis/index.js

@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('apis', () => {
+  describe('apis', function () {
+    this.tags('ciGroup5');
+
     loadTestFile(require.resolve('./es'));
     loadTestFile(require.resolve('./security'));
     loadTestFile(require.resolve('./monitoring'));

x-pack/test/functional/apps/dashboard_mode/index.js

@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('dashboard mode', function () {
+    this.tags('ciGroup3');
+
     loadTestFile(require.resolve('./dashboard_view_mode'));
   });
 }

x-pack/test/functional/apps/graph/index.js

@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('graph app', function () {
+    this.tags('ciGroup1');
+
     loadTestFile(require.resolve('./graph'));
   });
 }

x-pack/test/functional/apps/grok_debugger/index.js

@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('logstash', () => {
+  describe('logstash', function () {
+    this.tags('ciGroup2');
+
     loadTestFile(require.resolve('./grok_debugger'));
   });
 }

x-pack/test/functional/apps/infra/home_page.ts

@@ -15,6 +15,10 @@ export default ({ getPageObjects, getService }: KibanaFunctionalTestDefaultProvi
   const pageObjects = getPageObjects(['common', 'infraHome']);
 
   describe('Home page', () => {
+    before(async () => {
+      await esArchiver.load('empty_kibana');
+    });
+
     describe('without metrics present', () => {
       before(async () => await esArchiver.unload('infra'));
 

x-pack/test/functional/apps/infra/index.ts

@@ -8,7 +8,9 @@ import { KibanaFunctionalTestDefaultProviders } from '../../../types/providers';
 
 // tslint:disable-next-line:no-default-export
 export default ({ loadTestFile }: KibanaFunctionalTestDefaultProviders) => {
-  describe('InfraOps app', () => {
+  describe('InfraOps app', function() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./home_page'));
   });
 };

x-pack/test/functional/apps/logstash/index.js

@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('logstash', () => {
+  describe('logstash', function () {
+    this.tags('ciGroup2');
+
     loadTestFile(require.resolve('./pipeline_list'));
     loadTestFile(require.resolve('./pipeline_create'));
   });

x-pack/test/functional/apps/monitoring/index.js

@@ -5,7 +5,9 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('Monitoring app', () => {
+  describe('Monitoring app', function () {
+    this.tags('ciGroup1');
+
     loadTestFile(require.resolve('./cluster/list'));
     loadTestFile(require.resolve('./cluster/overview'));
     loadTestFile(require.resolve('./cluster/alerts'));

x-pack/test/functional/apps/security/index.js

@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('security app', function () {
+    this.tags('ciGroup4');
+
     loadTestFile(require.resolve('./security'));
     loadTestFile(require.resolve('./doc_level_security_roles'));
     loadTestFile(require.resolve('./management'));

x-pack/test/functional/apps/spaces/index.ts

@@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types';
 // tslint:disable:no-default-export
 export default function spacesApp({ loadTestFile }: TestInvoker) {
   describe('Spaces app', function spacesAppTestSuite() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./spaces_selection'));
   });
 }

x-pack/test/functional/apps/status_page/index.ts

@@ -8,6 +8,8 @@ import { TestInvoker } from './lib/types';
 // tslint:disable:no-default-export
 export default function statusPage({ loadTestFile }: TestInvoker) {
   describe('Status page', function statusPageTestSuite() {
+    (this as any).tags('ciGroup4');
+
     loadTestFile(require.resolve('./status_page'));
   });
 }

x-pack/test/functional/apps/watcher/index.js

@@ -6,6 +6,8 @@
 
 export default function ({ loadTestFile }) {
   describe('watcher app', function () {
+    this.tags('ciGroup1');
+
     //loadTestFile(require.resolve('./management'));
     loadTestFile(require.resolve('./watcher_test'));
   });

x-pack/test/reporting/api/chromium_tests.js

@@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
-  describe('chromium', () => {
+  describe('chromium', function () {
+    this.tags('ciGroup6');
+
     before(async () => {
       await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
       await esArchiver.load(OSS_DATA_ARCHIVE_PATH);

x-pack/test/reporting/api/phantom_tests.js

@@ -10,7 +10,9 @@ export default function ({ loadTestFile, getService }) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
-  describe('phantom', () => {
+  describe('phantom', function () {
+    this.tags('ciGroup6');
+
     before(async () => {
       await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
       await esArchiver.load(OSS_DATA_ARCHIVE_PATH);

x-pack/test/reporting/configs/chromium_functional.js

@@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) {
   return {
     ...functionalConfig,
     junit: {
-      reportName: 'X-Pack Chromium API Reporting Tests',
+      reportName: 'X-Pack Chromium Functional Reporting Tests',
     },
     testFiles: [require.resolve('../functional')],
     kbnTestServer: {

x-pack/test/reporting/configs/phantom_functional.js

@@ -13,7 +13,7 @@ export default async function ({ readConfigFile }) {
   return {
     ...functionalConfig,
     junit: {
-      reportName: 'X-Pack Phantom API Reporting Tests',
+      reportName: 'X-Pack Phantom Functional Reporting Tests',
     },
     testFiles: [require.resolve('../functional')],
     kbnTestServer: {

x-pack/test/reporting/functional/index.js

@@ -6,6 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('reporting app', function () {
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/saml_api_integration/apis/index.js

@@ -5,7 +5,8 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('apis SAML', () => {
+  describe('apis SAML', function () {
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./security'));
   });
 }

x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts

@@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('saved objects security and spaces enabled', () => {
+  describe('saved objects security and spaces enabled', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });

x-pack/test/saved_object_api_integration/security_only/apis/index.ts

@@ -12,7 +12,9 @@ export default function({ getService, loadTestFile }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('saved objects security only enabled', () => {
+  describe('saved objects security only enabled', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });

x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts

@@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types';
 
 // tslint:disable:no-default-export
 export default function({ loadTestFile }: TestInvoker) {
-  describe('saved objects spaces only enabled', () => {
+  describe('saved objects spaces only enabled', function() {
+    (this as any).tags('ciGroup5');
+
     loadTestFile(require.resolve('./bulk_create'));
     loadTestFile(require.resolve('./bulk_get'));
     loadTestFile(require.resolve('./create'));

x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts

@@ -12,7 +12,9 @@ export default function({ loadTestFile, getService }: TestInvoker) {
   const es = getService('es');
   const supertest = getService('supertest');
 
-  describe('spaces api with security', () => {
+  describe('spaces api with security', function() {
+    (this as any).tags('ciGroup5');
+
     before(async () => {
       await createUsersAndRoles(es, supertest);
     });

x-pack/test/spaces_api_integration/spaces_only/apis/index.ts

@@ -8,7 +8,9 @@ import { TestInvoker } from '../../common/lib/types';
 
 // tslint:disable:no-default-export
 export default function spacesOnlyTestSuite({ loadTestFile }: TestInvoker) {
-  describe('spaces api without security', () => {
+  describe('spaces api without security', function() {
+    (this as any).tags('ciGroup5');
+
     loadTestFile(require.resolve('./create'));
     loadTestFile(require.resolve('./delete'));
     loadTestFile(require.resolve('./get_all'));