[ftr] rework ciGroup validation to remove JOBS.yml and avoid duplication (#109149)

Co-authored-by: spalger <spalger@users.noreply.github.com>

.ci/ci_groups.yml

@@ -0,0 +1,29 @@
+root:
+  - ciGroup1
+  - ciGroup2
+  - ciGroup3
+  - ciGroup4
+  - ciGroup5
+  - ciGroup6
+  - ciGroup7
+  - ciGroup8
+  - ciGroup9
+  - ciGroup10
+  - ciGroup11
+  - ciGroup12
+
+xpack:
+  - ciGroup1
+  - ciGroup2
+  - ciGroup3
+  - ciGroup4
+  - ciGroup5
+  - ciGroup6
+  - ciGroup7
+  - ciGroup8
+  - ciGroup9
+  - ciGroup10
+  - ciGroup11
+  - ciGroup12
+  - ciGroup13
+  - ciGroupDocker

.ci/jobs.yml

@@ -1,41 +0,0 @@
-# This file is needed by node scripts/ensure_all_tests_in_ci_group  for the list of ciGroups. That must be changed before this file can be removed
-
-JOB:
-  - kibana-intake
-  - kibana-firefoxSmoke
-  - kibana-ciGroup1
-  - kibana-ciGroup2
-  - kibana-ciGroup3
-  - kibana-ciGroup4
-  - kibana-ciGroup5
-  - kibana-ciGroup6
-  - kibana-ciGroup7
-  - kibana-ciGroup8
-  - kibana-ciGroup9
-  - kibana-ciGroup10
-  - kibana-ciGroup11
-  - kibana-ciGroup12
-  - kibana-accessibility
-  - kibana-visualRegression
-
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-firefoxSmoke
-  - x-pack-ciGroup1
-  - x-pack-ciGroup2
-  - x-pack-ciGroup3
-  - x-pack-ciGroup4
-  - x-pack-ciGroup5
-  - x-pack-ciGroup6
-  - x-pack-ciGroup7
-  - x-pack-ciGroup8
-  - x-pack-ciGroup9
-  - x-pack-ciGroup10
-  - x-pack-ciGroup11
-  - x-pack-ciGroup12
-  - x-pack-ciGroup13
-  - x-pack-ciGroupDocker
-  - x-pack-accessibility
-  - x-pack-visualRegression
-
-# `~` is yaml for `null`
-exclude: ~

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -71,6 +71,7 @@ const defaultRelativeToConfigPath = (path: string) => {
 
 export const schema = Joi.object()
   .keys({
+    rootTags: Joi.array().items(Joi.string()),
     testFiles: Joi.array().items(Joi.string()),
     testRunner: Joi.func(),
 

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -21,7 +21,7 @@ function split(arr, fn) {
   return [a, b];
 }
 
-export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) {
+export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTags }) {
   // incremented at the start of each suite, decremented after
   // so that in each non-suite call we can know if we are within
   // a suite, or that when a suite is defined it is within a suite
@@ -62,7 +62,13 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) {
           });
 
           const relativeFilePath = relative(REPO_ROOT, this.file);
-          this._tags = isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath];
+          this._tags = [
+            ...(isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath]),
+            // we attach the "root tags" to all the child suites of the root suite, so that if they
+            // need to be excluded they can be removed from the root suite without removing the entire
+            // root suite
+            ...(this.parent.root ? [...(rootTags ?? [])] : []),
+          ];
           this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file
           this.tags = (tags) => {
             const newTags = Array.isArray(tags) ? tags : [tags];

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -62,6 +62,7 @@ export const loadTestFiles = ({
 
       const context = decorateMochaUi(log, lifecycle, global, {
         isDockerGroup,
+        rootTags: config.get('rootTags'),
       });
       mocha.suite.emit('pre-require', context, path, mocha);
 

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -52,9 +52,9 @@ export async function assertNoneExcluded({ configPath, options }) {
     throw new CliError(`
       ${stats.excludedTests.length} tests in the ${configPath} config
       are excluded when filtering by the tags run on CI. Make sure that all suites are
-      tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
+      tagged with one of the following tags:
 
-      tags: ${JSON.stringify(options.suiteTags)}
+      ${JSON.stringify(options.suiteTags)}
 
       - ${stats.excludedTests.join('\n      - ')}
     `);

packages/kbn-test/src/functional_tests/tasks.js

@@ -55,7 +55,7 @@ const makeSuccessMessage = (options) => {
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function runTests(options) {
-  if (!process.env.KBN_NP_PLUGINS_BUILT) {
+  if (!process.env.KBN_NP_PLUGINS_BUILT && !options.assertNoneExcluded) {
     const log = options.createLogger();
     log.warning('❗️❗️❗️');
     log.warning('❗️❗️❗️');

scripts/ensure_all_tests_in_ci_group.js

@@ -7,4 +7,4 @@
  */
 
 require('../src/setup_node_env');
-require('../src/dev/run_ensure_all_tests_in_ci_group');
+require('../src/dev/ensure_all_tests_in_ci_group').runEnsureAllTestsInCiGroupsCli();

src/dev/ensure_all_tests_in_ci_group.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fs from 'fs/promises';
+
+import execa from 'execa';
+import { safeLoad } from 'js-yaml';
+
+import { run, REPO_ROOT } from '@kbn/dev-utils';
+import { schema } from '@kbn/config-schema';
+
+const RELATIVE_JOBS_YAML_PATH = '.ci/ci_groups.yml';
+const JOBS_YAML_PATH = Path.resolve(REPO_ROOT, RELATIVE_JOBS_YAML_PATH);
+const SCHEMA = schema.object({
+  root: schema.arrayOf(schema.string()),
+  xpack: schema.arrayOf(schema.string()),
+});
+
+export function runEnsureAllTestsInCiGroupsCli() {
+  run(async ({ log }) => {
+    const { root, xpack } = SCHEMA.validate(safeLoad(await Fs.readFile(JOBS_YAML_PATH, 'utf-8')));
+
+    log.info(
+      'validating root tests directory contains all "root" ciGroups from',
+      RELATIVE_JOBS_YAML_PATH
+    );
+    await execa(process.execPath, [
+      'scripts/functional_tests',
+      ...root.map((tag) => `--include-tag=${tag}`),
+      '--include-tag=runOutsideOfCiGroups',
+      '--assert-none-excluded',
+    ]);
+
+    log.info(
+      'validating x-pack/tests directory contains all "xpack" ciGroups from',
+      RELATIVE_JOBS_YAML_PATH
+    );
+    await execa(process.execPath, [
+      'x-pack/scripts/functional_tests',
+      ...xpack.map((tag) => `--include-tag=${tag}`),
+      '--assert-none-excluded',
+    ]);
+
+    log.success('all tests are in a valid ciGroup');
+  });
+}

src/dev/run_ensure_all_tests_in_ci_group.js

@@ -1,50 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { readFileSync } from 'fs';
-import { resolve } from 'path';
-
-import execa from 'execa';
-import { safeLoad } from 'js-yaml';
-
-import { run } from '@kbn/dev-utils';
-
-const JOBS_YAML = readFileSync(resolve(__dirname, '../../.ci/jobs.yml'), 'utf8');
-const TEST_TAGS = safeLoad(JOBS_YAML)
-  .JOB.filter((id) => id.startsWith('kibana-ciGroup'))
-  .map((id) => id.replace(/^kibana-/, ''));
-
-run(async ({ log }) => {
-  try {
-    const result = await execa(process.execPath, [
-      'scripts/functional_test_runner',
-      ...TEST_TAGS.map((tag) => `--include-tag=${tag}`),
-      '--config',
-      'test/functional/config.js',
-      '--test-stats',
-    ]);
-    const stats = JSON.parse(result.stderr);
-
-    if (stats.excludedTests.length > 0) {
-      log.error(`
-          ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that
-          all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js"
-          knows about the tag that you are using.
-
-          tags: ${JSON.stringify({ include: TEST_TAGS })}
-
-          - ${stats.excludedTests.join('\n          - ')}
-        `);
-      process.exitCode = 1;
-      return;
-    }
-  } catch (error) {
-    log.error(error.stack);
-    process.exitCode = 1;
-  }
-});

test/api_integration/config.js

@@ -13,6 +13,7 @@ export default async function ({ readConfigFile }) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
 
   return {
+    rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./apis')],
     services,
     servers: commonConfig.get('servers'),

test/examples/config.js

@@ -21,6 +21,7 @@ export default async function ({ readConfigFile }) {
   );
 
   return {
+    rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./hello_world'),
       require.resolve('./embeddables'),

test/interpreter_functional/config.ts

@@ -20,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/run_pipeline')],
     services: functionalConfig.get('services'),
     pageObjects: functionalConfig.get('pageObjects'),

test/plugin_functional/config.ts

@@ -20,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./test_suites/usage_collection'),
       require.resolve('./test_suites/telemetry'),

test/scripts/jenkins_build_kibana.sh

@@ -11,24 +11,7 @@ fi
 export KBN_NP_PLUGINS_BUILT=true
 
 echo " -> Ensuring all functional tests are in a ciGroup"
-node scripts/ensure_all_tests_in_ci_group;
-
-echo " -> Ensuring all x-pack functional tests are in a ciGroup"
-node x-pack/scripts/functional_tests --assert-none-excluded \
-  --include-tag ciGroup1 \
-  --include-tag ciGroup2 \
-  --include-tag ciGroup3 \
-  --include-tag ciGroup4 \
-  --include-tag ciGroup5 \
-  --include-tag ciGroup6 \
-  --include-tag ciGroup7 \
-  --include-tag ciGroup8 \
-  --include-tag ciGroup9 \
-  --include-tag ciGroup10 \
-  --include-tag ciGroup11 \
-  --include-tag ciGroup12 \
-  --include-tag ciGroup13 \
-  --include-tag ciGroupDocker
+node scripts/ensure_all_tests_in_ci_group
 
 # Do not build kibana for code coverage run
 if [[ -z "$CODE_COVERAGE" ]] ; then

test/scripts/jenkins_code_coverage.sh

@@ -11,21 +11,4 @@ fi
 export KBN_NP_PLUGINS_BUILT=true
 
 echo " -> Ensuring all functional tests are in a ciGroup"
-node scripts/ensure_all_tests_in_ci_group;
-
-echo " -> Ensuring all x-pack functional tests are in a ciGroup"
-node x-pack/scripts/functional_tests --assert-none-excluded \
---include-tag ciGroup1 \
---include-tag ciGroup2 \
---include-tag ciGroup3 \
---include-tag ciGroup4 \
---include-tag ciGroup5 \
---include-tag ciGroup6 \
---include-tag ciGroup7 \
---include-tag ciGroup8 \
---include-tag ciGroup9 \
---include-tag ciGroup10 \
---include-tag ciGroup11 \
---include-tag ciGroup12 \
---include-tag ciGroup13 \
---include-tag ciGroupDocker
+node scripts/ensure_all_tests_in_ci_group