[CI] Add a check to ensure all FTR configs are in the manifest (#132857)

.buildkite/scripts/steps/checks.sh

@@ -6,6 +6,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/bootstrap.sh
 
 .buildkite/scripts/steps/checks/precommit_hook.sh
+.buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/bazel_packages.sh
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh

.buildkite/scripts/steps/checks/ftr_configs.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check FTR Configs
+checks-reporter-with-killswitch "Check FTR Configs" \
+  node scripts/check_ftr_configs

packages/kbn-test/src/functional_test_runner/index.ts

@@ -14,6 +14,7 @@ export {
   EsVersion,
   Lifecycle,
   LifecyclePhase,
+  runCheckFtrConfigsCli,
 } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';

packages/kbn-test/src/functional_test_runner/lib/config/index.ts

@@ -8,3 +8,4 @@
 
 export { Config } from './config';
 export { readConfigFile } from './read_config_file';
+export { runCheckFtrConfigsCli } from './run_check_ftr_configs_cli';

packages/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import execa from 'execa';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/utils';
+import { run } from '@kbn/dev-cli-runner';
+import { createFailError } from '@kbn/dev-cli-errors';
+
+import { FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest';
+
+export async function runCheckFtrConfigsCli() {
+  run(
+    async () => {
+      const { stdout } = await execa('git', [
+        'ls-tree',
+        '--full-tree',
+        '--name-only',
+        '-r',
+        'HEAD',
+      ]);
+
+      const files = stdout
+        .trim()
+        .split('\n')
+        .map((file) => resolve(REPO_ROOT, file));
+
+      const possibleConfigs = files.filter((file) => {
+        if (file.includes('run_check_ftr_configs_cli.ts')) {
+          return false;
+        }
+
+        if (!file.match(/(test|e2e).*config[^\/]*\.(t|j)s$/)) {
+          return false;
+        }
+
+        if (file.match(/\/__(fixtures|tests)__\//)) {
+          return false;
+        }
+
+        if (file.match(/\.test\.(t|j)s$/)) {
+          return false;
+        }
+
+        if (file.match(/\/common\/config.(t|j)s$/)) {
+          return false;
+        }
+
+        return readFileSync(file)
+          .toString()
+          .match(/(testRunner)|(testFiles)/);
+      });
+
+      for (const config of possibleConfigs) {
+        if (!FTR_CONFIGS_MANIFEST_PATHS.includes(config)) {
+          throw createFailError(
+            `${config} looks like a new FTR config. Please add it to .buildkite/ftr_configs.yml. If it's not an FTR config, please contact #kibana-operations`
+          );
+        }
+      }
+    },
+    {
+      description: 'Check that all FTR configs are in .buildkite/ftr_configs.yml',
+    }
+  );
+}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -8,7 +8,7 @@
 
 export { Lifecycle } from './lifecycle';
 export { LifecyclePhase } from './lifecycle_phase';
-export { readConfigFile, Config } from './config';
+export { readConfigFile, Config, runCheckFtrConfigsCli } from './config';
 export * from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';

scripts/check_ftr_configs.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('@kbn/test').runCheckFtrConfigsCli();