[CI] Add check for possible new FTR configs (#132731)

packages/kbn-test/src/functional_test_runner/index.ts

@@ -14,6 +14,7 @@ export {
   EsVersion,
   Lifecycle,
   LifecyclePhase,
+  FTR_CONFIGS_MANIFEST_PATHS,
 } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';

packages/kbn-test/src/functional_test_runner/lib/config/index.ts

@@ -8,3 +8,4 @@
 
 export { Config } from './config';
 export { readConfigFile } from './read_config_file';
+export { FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest';

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -8,7 +8,7 @@
 
 export { Lifecycle } from './lifecycle';
 export { LifecyclePhase } from './lifecycle_phase';
-export { readConfigFile, Config } from './config';
+export { readConfigFile, Config, FTR_CONFIGS_MANIFEST_PATHS } from './config';
 export * from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';

src/dev/precommit_hook/check_possible_new_ftr_config.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FTR_CONFIGS_MANIFEST_PATHS } from '@kbn/test';
+import { readFileSync } from 'fs';
+
+// This isn't meant to be 100% accurate at identifying new config files
+// But it should catch /most/ of them with no false positives
+// As of writing, when run against every file in the repo, it identifies about 90% of FTR configs with 0 false positives.
+export function checkPossibleNewFtrConfig(newFiles, ftrConfigs = FTR_CONFIGS_MANIFEST_PATHS) {
+  const possibleConfigs = newFiles
+    .filter(({ path }) => {
+      if (!path.match(/(test|e2e).*config[^\/]*\.(t|j)s$/)) {
+        return false;
+      }
+
+      if (path.match(/\/__(fixtures|tests)__\//)) {
+        return false;
+      }
+
+      if (path.match(/\.test\.(t|j)s$/)) {
+        return false;
+      }
+
+      if (path.match(/\/common\/config.(t|j)s$/)) {
+        return false;
+      }
+
+      return readFileSync(path)
+        .toString()
+        .match(/(testRunner)|(testFiles)/);
+    })
+    .map(({ relativePath }) => relativePath);
+
+  for (const config of possibleConfigs) {
+    if (!ftrConfigs.includes(config)) {
+      throw new Error(
+        `${config} looks like a new FTR config. Please add it to .buildkite/ftr_configs.yml. If it's not a new FTR config, please contact #kibana-operations`
+      );
+    }
+  }
+}

src/dev/precommit_hook/index.js

@@ -8,3 +8,4 @@
 
 export { checkFileCasing } from './check_file_casing';
 export { getFilesForCommit } from './get_files_for_commit';
+export { checkPossibleNewFtrConfig } from './check_possible_new_ftr_config';

src/dev/run_precommit_hook.js

@@ -8,13 +8,12 @@
 
 import SimpleGit from 'simple-git/promise';
 
-import { combineErrors } from '@kbn/dev-utils';
 import { run } from '@kbn/dev-cli-runner';
-import { createFlagError } from '@kbn/dev-cli-errors';
+import { createFlagError, combineErrors } from '@kbn/dev-cli-errors';
 import { REPO_ROOT } from '@kbn/utils';
 import * as Eslint from './eslint';
 import * as Stylelint from './stylelint';
-import { getFilesForCommit, checkFileCasing } from './precommit_hook';
+import { getFilesForCommit, checkFileCasing, checkPossibleNewFtrConfig } from './precommit_hook';
 
 run(
   async ({ log, flags }) => {
@@ -37,6 +36,12 @@ run(
       return;
     }
 
+    try {
+      checkPossibleNewFtrConfig(files);
+    } catch (error) {
+      errors.push(error);
+    }
+
     try {
       await checkFileCasing(log, files);
     } catch (error) {