Filter test files from command line (#23010) (#23017)

* Filter test files from command line

* [ftr] add option to exclude test files

* [ftr] fail if exclude paths were not found

* [ftr] list excluded paths that did not match

src/functional_test_runner/cli.js

@@ -28,11 +28,20 @@ const cmd = new Command('node scripts/functional_test_runner');
 const resolveConfigPath = v => resolve(process.cwd(), v);
 const defaultConfigPath = resolveConfigPath('test/functional/config.js');
 
+const collectExcludePaths = () => {
+  const paths = [];
+  return (arg) => {
+    paths.push(resolve(arg));
+    return paths;
+  };
+};
+
 cmd
   .option('--config [path]', 'Path to a config file', resolveConfigPath, defaultConfigPath)
   .option('--bail', 'stop tests after the first failure', false)
   .option('--grep <pattern>', 'pattern used to select which tests to run')
   .option('--invert', 'invert grep to exclude tests', false)
+  .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
@@ -58,7 +67,8 @@ const functionalTestRunner = createFunctionalTestRunner({
       grep: cmd.grep,
       invert: cmd.invert,
     },
-    updateBaselines: cmd.updateBaselines
+    updateBaselines: cmd.updateBaselines,
+    excludeTestFiles: cmd.exclude
   }
 });
 

src/functional_test_runner/lib/config/schema.js

@@ -60,6 +60,8 @@ export const schema = Joi.object().keys({
     otherwise: Joi.default([]),
   }),
 
+  excludeTestFiles: Joi.array().items(Joi.string()).default([]),
+
   services: Joi.object().pattern(
     ID_PATTERN,
     Joi.func().required()

src/functional_test_runner/lib/mocha/load_test_files.js

@@ -31,12 +31,20 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = (mocha, log, lifecycle, providers, paths, updateBaselines) => {
+export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, excludePaths, updateBaselines }) => {
+  const pendingExcludes = new Set(excludePaths.slice(0));
+
   const innerLoadTestFile = (path) => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');
     }
 
+    if (pendingExcludes.has(path)) {
+      pendingExcludes.delete(path);
+      log.warning('Skipping test file %s', path);
+      return;
+    }
+
     loadTracer(path, `testFile[${path}]`, () => {
       log.verbose('Loading test file %s', path);
 
@@ -74,9 +82,14 @@ export const loadTestFiles = (mocha, log, lifecycle, providers, paths, updateBas
 
       mocha.suite.emit('require', returnVal, path, mocha);
       mocha.suite.emit('post-require', global, path, mocha);
+
       context.revertProxiedAssignments();
     });
   };
 
   paths.forEach(innerLoadTestFile);
+
+  if (pendingExcludes.size) {
+    throw new Error(`After loading all test files some exclude paths were not consumed:${['', ...pendingExcludes].join('\n  -')}`);
+  }
 };

src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -46,6 +46,14 @@ export async function setupMocha(lifecycle, log, config, providers) {
     await lifecycle.trigger('beforeEachTest');
   });
 
-  loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles'), config.get('updateBaselines'));
+  loadTestFiles({
+    mocha,
+    log,
+    lifecycle,
+    providers,
+    paths: config.get('testFiles'),
+    excludePaths: config.get('excludeTestFiles'),
+    updateBaselines: config.get('updateBaselines'),
+  });
   return mocha;
 }