[ftr] support filtering tests by es version (#123289)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-dev-utils/src/serializers/recursive_serializer.ts

@@ -6,11 +6,26 @@
  * Side Public License, v 1.
  */
 
-export function createRecursiveSerializer(test: (v: any) => boolean, print: (v: any) => string) {
+class RawPrint {
+  static fromString(s: string) {
+    return new RawPrint(s);
+  }
+  constructor(public readonly v: string) {}
+}
+
+export function createRecursiveSerializer(
+  test: (v: any) => boolean,
+  print: (v: any, printRaw: (v: string) => RawPrint) => string | RawPrint
+) {
   return {
     test: (v: any) => test(v),
     serialize: (v: any, ...rest: any[]) => {
-      const replacement = print(v);
+      const replacement = print(v, RawPrint.fromString);
+
+      if (replacement instanceof RawPrint) {
+        return replacement.v;
+      }
+
       const printer = rest.pop()!;
       return printer(replacement, ...rest);
     },

packages/kbn-es-archiver/src/cli.ts

@@ -18,7 +18,7 @@ import readline from 'readline';
 import Fs from 'fs';
 
 import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils';
-import { readConfigFile, KbnClient } from '@kbn/test';
+import { readConfigFile, KbnClient, EsVersion } from '@kbn/test';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
 
 import { EsArchiver } from './es_archiver';
@@ -45,7 +45,7 @@ export function runCli() {
       if (typeof configPath !== 'string') {
         throw createFlagError('--config must be a string');
       }
-      const config = await readConfigFile(log, Path.resolve(configPath));
+      const config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(configPath));
       statsMeta.set('ftrConfigPath', configPath);
 
       let esUrl = flags['es-url'];

packages/kbn-test/BUILD.bazel

@@ -69,6 +69,7 @@ RUNTIME_DEPS = [
     "@npm//react-router-dom",
     "@npm//redux",
     "@npm//rxjs",
+    "@npm//semver",
     "@npm//strip-ansi",
     "@npm//xmlbuilder",
     "@npm//xml2js",
@@ -108,6 +109,7 @@ TYPES_DEPS = [
   "@npm//@types/react-dom",
   "@npm//@types/react-redux",
   "@npm//@types/react-router-dom",
+  "@npm//@types/semver",
   "@npm//@types/xml2js",
 ]
 

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -35,6 +35,11 @@ export function runFtrCli() {
   const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner');
   run(
     async ({ flags, log }) => {
+      const esVersion = flags['es-version'] || undefined; // convert "" to undefined
+      if (esVersion !== undefined && typeof esVersion !== 'string') {
+        throw createFlagError('expected --es-version to be a string');
+      }
+
       const functionalTestRunner = new FunctionalTestRunner(
         log,
         makeAbsolutePath(flags.config as string),
@@ -57,7 +62,8 @@ export function runFtrCli() {
           },
           updateBaselines: flags.updateBaselines || flags.u,
           updateSnapshots: flags.updateSnapshots || flags.u,
-        }
+        },
+        esVersion
       );
 
       if (flags.throttle) {
@@ -131,6 +137,7 @@ export function runFtrCli() {
           'include-tag',
           'exclude-tag',
           'kibana-install-dir',
+          'es-version',
         ],
         boolean: [
           'bail',
@@ -150,6 +157,7 @@ export function runFtrCli() {
           --bail             stop tests after the first failure
           --grep <pattern>   pattern used to select which tests to run
           --invert           invert grep to exclude tests
+          --es-version       the elasticsearch version, formatted as "x.y.z"
           --include=file     a test file to be included, pass multiple times for multiple files
           --exclude=file     a test file to be excluded, pass multiple times for multiple files
           --include-tag=tag  a tag to be included, pass multiple times for multiple tags. Only

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import type { Client as EsClient } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 
 import { Suite, Test } from './fake_mocha_types';
@@ -21,6 +22,7 @@ import {
   DockerServersService,
   Config,
   SuiteTracker,
+  EsVersion,
 } from './lib';
 
 export class FunctionalTestRunner {
@@ -28,10 +30,12 @@ export class FunctionalTestRunner {
   public readonly failureMetadata = new FailureMetadata(this.lifecycle);
   private closed = false;
 
+  private readonly esVersion: EsVersion;
   constructor(
     private readonly log: ToolingLog,
     private readonly configFile: string,
-    private readonly configOverrides: any
+    private readonly configOverrides: any,
+    esVersion?: string | EsVersion
   ) {
     for (const [key, value] of Object.entries(this.lifecycle)) {
       if (value instanceof LifecyclePhase) {
@@ -39,6 +43,12 @@ export class FunctionalTestRunner {
         value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
       }
     }
+    this.esVersion =
+      esVersion === undefined
+        ? EsVersion.getDefault()
+        : esVersion instanceof EsVersion
+        ? esVersion
+        : new EsVersion(esVersion);
   }
 
   async run() {
@@ -51,6 +61,27 @@ export class FunctionalTestRunner {
         ...readProviderSpec('PageObject', config.get('pageObjects')),
       ]);
 
+      // validate es version
+      if (providers.hasService('es')) {
+        const es = (await providers.getService('es')) as unknown as EsClient;
+        let esInfo;
+        try {
+          esInfo = await es.info();
+        } catch (error) {
+          throw new Error(
+            `attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ${error.stack}`
+          );
+        }
+
+        if (!this.esVersion.eql(esInfo.version.number)) {
+          throw new Error(
+            `ES reports a version number "${
+              esInfo.version.number
+            }" which doesn't match supplied es version "${this.esVersion.toString()}"`
+          );
+        }
+      }
+
       await providers.loadAll();
 
       const customTestRunner = config.get('testRunner');
@@ -61,7 +92,7 @@ export class FunctionalTestRunner {
         return (await providers.invokeProviderFn(customTestRunner)) || 0;
       }
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
+      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
       await this.lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 
@@ -107,14 +138,14 @@ export class FunctionalTestRunner {
         ...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
       ]);
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
+      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
 
       const countTests = (suite: Suite): number =>
         suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length);
 
       return {
         testCount: countTests(mocha.suite),
-        excludedTests: mocha.excludedTests.map((t: Test) => t.fullTitle()),
+        testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()),
       };
     });
   }
@@ -125,7 +156,12 @@ export class FunctionalTestRunner {
     let runErrorOccurred = false;
 
     try {
-      const config = await readConfigFile(this.log, this.configFile, this.configOverrides);
+      const config = await readConfigFile(
+        this.log,
+        this.esVersion,
+        this.configFile,
+        this.configOverrides
+      );
       this.log.info('Config loaded');
 
       if (
@@ -148,6 +184,7 @@ export class FunctionalTestRunner {
         failureMetadata: () => this.failureMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
+        esVersion: () => this.esVersion,
       });
 
       return await handler(config, coreProviders);

packages/kbn-test/src/functional_test_runner/index.ts

@@ -7,7 +7,7 @@
  */
 
 export { FunctionalTestRunner } from './functional_test_runner';
-export { readConfigFile, Config } from './lib';
+export { readConfigFile, Config, EsVersion } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js

@@ -9,34 +9,41 @@
 import { ToolingLog } from '@kbn/dev-utils';
 import { readConfigFile } from './read_config_file';
 import { Config } from './config';
+import { EsVersion } from '../es_version';
 
 const log = new ToolingLog();
+const esVersion = new EsVersion('8.0.0');
 
 describe('readConfigFile()', () => {
   it('reads config from a file, returns an instance of Config class', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1'));
+    const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.1'));
     expect(config instanceof Config).toBeTruthy();
     expect(config.get('testFiles')).toEqual(['config.1']);
   });
 
   it('merges setting overrides into log', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1'), {
-      screenshots: {
-        directory: 'foo.bar',
-      },
-    });
+    const config = await readConfigFile(
+      log,
+      esVersion,
+      require.resolve('./__fixtures__/config.1'),
+      {
+        screenshots: {
+          directory: 'foo.bar',
+        },
+      }
+    );
 
     expect(config.get('screenshots.directory')).toBe('foo.bar');
   });
 
   it('supports loading config files from within config files', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.2'));
+    const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.2'));
     expect(config.get('testFiles')).toEqual(['config.1', 'config.2']);
   });
 
   it('throws if settings are invalid', async () => {
     try {
-      await readConfigFile(log, require.resolve('./__fixtures__/config.invalid'));
+      await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.invalid'));
       throw new Error('expected readConfigFile() to fail');
     } catch (err) {
       expect(err.message).toMatch(/"foo"/);

packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts

@@ -10,10 +10,16 @@ import { ToolingLog } from '@kbn/dev-utils';
 import { defaultsDeep } from 'lodash';
 
 import { Config } from './config';
+import { EsVersion } from '../es_version';
 
 const cache = new WeakMap();
 
-async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrides: any) {
+async function getSettingsFromFile(
+  log: ToolingLog,
+  esVersion: EsVersion,
+  path: string,
+  settingOverrides: any
+) {
   const configModule = require(path); // eslint-disable-line @typescript-eslint/no-var-requires
   const configProvider = configModule.__esModule ? configModule.default : configModule;
 
@@ -23,9 +29,10 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid
       configProvider,
       configProvider({
         log,
+        esVersion,
         async readConfigFile(p: string, o: any) {
           return new Config({
-            settings: await getSettingsFromFile(log, p, o),
+            settings: await getSettingsFromFile(log, esVersion, p, o),
             primary: false,
             path: p,
           });
@@ -43,9 +50,14 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid
   return settingsWithDefaults;
 }
 
-export async function readConfigFile(log: ToolingLog, path: string, settingOverrides: any = {}) {
+export async function readConfigFile(
+  log: ToolingLog,
+  esVersion: EsVersion,
+  path: string,
+  settingOverrides: any = {}
+) {
   return new Config({
-    settings: await getSettingsFromFile(log, path, settingOverrides),
+    settings: await getSettingsFromFile(log, esVersion, path, settingOverrides),
     primary: true,
     path,
   });

packages/kbn-test/src/functional_test_runner/lib/es_version.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import semver from 'semver';
+import { kibanaPackageJson } from '@kbn/utils';
+
+export class EsVersion {
+  static getDefault() {
+    // example: https://storage.googleapis.com/kibana-ci-es-snapshots-daily/8.0.0/manifest-latest-verified.json
+    const manifestUrl = process.env.ES_SNAPSHOT_MANIFEST;
+    if (manifestUrl) {
+      const match = manifestUrl.match(/\d+\.\d+\.\d+/);
+      if (!match) {
+        throw new Error('unable to extract es version from ES_SNAPSHOT_MANIFEST_URL');
+      }
+      return new EsVersion(match[0]);
+    }
+
+    return new EsVersion(process.env.TEST_ES_BRANCH || kibanaPackageJson.version);
+  }
+
+  public readonly parsed: semver.SemVer;
+
+  constructor(version: string) {
+    const parsed = semver.coerce(version);
+    if (!parsed) {
+      throw new Error(`unable to parse es version [${version}]`);
+    }
+    this.parsed = parsed;
+  }
+
+  toString() {
+    return this.parsed.version;
+  }
+
+  /**
+   * Determine if the ES version matches a semver range, like >=7 or ^8.1.0
+   */
+  matchRange(range: string) {
+    return semver.satisfies(this.parsed, range);
+  }
+
+  /**
+   * Determine if the ES version matches a specific version, ignores things like -SNAPSHOT
+   */
+  eql(version: string) {
+    const other = semver.coerce(version);
+    return other && semver.compareLoose(this.parsed, other) === 0;
+  }
+}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -17,3 +17,4 @@ export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 
 export type { Provider } from './providers';
+export * from './es_version';

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -84,6 +84,9 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTa
 
             this._tags = [...this._tags, ...tagsToAdd];
           };
+          this.onlyEsVersion = (semver) => {
+            this._esVersionRequirement = semver;
+          };
 
           provider.call(this);
 

packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js

@@ -12,9 +12,10 @@ import Mocha from 'mocha';
 import { create as createSuite } from 'mocha/lib/suite';
 import Test from 'mocha/lib/test';
 
-import { filterSuitesByTags } from './filter_suites_by_tags';
+import { filterSuites } from './filter_suites';
+import { EsVersion } from '../es_version';
 
-function setup({ include, exclude }) {
+function setup({ include, exclude, esVersion }) {
   return new Promise((resolve) => {
     const history = [];
 
@@ -55,6 +56,7 @@ function setup({ include, exclude }) {
 
     const level1b = createSuite(level1, 'level 1b');
     level1b._tags = ['level1b'];
+    level1b._esVersionRequirement = '<=8';
     level1b.addTest(new Test('test 1b', () => {}));
 
     const level2 = createSuite(mocha.suite, 'level 2');
@@ -62,7 +64,7 @@ function setup({ include, exclude }) {
     level2a._tags = ['level2a'];
     level2a.addTest(new Test('test 2a', () => {}));
 
-    filterSuitesByTags({
+    filterSuites({
       log: {
         info(...args) {
           history.push(`info: ${format(...args)}`);
@@ -71,6 +73,7 @@ function setup({ include, exclude }) {
       mocha,
       include,
       exclude,
+      esVersion,
     });
 
     mocha.run();
@@ -208,3 +211,27 @@ it('does nothing if everything excluded', async () => {
     ]
   `);
 });
+
+it(`excludes tests which don't meet the esVersionRequirement`, async () => {
+  const { history } = await setup({
+    include: [],
+    exclude: [],
+    esVersion: new EsVersion('9.0.0'),
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+    Array [
+      "info: Only running suites which are compatible with ES version 9.0.0",
+      "suite: ",
+      "suite: level 1",
+      "suite: level 1 level 1a",
+      "hook:  \\"before each\\" hook: rootBeforeEach for \\"test 1a\\"",
+      "hook:  level 1 \\"before each\\" hook: level1BeforeEach for \\"test 1a\\"",
+      "test:  level 1 level 1a test 1a",
+      "suite: level 2",
+      "suite: level 2 level 2a",
+      "hook:  \\"before each\\" hook: rootBeforeEach for \\"test 2a\\"",
+      "test:  level 2 level 2a test 2a",
+    ]
+  `);
+});

packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.ts

@@ -6,6 +6,24 @@
  * Side Public License, v 1.
  */
 
+import { ToolingLog } from '@kbn/dev-utils';
+import { Suite, Test } from '../../fake_mocha_types';
+import { EsVersion } from '../es_version';
+
+interface SuiteInternal extends Suite {
+  _tags?: string[];
+  _esVersionRequirement?: string;
+  suites: SuiteInternal[];
+}
+
+interface Options {
+  log: ToolingLog;
+  mocha: any;
+  include: string[];
+  exclude: string[];
+  esVersion?: EsVersion;
+}
+
 /**
  * Given a mocha instance that has already loaded all of its suites, filter out
  * the suites based on the include/exclude tags. If there are include tags then
@@ -16,23 +34,50 @@
  * @param options.include an array of tags that suites must be tagged with to be run
  * @param options.exclude an array of tags that will be used to exclude suites from the run
  */
-export function filterSuitesByTags({ log, mocha, include, exclude }) {
-  mocha.excludedTests = [];
+export function filterSuites({ log, mocha, include, exclude, esVersion }: Options) {
+  mocha.testsExcludedByTag = [];
+  mocha.testsExcludedByEsVersion = [];
+
   // collect all the tests from some suite, including it's children
-  const collectTests = (suite) =>
+  const collectTests = (suite: SuiteInternal): Test[] =>
     suite.suites.reduce((acc, s) => acc.concat(collectTests(s)), suite.tests);
 
+  if (esVersion) {
+    // traverse the test graph and exclude any tests which don't meet their esVersionRequirement
+    log.info('Only running suites which are compatible with ES version', esVersion.toString());
+    (function recurse(parentSuite: SuiteInternal) {
+      const children = parentSuite.suites;
+      parentSuite.suites = [];
+
+      const meetsEsVersionRequirement = (suite: SuiteInternal) =>
+        !suite._esVersionRequirement || esVersion.matchRange(suite._esVersionRequirement);
+
+      for (const child of children) {
+        if (meetsEsVersionRequirement(child)) {
+          parentSuite.suites.push(child);
+          recurse(child);
+        } else {
+          mocha.testsExcludedByEsVersion = mocha.testsExcludedByEsVersion.concat(
+            collectTests(child)
+          );
+        }
+      }
+    })(mocha.suite);
+  }
+
   // if include tags were provided, filter the tree once to
   // only include branches that are included at some point
   if (include.length) {
     log.info('Only running suites (and their sub-suites) if they include the tag(s):', include);
 
-    const isIncluded = (suite) =>
+    const isIncludedByTags = (suite: SuiteInternal) =>
       !suite._tags ? false : suite._tags.some((t) => include.includes(t));
-    const isChildIncluded = (suite) =>
+
+    const isIncluded = (suite: SuiteInternal) => isIncludedByTags(suite);
+    const isChildIncluded = (suite: SuiteInternal): boolean =>
       suite.suites.some((s) => isIncluded(s) || isChildIncluded(s));
 
-    (function recurse(parentSuite) {
+    (function recurse(parentSuite: SuiteInternal) {
       const children = parentSuite.suites;
       parentSuite.suites = [];
 
@@ -47,13 +92,13 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
         // itself, so strip out its tests and recurse to filter
         // out child suites which are not included
         if (isChildIncluded(child)) {
-          mocha.excludedTests = mocha.excludedTests.concat(child.tests);
+          mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(child.tests);
           child.tests = [];
           parentSuite.suites.push(child);
           recurse(child);
           continue;
         } else {
-          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
+          mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(collectTests(child));
         }
       }
     })(mocha.suite);
@@ -64,9 +109,10 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
   if (exclude.length) {
     log.info('Filtering out any suites that include the tag(s):', exclude);
 
-    const isNotExcluded = (suite) => !suite._tags || !suite._tags.some((t) => exclude.includes(t));
+    const isNotExcluded = (suite: SuiteInternal) =>
+      !suite._tags || !suite._tags.some((t) => exclude.includes(t));
 
-    (function recurse(parentSuite) {
+    (function recurse(parentSuite: SuiteInternal) {
       const children = parentSuite.suites;
       parentSuite.suites = [];
 
@@ -77,7 +123,7 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
           parentSuite.suites.push(child);
           recurse(child);
         } else {
-          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
+          mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(collectTests(child));
         }
       }
     })(mocha.suite);

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -11,7 +11,7 @@ import { relative } from 'path';
 import { REPO_ROOT } from '@kbn/utils';
 
 import { loadTestFiles } from './load_test_files';
-import { filterSuitesByTags } from './filter_suites_by_tags';
+import { filterSuites } from './filter_suites';
 import { MochaReporterProvider } from './reporter';
 import { validateCiGroupTags } from './validate_ci_group_tags';
 
@@ -22,9 +22,10 @@ import { validateCiGroupTags } from './validate_ci_group_tags';
  *  @param  {ToolingLog} log
  *  @param  {Config} config
  *  @param  {ProviderCollection} providers
+ *  @param  {EsVersion} esVersion
  *  @return {Promise<Mocha>}
  */
-export async function setupMocha(lifecycle, log, config, providers) {
+export async function setupMocha(lifecycle, log, config, providers, esVersion) {
   // configure mocha
   const mocha = new Mocha({
     ...config.get('mochaOpts'),
@@ -50,18 +51,26 @@ export async function setupMocha(lifecycle, log, config, providers) {
   // valiate that there aren't any tests in multiple ciGroups
   validateCiGroupTags(log, mocha);
 
+  filterSuites({
+    log,
+    mocha,
+    include: [],
+    exclude: [],
+    esVersion,
+  });
+
   // Each suite has a tag that is the path relative to the root of the repo
   // So we just need to take input paths, make them relative to the root, and use them as tags
   // Also, this is a separate filterSuitesByTags() call so that the test suites will be filtered first by
   //  files, then by tags. This way, you can target tags (like smoke) in a specific file.
-  filterSuitesByTags({
+  filterSuites({
     log,
     mocha,
     include: config.get('suiteFiles.include').map((file) => relative(REPO_ROOT, file)),
     exclude: config.get('suiteFiles.exclude').map((file) => relative(REPO_ROOT, file)),
   });
 
-  filterSuitesByTags({
+  filterSuites({
     log,
     mocha,
     include: config.get('suiteTags.include').map((tag) => tag.replace(/-\d+$/, '')),

packages/kbn-test/src/functional_test_runner/public_types.ts

@@ -6,10 +6,10 @@
  * Side Public License, v 1.
  */
 
-import { ToolingLog } from '@kbn/dev-utils';
+import type { ToolingLog } from '@kbn/dev-utils';
 
-import { Config, Lifecycle, FailureMetadata, DockerServersService } from './lib';
-import { Test, Suite } from './fake_mocha_types';
+import type { Config, Lifecycle, FailureMetadata, DockerServersService, EsVersion } from './lib';
+import type { Test, Suite } from './fake_mocha_types';
 
 export { Lifecycle, Config, FailureMetadata };
 
@@ -57,7 +57,7 @@ export interface GenericFtrProviderContext<
    * @param serviceName
    */
   hasService(
-    serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata' | 'dockerServers'
+    serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata' | 'dockerServers' | 'esVersion'
   ): true;
   hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
   hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
@@ -72,6 +72,7 @@ export interface GenericFtrProviderContext<
   getService(serviceName: 'lifecycle'): Lifecycle;
   getService(serviceName: 'dockerServers'): DockerServersService;
   getService(serviceName: 'failureMetadata'): FailureMetadata;
+  getService(serviceName: 'esVersion'): EsVersion;
   getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
 
   /**
@@ -100,6 +101,7 @@ export class GenericFtrService<ProviderContext extends GenericFtrProviderContext
 
 export interface FtrConfigProviderContext {
   log: ToolingLog;
+  esVersion: EsVersion;
   readConfigFile(path: string): Promise<Config>;
 }
 

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -37,6 +37,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -58,6 +59,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -80,6 +82,7 @@ Object {
   "createLogger": [Function],
   "debug": true,
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -101,6 +104,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -124,6 +128,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
@@ -146,6 +151,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "quiet": true,
   "suiteFiles": Object {
@@ -167,6 +173,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "silent": true,
   "suiteFiles": Object {
@@ -188,6 +195,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "source",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -208,6 +216,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "source",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -228,6 +237,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "installDir": "foo",
   "suiteFiles": Object {
@@ -249,6 +259,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "grep": "management",
   "suiteFiles": Object {
@@ -270,6 +281,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],
@@ -291,6 +303,7 @@ Object {
   ],
   "createLogger": [Function],
   "esFrom": "snapshot",
+  "esVersion": "999.999.999",
   "extraKbnOpts": undefined,
   "suiteFiles": Object {
     "exclude": Array [],

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -10,6 +10,7 @@ import { resolve } from 'path';
 
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+import { EsVersion } from '../../../functional_test_runner';
 
 const options = {
   help: { desc: 'Display this menu and exit.' },
@@ -147,6 +148,7 @@ export function processOptions(userOptions, defaultConfigPaths) {
     configs: configs.map((c) => resolve(c)),
     createLogger,
     extraKbnOpts: userOptions._,
+    esVersion: EsVersion.getDefault(),
   };
 }
 

packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js

@@ -6,9 +6,20 @@
  * Side Public License, v 1.
  */
 
-import { displayHelp, processOptions } from './args';
 import { createAbsolutePathSerializer } from '@kbn/dev-utils';
 
+import { displayHelp, processOptions } from './args';
+
+jest.mock('../../../functional_test_runner/lib/es_version', () => {
+  return {
+    EsVersion: class {
+      static getDefault() {
+        return '999.999.999';
+      }
+    },
+  };
+});
+
 expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd()));
 
 const INITIAL_TEST_ES_FROM = process.env.TEST_ES_FROM;

packages/kbn-test/src/functional_tests/lib/run_ftr.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 import type { ToolingLog } from '@kbn/dev-utils';
-import { FunctionalTestRunner, readConfigFile } from '../../functional_test_runner';
+import { FunctionalTestRunner, readConfigFile, EsVersion } from '../../functional_test_runner';
 import { CliError } from './run_cli';
 
 export interface CreateFtrOptions {
@@ -26,6 +26,7 @@ export interface CreateFtrOptions {
     exclude?: string[];
   };
   updateSnapshots?: boolean;
+  esVersion: EsVersion;
 }
 
 export interface CreateFtrParams {
@@ -34,31 +35,46 @@ export interface CreateFtrParams {
 }
 async function createFtr({
   configPath,
-  options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags, updateSnapshots },
+  options: {
+    installDir,
+    log,
+    bail,
+    grep,
+    updateBaselines,
+    suiteFiles,
+    suiteTags,
+    updateSnapshots,
+    esVersion,
+  },
 }: CreateFtrParams) {
-  const config = await readConfigFile(log, configPath);
+  const config = await readConfigFile(log, esVersion, configPath);
 
   return {
     config,
-    ftr: new FunctionalTestRunner(log, configPath, {
-      mochaOpts: {
-        bail: !!bail,
-        grep,
+    ftr: new FunctionalTestRunner(
+      log,
+      configPath,
+      {
+        mochaOpts: {
+          bail: !!bail,
+          grep,
+        },
+        kbnTestServer: {
+          installDir,
+        },
+        updateBaselines,
+        updateSnapshots,
+        suiteFiles: {
+          include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')],
+          exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')],
+        },
+        suiteTags: {
+          include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')],
+          exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')],
+        },
       },
-      kbnTestServer: {
-        installDir,
-      },
-      updateBaselines,
-      updateSnapshots,
-      suiteFiles: {
-        include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')],
-        exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')],
-      },
-      suiteTags: {
-        include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')],
-        exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')],
-      },
-    }),
+      esVersion
+    ),
   };
 }
 
@@ -71,15 +87,15 @@ export async function assertNoneExcluded({ configPath, options }: CreateFtrParam
   }
 
   const stats = await ftr.getTestStats();
-  if (stats.excludedTests.length > 0) {
+  if (stats.testsExcludedByTag.length > 0) {
     throw new CliError(`
-      ${stats.excludedTests.length} tests in the ${configPath} config
+      ${stats.testsExcludedByTag.length} tests in the ${configPath} config
       are excluded when filtering by the tags run on CI. Make sure that all suites are
       tagged with one of the following tags:
 
       ${JSON.stringify(options.suiteTags)}
 
-      - ${stats.excludedTests.join('\n      - ')}
+      - ${stats.testsExcludedByTag.join('\n      - ')}
     `);
   }
 }

packages/kbn-test/src/functional_tests/tasks.ts

@@ -23,7 +23,7 @@ import {
   CreateFtrOptions,
 } from './lib';
 
-import { readConfigFile } from '../functional_test_runner/lib';
+import { readConfigFile, EsVersion } from '../functional_test_runner/lib';
 
 const makeSuccessMessage = (options: StartServerOptions) => {
   const installDirFlag = options.installDir ? ` --kibana-install-dir=${options.installDir}` : '';
@@ -55,6 +55,7 @@ interface RunTestsParams extends CreateFtrOptions {
   configs: string[];
   /** run from source instead of snapshot */
   esFrom?: string;
+  esVersion: EsVersion;
   createLogger: () => ToolingLog;
   extraKbnOpts: string[];
   assertNoneExcluded: boolean;
@@ -105,7 +106,7 @@ export async function runTests(options: RunTestsParams) {
       log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
 
       await withProcRunner(log, async (procs) => {
-        const config = await readConfigFile(log, configPath);
+        const config = await readConfigFile(log, options.esVersion, configPath);
 
         let es;
         try {
@@ -145,6 +146,7 @@ interface StartServerOptions {
   createLogger: () => ToolingLog;
   extraKbnOpts: string[];
   useDefaultConfig?: boolean;
+  esVersion: EsVersion;
 }
 
 export async function startServers({ ...options }: StartServerOptions) {
@@ -162,7 +164,7 @@ export async function startServers({ ...options }: StartServerOptions) {
   };
 
   await withProcRunner(log, async (procs) => {
-    const config = await readConfigFile(log, options.config);
+    const config = await readConfigFile(log, options.esVersion, options.config);
 
     const es = await runElasticsearch({ config, options: opts });
     await runKibanaServer({

packages/kbn-test/src/kbn_archiver_cli.ts

@@ -12,7 +12,7 @@ import Url from 'url';
 import { RunWithCommands, createFlagError, Flags } from '@kbn/dev-utils';
 import { KbnClient } from './kbn_client';
 
-import { readConfigFile } from './functional_test_runner';
+import { readConfigFile, EsVersion } from './functional_test_runner';
 
 function getSinglePositionalArg(flags: Flags) {
   const positional = flags._;
@@ -57,7 +57,7 @@ export function runKbnArchiverCli() {
           throw createFlagError('expected --config to be a string');
         }
 
-        config = await readConfigFile(log, Path.resolve(flags.config));
+        config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(flags.config));
         statsMeta.set('ftrConfigPath', flags.config);
       }
 

packages/kbn-test/types/ftr_globals/mocha.d.ts

@@ -14,5 +14,11 @@ declare module 'mocha' {
      * Assign tags to the test suite to determine in which CI job it should be run.
      */
     tags(tags: string[] | string): void;
+    /**
+     * Define the ES versions for which this test requires, any version which doesn't meet this range will
+     * cause these tests to be skipped
+     * @param semver any valid semver range, like ">=8"
+     */
+    onlyEsVersion(semver: string): void;
   }
 }