[7.17] Test cypress OOM and chrome upgrade (#163363)

This PR introduces several changes to the Cypress config for 7.17 that tries to mimic the state we currently have on main. The most notable changes are the upgrade to cypress v12 and the addition of the parallelisation. Both have required fundamentally bigger changes to what we had before on that branch. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tiago Costa <tiago.costa@elastic.co> Co-authored-by: Patryk Kopycinski <contact@patrykkopycinski.com>
2025-04-23 17:28:26 -04:00 · 2023-08-14 20:01:14 -05:00 · 2023-08-14 20:01:14 -05:00 · ac50132159
commit ac50132159
parent 1dae582537
190 changed files with 11952 additions and 1061 deletions
--- a/.buildkite/pipelines/pull_request/response_ops.yml
+++ b/.buildkite/pipelines/pull_request/response_ops.yml
@ -2,10 +2,13 @@ steps:
  - command: .buildkite/scripts/steps/functional/response_ops.sh
    label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
    agents:
-      queue: ci-group-6
+      queue: n2-4-spot
    depends_on: build
    timeout_in_minutes: 120
+    parallelism: 4
    retry:
      automatic:
        - exit_status: '*'
          limit: 1
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"
--- a/.buildkite/pipelines/pull_request/response_ops_cases.yml
+++ b/.buildkite/pipelines/pull_request/response_ops_cases.yml
@ -1,11 +0,0 @@
-steps:
-  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
-    label: 'Cases Cypress Tests on Security Solution'
-    agents:
-      queue: ci-group-6
-    depends_on: build
-    timeout_in_minutes: 120
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
--- a/.buildkite/pipelines/pull_request/security_solution.yml
+++ b/.buildkite/pipelines/pull_request/security_solution.yml
@ -1,11 +1,14 @@
 steps:
  - command: .buildkite/scripts/steps/functional/security_solution.sh
-    label: 'Security Solution Tests'
+    label: 'Security Solution Cypress Tests'
    agents:
-      queue: ci-group-6
+      queue: n2-4-spot
    depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
+    parallelism: 10
    retry:
      automatic:
        - exit_status: '*'
          limit: 1
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"
--- a/.buildkite/scripts/steps/functional/common_cypress.sh
+++ b/.buildkite/scripts/steps/functional/common_cypress.sh
@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+Xvfb -screen 0 1680x946x24 :99 &
+export DISPLAY=:99
--- a/.buildkite/scripts/steps/functional/response_ops.sh
+++ b/.buildkite/scripts/steps/functional/response_ops.sh
@ -3,15 +3,11 @@
 set -euo pipefail

 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh

 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

 echo "--- Response Ops Cypress Tests on Security Solution"

-cd "$XPACK_DIR"
-
-checks-reporter-with-killswitch "Response Ops Cypress Tests on Security Solution" \
- node scripts/functional_tests \
-   --debug --bail \
-   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-   --config test/security_solution_cypress/response_ops_cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run:respops
--- a/.buildkite/scripts/steps/functional/security_solution.sh
+++ b/.buildkite/scripts/steps/functional/security_solution.sh
@ -3,15 +3,11 @@
 set -euo pipefail

 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh

 export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

-echo "--- Security Solution tests (Chrome)"
+echo "--- Security Solution Cypress tests (Chrome)"

-cd "$XPACK_DIR"
-
-checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome)" \
- node scripts/functional_tests \
-   --debug --bail \
-   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-   --config test/security_solution_cypress/cli_config.ts
+yarn --cwd x-pack/plugins/security_solution cypress:run
--- a/package.json
+++ b/package.json
@ -201,6 +201,7 @@
    "chokidar": "^3.4.3",
    "chroma-js": "^1.4.1",
    "classnames": "2.2.6",
+    "cli-table3": "^0.6.3",
    "color": "1.0.3",
    "commander": "^3.0.2",
    "compare-versions": "3.5.1",
@ -439,7 +440,7 @@
    "@bazel/ibazel": "^0.16.2",
    "@bazel/typescript": "^3.8.0",
    "@cypress/snapshot": "^2.1.7",
-    "@cypress/webpack-preprocessor": "^5.6.0",
+    "@cypress/webpack-preprocessor": "^5.12.2",
    "@elastic/eslint-plugin-eui": "0.0.2",
    "@elastic/github-checks-reporter": "0.0.20b3",
    "@elastic/makelogs": "^6.1.1",
@ -513,7 +514,6 @@
    "@types/cmd-shim": "^2.0.0",
    "@types/color": "^3.0.0",
    "@types/compression-webpack-plugin": "^2.0.2",
-    "@types/cypress-cucumber-preprocessor": "^1.14.1",
    "@types/cytoscape": "^3.14.0",
    "@types/d3": "^3.5.43",
    "@types/d3-array": "^1.2.7",
@ -679,12 +679,12 @@
    "cpy": "^8.1.1",
    "css-loader": "^3.4.2",
    "cssnano": "5.0.0",
-    "cypress": "^8.5.0",
-    "cypress-axe": "^0.13.0",
+    "cypress": "12.13.0",
+    "cypress-axe": "^1.4.0",
    "cypress-file-upload": "^5.0.8",
-    "cypress-multi-reporters": "^1.5.0",
+    "cypress-multi-reporters": "^1.6.3",
    "cypress-pipe": "^2.0.0",
-    "cypress-real-events": "^1.5.1",
+    "cypress-real-events": "^1.10.0",
    "debug": "^2.6.9",
    "delete-empty": "^2.0.0",
    "dependency-check": "^4.1.0",
--- a/packages/kbn-dev-utils/src/proc_runner/proc.ts
+++ b/packages/kbn-dev-utils/src/proc_runner/proc.ts
@ -6,8 +6,10 @@
 * Side Public License, v 1.
 */

+import Fs from 'fs';
+import Path from 'path';
 import execa from 'execa';
-import { statSync } from 'fs';
+import stripAnsi from 'strip-ansi';

 import * as Rx from 'rxjs';
 import { tap, share, take, mergeMap, map, ignoreElements } from 'rxjs/operators';
@ -30,6 +32,7 @@ export interface ProcOptions {
  cwd: string;
  env?: Record<string, string | undefined>;
  stdin?: string;
+  writeLogsToPath?: string;
 }

 async function withTimeout(
@ -57,13 +60,21 @@ export type Proc = ReturnType<typeof startProc>;
 export function startProc(name: string, options: ProcOptions, log: ToolingLog) {
  const { cmd, args, cwd, env, stdin } = options;

-  log.info('[%s] > %s', name, cmd === process.execPath ? 'node' : cmd, args.join(' '));
+  let stdioTarget: undefined | NodeJS.WritableStream;
+  if (!options.writeLogsToPath) {
+    log.info('starting [%s] > %s', name, cmd === process.execPath ? 'node' : cmd, args.join(' '));
+  } else {
+    stdioTarget = Fs.createWriteStream(options.writeLogsToPath, 'utf8');
+    const exec = cmd === process.execPath ? 'node' : cmd;
+    const relOut = Path.relative(process.cwd(), options.writeLogsToPath);
+    log.info(`starting [${name}] and writing output to ${relOut} > ${exec} ${args.join(' ')}`);
+  }

  // spawn fails with ENOENT when either the
  // cmd or cwd don't exist, so we check for the cwd
  // ahead of time so that the error is less ambiguous
  try {
-    if (!statSync(cwd).isDirectory()) {
+    if (!Fs.statSync(cwd).isDirectory()) {
      throw new Error(`cwd "${cwd}" exists but is not a directory`);
    }
  } catch (err) {
@ -117,7 +128,20 @@ export function startProc(name: string, options: ProcOptions, log: ToolingLog) {
    observeLines(childProcess.stdout!), // TypeScript note: As long as the proc stdio[1] is 'pipe', then stdout will not be null
    observeLines(childProcess.stderr!) // TypeScript note: As long as the proc stdio[1] is 'pipe', then stderr will not be null
  ).pipe(
-    tap((line) => log.write(` ${chalk.gray('proc')} [${chalk.gray(name)}] ${line}`)),
+    tap({
+      next(line) {
+        if (stdioTarget) {
+          stdioTarget.write(stripAnsi(line) + '\n');
+        } else {
+          log.write(` ${chalk.gray('proc')} [${chalk.gray(name)}] ${line}`);
+        }
+      },
+      complete() {
+        if (stdioTarget) {
+          stdioTarget.end();
+        }
+      },
+    }),
    share()
  );

--- a/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts
+++ b/packages/kbn-dev-utils/src/proc_runner/proc_runner.ts
@ -37,12 +37,12 @@ export class ProcRunner {
  private procs: Proc[] = [];
  private signalUnsubscribe: () => void;

-  constructor(private log: ToolingLog) {
+  constructor(private readonly log: ToolingLog) {
    this.log = log.withType('ProcRunner');

    this.signalUnsubscribe = exitHook(() => {
      this.teardown().catch((error) => {
-        log.error(`ProcRunner teardown error: ${error.stack}`);
+        this.log.error(`ProcRunner teardown error: ${error.stack}`);
      });
    });
  }
@ -59,6 +59,7 @@ export class ProcRunner {
      waitTimeout = 15 * MINUTE,
      env = process.env,
      onEarlyExit,
+      writeLogsToPath,
    } = options;
    const cmd = options.cmd === 'node' ? process.execPath : options.cmd;

@ -80,6 +81,7 @@ export class ProcRunner {
      cwd,
      env,
      stdin,
+      writeLogsToPath,
    });

    if (onEarlyExit) {
--- a/packages/kbn-dev-utils/src/run/flags.ts
+++ b/packages/kbn-dev-utils/src/run/flags.ts
@ -53,6 +53,10 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
  };
 }

+export const DEFAULT_FLAG_ALIASES = {
+  v: 'verbose',
+};
+
 export function getFlags(
  argv: string[],
  flagOptions: RunOptions['flags'] = {},
@ -67,7 +71,7 @@ export function getFlags(
    boolean: [...(flagOptions.boolean || []), ...logLevelFlags, 'help'],
    alias: {
      ...flagOptions.alias,
-      v: 'verbose',
+      ...DEFAULT_FLAG_ALIASES,
    },
    default: flagOptions.default,
    unknown: (name: string) => {
--- a/packages/kbn-dev-utils/src/run/flags_reader.test.ts
+++ b/packages/kbn-dev-utils/src/run/flags_reader.test.ts
@ -0,0 +1,344 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+
+import { getFlags } from './flags';
+import { FlagsReader } from './flags_reader';
+
+const FLAGS = {
+  string: 'string',
+  astring: ['foo', 'bar'],
+  num: '1234',
+  bool: true,
+  missing: undefined,
+};
+
+const basic = new FlagsReader(FLAGS);
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
+
+describe('#string()', () => {
+  it('returns a single string, regardless of flag count', () => {
+    expect(basic.string('string')).toMatchInlineSnapshot(`"string"`);
+    expect(basic.string('astring')).toBe(FLAGS.astring.at(-1));
+  });
+
+  it('returns undefined when flag is missing', () => {
+    expect(basic.string('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  it('throws for non-string flags', () => {
+    expect(() => basic.string('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws when flag is missing', () => {
+      expect(() => basic.requiredString('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#arrayOfStrings()', () => {
+  it('returns an array of strings for string flags, regardless of count', () => {
+    expect(basic.arrayOfStrings('string')).toMatchInlineSnapshot(`
+      Array [
+        "string",
+      ]
+    `);
+    expect(basic.arrayOfStrings('astring')).toMatchInlineSnapshot(`
+      Array [
+        "foo",
+        "bar",
+      ]
+    `);
+  });
+
+  it('returns undefined when flag is missing', () => {
+    expect(basic.arrayOfStrings('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  it('throws for non-string flags', () => {
+    expect(() => basic.arrayOfStrings('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws when flag is missing', () => {
+      expect(() => basic.requiredArrayOfStrings('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#enum()', () => {
+  it('validates that values match options', () => {
+    expect(basic.enum('string', ['a', 'string', 'b'])).toMatchInlineSnapshot(`"string"`);
+    expect(basic.enum('missing', ['a', 'b'])).toMatchInlineSnapshot(`undefined`);
+    expect(() => basic.enum('string', ['a', 'b'])).toThrowErrorMatchingInlineSnapshot(
+      `"invalid --string, expected one of \\"a\\", \\"b\\""`
+    );
+  });
+});
+
+describe('#path()', () => {
+  it('parses the string to an absolute path based on CWD', () => {
+    expect(basic.path('string')).toMatchInlineSnapshot(`<absolute path>/string`);
+    expect(basic.path('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  describe('required version', () => {
+    it('throws if the flag is missing', () => {
+      expect(() => basic.requiredPath('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+
+  describe('array version', () => {
+    it('parses a list of paths', () => {
+      expect(basic.arrayOfPaths('astring')).toMatchInlineSnapshot(`
+        Array [
+          <absolute path>/foo,
+          <absolute path>/bar,
+        ]
+      `);
+    });
+
+    describe('required version', () => {
+      it('throws if the flag is missing', () => {
+        expect(() => basic.requiredArrayOfPaths('missing')).toThrowErrorMatchingInlineSnapshot(
+          `"missing required flag --missing"`
+        );
+      });
+    });
+  });
+});
+
+describe('#number()', () => {
+  it('parses strings as numbers', () => {
+    expect(basic.number('num')).toMatchInlineSnapshot(`1234`);
+    expect(basic.number('missing')).toMatchInlineSnapshot(`undefined`);
+    expect(() => basic.number('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+    expect(() => basic.number('string')).toThrowErrorMatchingInlineSnapshot(
+      `"unable to parse --string value [string] as a number"`
+    );
+    expect(() => basic.number('astring')).toThrowErrorMatchingInlineSnapshot(
+      `"unable to parse --astring value [bar] as a number"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws if the flag is missing', () => {
+      expect(() => basic.requiredNumber('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#boolean()', () => {
+  it('ensures flag is boolean, requires value', () => {
+    expect(basic.boolean('bool')).toMatchInlineSnapshot(`true`);
+    expect(() => basic.boolean('missing')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --missing to be a boolean"`
+    );
+    expect(() => basic.boolean('string')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --string to be a boolean"`
+    );
+    expect(() => basic.boolean('astring')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --astring to be a boolean"`
+    );
+  });
+});
+
+describe('#getPositionals()', () => {
+  it('returns all positional arguments in flags', () => {
+    const flags = new FlagsReader({
+      ...FLAGS,
+      _: ['a', 'b', 'c'],
+    });
+
+    expect(flags.getPositionals()).toMatchInlineSnapshot(`
+      Array [
+        "a",
+        "b",
+        "c",
+      ]
+    `);
+  });
+
+  it('handles missing _ flag', () => {
+    const flags = new FlagsReader({});
+    expect(flags.getPositionals()).toMatchInlineSnapshot(`Array []`);
+  });
+});
+
+describe('#getUnused()', () => {
+  it('returns a map of all unused flags', () => {
+    const flags = new FlagsReader({
+      a: '1',
+      b: '2',
+      c: '3',
+    });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+        "b" => "2",
+        "c" => "3",
+      }
+    `);
+
+    flags.number('a');
+    flags.number('b');
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "c" => "3",
+      }
+    `);
+  });
+
+  it('ignores the default flags which are forced on commands', () => {
+    const rawFlags = getFlags(['--a=1'], {
+      string: ['a'],
+    });
+
+    const flags = new FlagsReader(rawFlags, {
+      aliases: {
+        v: 'verbose',
+      },
+    });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+      }
+    `);
+    flags.number('a');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+
+  it('treats aliased flags as used', () => {
+    const flags = new FlagsReader(
+      {
+        f: true,
+        force: true,
+        v: true,
+        verbose: true,
+      },
+      {
+        aliases: {
+          f: 'force',
+          v: 'verbose',
+        },
+      }
+    );
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "f" => true,
+        "force" => true,
+      }
+    `);
+    flags.boolean('force');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+    flags.boolean('v');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+
+  it('treats failed reads as "uses"', () => {
+    const flags = new FlagsReader({ a: 'b' });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "b",
+      }
+    `);
+    expect(() => flags.number('a')).toThrowError();
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+});
+
+describe('#getUsed()', () => {
+  it('returns a map of all used flags', () => {
+    const flags = new FlagsReader({
+      a: '1',
+      b: '2',
+      c: '3',
+    });
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+
+    flags.number('a');
+    flags.number('b');
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+        "b" => "2",
+      }
+    `);
+  });
+
+  it('treats aliases flags as used', () => {
+    const flags = new FlagsReader(
+      {
+        f: true,
+        force: true,
+        v: true,
+        verbose: true,
+      },
+      {
+        aliases: {
+          f: 'force',
+          v: 'verbose',
+        },
+      }
+    );
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+    flags.boolean('force');
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "force" => true,
+        "f" => true,
+      }
+    `);
+    flags.boolean('v');
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "force" => true,
+        "f" => true,
+        "v" => true,
+        "verbose" => true,
+      }
+    `);
+  });
+
+  it('treats failed reads as "uses"', () => {
+    const flags = new FlagsReader({ a: 'b' });
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+    expect(() => flags.number('a')).toThrowError();
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "b",
+      }
+    `);
+  });
+});
--- a/packages/kbn-dev-utils/src/run/flags_reader.ts
+++ b/packages/kbn-dev-utils/src/run/flags_reader.ts
@ -0,0 +1,267 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { createFlagError } from './fail';
+import { LOG_LEVEL_FLAGS } from '../tooling_log';
+
+type FlagValue = string | string[] | boolean;
+const FORCED_FLAGS = new Set([...LOG_LEVEL_FLAGS.map((l) => l.name), 'help']);
+
+const makeAbsolute = (rel: string) => Path.resolve(process.cwd(), rel);
+
+const nonUndefinedValues = (e: [string, FlagValue | undefined]): e is [string, FlagValue] =>
+  e[1] !== undefined;
+
+export class FlagsReader {
+  private readonly used: Map<string, FlagValue>;
+  private readonly unused: Map<string, FlagValue>;
+  private readonly _: string[];
+  private readonly aliasMap: Map<string, string>;
+
+  constructor(
+    flags: Record<string, FlagValue | undefined>,
+    private readonly opts?: { aliases?: Record<string, string> }
+  ) {
+    this.used = new Map();
+    this.unused = new Map(
+      Object.entries(flags)
+        .filter(nonUndefinedValues)
+        .filter((e) => e[0] !== 'unexpected')
+    );
+    this.aliasMap = new Map(
+      Object.entries(this.opts?.aliases ?? []).flatMap(([a, b]) => [
+        [a, b],
+        [b, a],
+      ])
+    );
+
+    this._ = this.arrayOfStrings('_') ?? [];
+  }
+
+  private use(key: string) {
+    const alias = this.aliasMap.get(key);
+
+    const used = this.used.get(key);
+    if (used !== undefined) {
+      return used;
+    }
+
+    const unused = this.unused.get(key);
+    if (unused !== undefined) {
+      this.used.set(key, unused);
+      this.unused.delete(key);
+
+      if (alias !== undefined) {
+        this.used.set(alias, unused);
+        this.unused.delete(alias);
+      }
+    }
+
+    return unused;
+  }
+
+  /**
+   * Read a string flag that supports multiple instances into an array of strings. If the
+   * flag is only passed once an array with a single item will be returned. If the flag is not
+   * passed then undefined will be returned.
+   */
+  arrayOfStrings(key: string) {
+    const value = this.use(key);
+
+    switch (typeof value) {
+      case 'boolean':
+        throw createFlagError(`expected --${key} to be a string`);
+      case 'string':
+        return value ? [value] : [];
+      default:
+        return value;
+    }
+  }
+
+  /**
+   * Same as #arrayOfStrings() except when the flag is not passed a "flag error" is thrown telling
+   * the user that the flag is required and shows them the help text.
+   */
+  requiredArrayOfStrings(key: string) {
+    const value = this.arrayOfStrings(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Read the value of a string flag. If the flag is passed multiple times the last value is returned. If
+   * the flag is not passed then undefined is returned.
+   */
+  string(key: string) {
+    const value = this.use(key);
+
+    switch (typeof value) {
+      case 'undefined':
+        return undefined;
+      case 'string':
+        return value || undefined; // convert "" to undefined
+      case 'object':
+        const last = value.at(-1);
+        if (last === undefined) {
+          throw createFlagError(`expected --${key} to be a string`);
+        }
+        return last || undefined; // convert "" to undefined
+      default:
+        throw createFlagError(`expected --${key} to be a string`);
+    }
+  }
+
+  /**
+   * Same as #string() except when the flag is passed it is validated against a list
+   * of valid values
+   */
+  enum<T extends string>(key: string, values: readonly T[]) {
+    const value = this.string(key);
+    if (value === undefined) {
+      return;
+    }
+
+    if (values.includes(value as T)) {
+      return value as T;
+    }
+
+    throw createFlagError(`invalid --${key}, expected one of "${values.join('", "')}"`);
+  }
+
+  /**
+   * Same as #string() except when a flag is not passed a "flag error" is thrown telling the user
+   * that the flag is required and shows them the help text.
+   */
+  requiredString(key: string) {
+    const value = this.string(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Same as #string(), except that when there is a value for the string it is resolved to an
+   * absolute path based on the current working directory
+   */
+  path(key: string) {
+    const value = this.string(key);
+    if (value !== undefined) {
+      return makeAbsolute(value);
+    }
+  }
+
+  /**
+   * Same as #requiredString() except that values are converted to absolute paths based on the
+   * current working directory
+   */
+  requiredPath(key: string) {
+    return makeAbsolute(this.requiredString(key));
+  }
+
+  /**
+   * Same as #arrayOfStrings(), except that when there are values they are resolved to
+   * absolute paths based on the current working directory
+   */
+  arrayOfPaths(key: string) {
+    const value = this.arrayOfStrings(key);
+    if (value !== undefined) {
+      return value.map(makeAbsolute);
+    }
+  }
+
+  /**
+   * Same as #requiredArrayOfStrings(), except that values are resolved to absolute paths
+   * based on the current working directory
+   */
+  requiredArrayOfPaths(key: string) {
+    return this.requiredArrayOfStrings(key).map(makeAbsolute);
+  }
+
+  /**
+   * Parsed the provided flag as a number, if the value does not parse to a valid number
+   * using Number.parseFloat() then a "flag error" is thrown. If the flag is not passed
+   * undefined is returned.
+   */
+  number(key: string) {
+    const value = this.string(key);
+    if (value === undefined) {
+      return;
+    }
+
+    const num = Number.parseFloat(value);
+    if (Number.isNaN(num)) {
+      throw createFlagError(`unable to parse --${key} value [${value}] as a number`);
+    }
+
+    return num;
+  }
+
+  /**
+   * Same as #number() except that when the flag is missing a "flag error" is thrown
+   */
+  requiredNumber(key: string) {
+    const value = this.number(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Read a boolean flag value, if the flag is properly defined as a "boolean" in the run options
+   * then the value will always be a boolean, defaulting to `false`, so there is no need for an
+   * optional/requiredBoolean() method.
+   */
+  boolean(key: string) {
+    const value = this.use(key);
+    if (typeof value !== 'boolean') {
+      throw createFlagError(`expected --${key} to be a boolean`);
+    }
+    return value;
+  }
+
+  /**
+   * Get the positional arguments passed, includes any values that are not associated with
+   * a specific --flag
+   */
+  getPositionals() {
+    return this._.slice(0);
+  }
+
+  /**
+   * Returns all of the unused flags. When a flag is read via any of the key-specific methods
+   * the key is marked as "used" and this method will return a map of just the flags which
+   * have not been used yet (excluding the default flags like --debug, --verbose, and --help)
+   */
+  getUnused() {
+    return new Map(
+      [...this.unused.entries()].filter(([key]) => {
+        const alias = this.aliasMap.get(key);
+        if (alias !== undefined && FORCED_FLAGS.has(alias)) {
+          return false;
+        }
+
+        return !FORCED_FLAGS.has(key);
+      })
+    );
+  }
+
+  /**
+   * Returns all of the used flags. When a flag is read via any of the key-specific methods
+   * the key is marked as "used" and from then on this method will return a map including that
+   * and any other key used by these methods.
+   */
+  getUsed() {
+    return new Map(this.used);
+  }
+}
--- a/packages/kbn-dev-utils/src/run/run.ts
+++ b/packages/kbn-dev-utils/src/run/run.ts
@ -8,11 +8,12 @@

 import { pickLevelFromFlags, ToolingLog, LogLevel } from '../tooling_log';
 import { createFlagError } from './fail';
-import { Flags, getFlags, FlagOptions } from './flags';
+import { Flags, getFlags, FlagOptions, DEFAULT_FLAG_ALIASES } from './flags';
 import { ProcRunner, withProcRunner } from '../proc_runner';
 import { getHelp } from './help';
 import { CleanupTask, Cleanup } from './cleanup';
 import { Metrics, MetricsMeta } from './metrics';
+import { FlagsReader } from './flags_reader';

 export interface RunContext {
  log: ToolingLog;
@ -20,6 +21,7 @@ export interface RunContext {
  procRunner: ProcRunner;
  statsMeta: MetricsMeta;
  addCleanupTask: (task: CleanupTask) => void;
+  flagsReader: FlagsReader;
 }
 export type RunFn = (context: RunContext) => Promise<void> | void;

@ -70,6 +72,12 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
        procRunner,
        statsMeta: metrics.meta,
        addCleanupTask: cleanup.add.bind(cleanup),
+        flagsReader: new FlagsReader(flags, {
+          aliases: {
+            ...options.flags?.alias,
+            ...DEFAULT_FLAG_ALIASES,
+          },
+        }),
      });
    });
  } catch (error) {
--- a/packages/kbn-dev-utils/src/run/run_with_commands.test.ts
+++ b/packages/kbn-dev-utils/src/run/run_with_commands.test.ts
@ -9,6 +9,7 @@
 import { RunWithCommands } from './run_with_commands';
 import { ToolingLog, ToolingLogCollectingWriter } from '../tooling_log';
 import { ProcRunner } from '../proc_runner';
+import { FlagsReader } from './flags_reader';
 jest.mock('./metrics');

 const testLog = new ToolingLog();
@ -44,6 +45,7 @@ it('extends the context using extendContext()', async () => {
  expect(context).toEqual({
    log: expect.any(ToolingLog),
    flags: expect.any(Object),
+    flagsReader: expect.any(FlagsReader),
    addCleanupTask: expect.any(Function),
    procRunner: expect.any(ProcRunner),
    statsMeta: undefined,
--- a/packages/kbn-dev-utils/src/run/run_with_commands.ts
+++ b/packages/kbn-dev-utils/src/run/run_with_commands.ts
@ -8,7 +8,8 @@

 import { ToolingLog, pickLevelFromFlags } from '../tooling_log';
 import { RunContext, RunOptions } from './run';
-import { getFlags, FlagOptions, mergeFlagOptions } from './flags';
+import { getFlags, FlagOptions, mergeFlagOptions, DEFAULT_FLAG_ALIASES } from './flags';
+import { FlagsReader } from './flags_reader';
 import { Cleanup } from './cleanup';
 import { getHelpForAllCommands, getCommandLevelHelp } from './help';
 import { createFlagError } from './fail';
@ -115,6 +116,12 @@ export class RunWithCommands<T> {
          procRunner,
          statsMeta: metrics.meta,
          addCleanupTask: cleanup.add.bind(cleanup),
+          flagsReader: new FlagsReader(commandFlags, {
+            aliases: {
+              ...commandFlagOptions.alias,
+              ...DEFAULT_FLAG_ALIASES,
+            },
+          }),
        };

        const extendedContext = {
--- a/packages/kbn-dev-utils/src/tooling_log/index.ts
+++ b/packages/kbn-dev-utils/src/tooling_log/index.ts
@ -11,7 +11,7 @@ export type { ToolingLogOptions } from './tooling_log';
 export type { ToolingLogTextWriterConfig } from './tooling_log_text_writer';
 export { ToolingLogTextWriter } from './tooling_log_text_writer';
 export type { LogLevel, ParsedLogLevel } from './log_levels';
-export { pickLevelFromFlags, parseLogLevel } from './log_levels';
+export { pickLevelFromFlags, parseLogLevel, LOG_LEVEL_FLAGS } from './log_levels';
 export { ToolingLogCollectingWriter } from './tooling_log_collecting_writer';
 export type { Writer } from './writer';
 export type { Message } from './message';
--- a/packages/kbn-pm/dist/index.js
+++ b/packages/kbn-pm/dist/index.js
@ -533,6 +533,12 @@ module.exports = require("path");
 Object.defineProperty(exports, "__esModule", {
  value: true
 });
+Object.defineProperty(exports, "LOG_LEVEL_FLAGS", {
+  enumerable: true,
+  get: function () {
+    return _log_levels.LOG_LEVEL_FLAGS;
+  }
+});
 Object.defineProperty(exports, "ToolingLog", {
  enumerable: true,
  get: function () {
--- a/packages/kbn-test/BUILD.bazel
+++ b/packages/kbn-test/BUILD.bazel
@ -54,6 +54,7 @@ RUNTIME_DEPS = [
    "@npm//execa",
    "@npm//exit-hook",
    "@npm//form-data",
+    "@npm//get-port",
    "@npm//globby",
    "@npm//he",
    "@npm//history",
@ -62,6 +63,7 @@ RUNTIME_DEPS = [
    "@npm//jest-snapshot",
    "@npm//jest-styled-components",
    "@npm//joi",
+    "@npm//js-yaml",
    "@npm//mustache",
    "@npm//prettier",
    "@npm//react-dom",
@ -88,6 +90,7 @@ TYPES_DEPS = [
  "@npm//elastic-apm-node",
  "@npm//del",
  "@npm//form-data",
+  "@npm//get-port",
  "@npm//globby",
  "@npm//jest",
  "@npm//jest-cli",
@ -103,6 +106,7 @@ TYPES_DEPS = [
  "@npm//@types/jest",
  "@npm//joi",
  "@npm//@types/lodash",
+  "@npm//@types/js-yaml",
  "@npm//@types/mustache",
  "@npm//@types/node",
  "@npm//@types/prettier",
--- a/packages/kbn-test/src/failed_tests_reporter/fixtures/cypress_report.xml
+++ b/packages/kbn-test/src/failed_tests_reporter/fixtures/cypress_report.xml
@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites name="Mocha Tests" time="16.198" tests="2" failures="1">
-  <testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0">
+  <testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/e2e/timeline_flyout_button.cy.ts" failures="0" time="0">
  </testsuite>
  <testsuite name="timeline flyout button" timestamp="2020-07-22T15:06:26" tests="2" failures="1" time="16.198">
    <testcase name="timeline flyout button toggles open the timeline" time="8.099" classname="toggles open the timeline">
--- a/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts
+++ b/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts
@ -282,9 +282,9 @@ it('rewrites cypress reports with minimal changes', async () => {
    -‹?xml version="1.0" encoding="UTF-8"?›
    +‹?xml version="1.0" encoding="utf-8"?›
     ‹testsuites name="Mocha Tests" time="16.198" tests="2" failures="1"›
-    -  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0"›
+    -  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/e2e/timeline_flyout_button.cy.ts" failures="0" time="0"›
    -  ‹/testsuite›
-    +  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0"/›
+    +  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/e2e/timeline_flyout_button.cy.ts" failures="0" time="0"/›
       ‹testsuite name="timeline flyout button" timestamp="2020-07-22T15:06:26" tests="2" failures="1" time="16.198"›
    -    ‹testcase name="timeline flyout button toggles open the timeline" time="8.099" classname="toggles open the timeline"›
    -    ‹/testcase›
--- a/packages/kbn-test/src/functional_test_runner/cli.ts
+++ b/packages/kbn-test/src/functional_test_runner/cli.ts
@ -9,23 +9,12 @@
 import { resolve } from 'path';
 import { inspect } from 'util';

-import { run, createFlagError, Flags, ToolingLog, getTimeReporter } from '@kbn/dev-utils';
+import { run, createFlagError, ToolingLog, getTimeReporter } from '@kbn/dev-utils';
 import exitHook from 'exit-hook';

+import { readConfigFile, EsVersion } from './lib';
 import { FunctionalTestRunner } from './functional_test_runner';

-const makeAbsolutePath = (v: string) => resolve(process.cwd(), v);
-const toArray = (v: string | string[]) => ([] as string[]).concat(v || []);
-const parseInstallDir = (flags: Flags) => {
-  const flag = flags['kibana-install-dir'];
-
-  if (typeof flag !== 'string' && flag !== undefined) {
-    throw createFlagError('--kibana-install-dir must be a string or not defined');
-  }
-
-  return flag ? makeAbsolutePath(flag) : undefined;
-};
-
 export function runFtrCli() {
  const runStartTime = Date.now();
  const toolingLog = new ToolingLog({
@ -34,38 +23,50 @@ export function runFtrCli() {
  });
  const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner');
  run(
-    async ({ flags, log }) => {
-      const esVersion = flags['es-version'] || undefined; // convert "" to undefined
-      if (esVersion !== undefined && typeof esVersion !== 'string') {
-        throw createFlagError('expected --es-version to be a string');
+    async ({ flagsReader, log }) => {
+      const esVersionInput = flagsReader.string('es-version');
+
+      const configPaths = [...(flagsReader.arrayOfStrings('config') ?? [])].map((rel) =>
+        resolve(rel)
+      );
+      if (configPaths.length !== 1) {
+        throw createFlagError(`Expected there to be exactly one --config flag`);
      }

-      const functionalTestRunner = new FunctionalTestRunner(
-        log,
-        makeAbsolutePath(flags.config as string),
-        {
-          mochaOpts: {
-            bail: flags.bail,
-            dryRun: flags['dry-run'],
-            grep: flags.grep || undefined,
-            invert: flags.invert,
-          },
-          kbnTestServer: {
-            installDir: parseInstallDir(flags),
-          },
-          suiteFiles: {
-            include: toArray(flags.include as string | string[]).map(makeAbsolutePath),
-            exclude: toArray(flags.exclude as string | string[]).map(makeAbsolutePath),
-          },
-          suiteTags: {
-            include: toArray(flags['include-tag'] as string | string[]),
-            exclude: toArray(flags['exclude-tag'] as string | string[]),
-          },
-          updateBaselines: flags.updateBaselines || flags.u,
-          updateSnapshots: flags.updateSnapshots || flags.u,
+      const esVersion = esVersionInput ? new EsVersion(esVersionInput) : EsVersion.getDefault();
+      const settingOverrides = {
+        mochaOpts: {
+          bail: flagsReader.boolean('bail'),
+          dryRun: flagsReader.boolean('dry-run'),
+          grep: flagsReader.string('grep'),
+          invert: flagsReader.boolean('invert'),
        },
-        esVersion
-      );
+        kbnTestServer: {
+          installDir: flagsReader.path('kibana-install-dir'),
+        },
+        suiteFiles: {
+          include: flagsReader.arrayOfPaths('include') ?? [],
+          exclude: flagsReader.arrayOfPaths('exclude') ?? [],
+        },
+        suiteTags: {
+          include: flagsReader.arrayOfStrings('include-tag') ?? [],
+          exclude: flagsReader.arrayOfStrings('exclude-tag') ?? [],
+        },
+        updateBaselines: flagsReader.boolean('updateBaselines') || flagsReader.boolean('u'),
+        updateSnapshots: flagsReader.boolean('updateSnapshots') || flagsReader.boolean('u'),
+      };
+
+      const config = await readConfigFile(log, esVersion, configPaths[0], settingOverrides);
+
+      const functionalTestRunner = new FunctionalTestRunner(log, config, esVersion);
+
+      if (flagsReader.boolean('throttle')) {
+        process.env.TEST_THROTTLE_NETWORK = '1';
+      }
+
+      if (flagsReader.boolean('headless')) {
+        process.env.TEST_BROWSER_HEADLESS = '1';
+      }

      let teardownRun = false;
      const teardown = async (err?: Error) => {
@ -76,7 +77,7 @@ export function runFtrCli() {
          await reportTime(runStartTime, 'total', {
            success: false,
            err: err.message,
-            ...flags,
+            ...Object.fromEntries(flagsReader.getUsed().entries()),
          });
          log.indent(-log.getIndent());
          log.error(err);
@ -84,15 +85,11 @@ export function runFtrCli() {
        } else {
          await reportTime(runStartTime, 'total', {
            success: true,
-            ...flags,
+            ...Object.fromEntries(flagsReader.getUsed().entries()),
          });
        }

-        try {
-          await functionalTestRunner.close();
-        } finally {
-          process.exit();
-        }
+        process.exit();
      };

      process.on('unhandledRejection', (err) =>
@ -103,7 +100,7 @@ export function runFtrCli() {
      exitHook(teardown);

      try {
-        if (flags['test-stats']) {
+        if (flagsReader.boolean('test-stats')) {
          process.stderr.write(
            JSON.stringify(await functionalTestRunner.getTestStats(), null, 2) + '\n'
          );
@ -139,11 +136,10 @@ export function runFtrCli() {
          'updateBaselines',
          'updateSnapshots',
          'u',
+          'throttle',
+          'headless',
          'dry-run',
        ],
-        default: {
-          config: 'test/functional/config.js',
-        },
        help: `
          --config=path      path to a config file
          --bail             stop tests after the first failure
@ -164,6 +160,8 @@ export function runFtrCli() {
          --updateSnapshots  replace inline and file snapshots with whatever is generated from the test
          -u                 replace both baseline screenshots and snapshots
          --kibana-install-dir  directory where the Kibana install being tested resides
+          --throttle         enable network throttling in Chrome browser
+          --headless         run browser in headless mode
          --dry-run          report tests without executing them
        `,
      },
--- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts
+++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts
@ -14,10 +14,8 @@ import { REPO_ROOT } from '@kbn/utils';
 import { Suite, Test } from './fake_mocha_types';
 import {
  Lifecycle,
-  LifecyclePhase,
-  TestMetadata,
-  readConfigFile,
  ProviderCollection,
+  Providers,
  readProviderSpec,
  setupMocha,
  runTests,
@ -25,27 +23,17 @@ import {
  Config,
  SuiteTracker,
  EsVersion,
+  TestMetadata,
 } from './lib';
 import { createEsClientForFtrConfig } from '../es';

 export class FunctionalTestRunner {
-  public readonly lifecycle = new Lifecycle();
-  public readonly testMetadata = new TestMetadata(this.lifecycle);
-  private closed = false;
-
  private readonly esVersion: EsVersion;
  constructor(
    private readonly log: ToolingLog,
-    private readonly configFile: string,
-    private readonly configOverrides: any,
+    private readonly config: Config,
    esVersion?: string | EsVersion
  ) {
-    for (const [key, value] of Object.entries(this.lifecycle)) {
-      if (value instanceof LifecyclePhase) {
-        value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-        value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-      }
-    }
    this.esVersion =
      esVersion === undefined
        ? EsVersion.getDefault()
@ -55,21 +43,30 @@ export class FunctionalTestRunner {
  }

  async run(abortSignal?: AbortSignal) {
-    return await this._run(async (config, coreProviders) => {
-      SuiteTracker.startTracking(this.lifecycle, this.configFile);
+    const testStats = await this.getTestStats();

-      const providers = new ProviderCollection(this.log, [
-        ...coreProviders,
-        ...readProviderSpec('Service', config.get('services')),
-        ...readProviderSpec('PageObject', config.get('pageObjects')),
-      ]);
+    return await this.runHarness(async (lifecycle, coreProviders) => {
+      SuiteTracker.startTracking(lifecycle, this.config.path);

-      if (providers.hasService('es')) {
-        await this.validateEsVersion(config);
+      const realServices =
+        !testStats || (testStats.testCount > 0 && testStats.nonSkippedTestCount > 0);
+
+      const providers = realServices
+        ? new ProviderCollection(this.log, [
+            ...coreProviders,
+            ...readProviderSpec('Service', this.config.get('services')),
+            ...readProviderSpec('PageObject', this.config.get('pageObjects')),
+          ])
+        : this.getStubProviderCollection(coreProviders);
+
+      if (realServices) {
+        if (providers.hasService('es')) {
+          await this.validateEsVersion();
+        }
+        await providers.loadAll();
      }
-      await providers.loadAll();

-      const customTestRunner = config.get('testRunner');
+      const customTestRunner = this.config.get('testRunner');
      if (customTestRunner) {
        this.log.warning(
          'custom test runner defined, ignoring all mocha/suite/filtering related options'
@ -79,7 +76,7 @@ export class FunctionalTestRunner {

      let reporter;
      let reporterOptions;
-      if (config.get('mochaOpts.dryRun')) {
+      if (this.config.get('mochaOpts.dryRun')) {
        // override default reporter for dryRun results
        const targetFile = Path.resolve(REPO_ROOT, 'target/functional-tests/dryRunOutput.json');
        reporter = 'json';
@ -89,43 +86,43 @@ export class FunctionalTestRunner {
        this.log.info(`Dry run results will be stored in ${targetFile}`);
      }

-      const mocha = await setupMocha(
-        this.lifecycle,
-        this.log,
-        config,
+      const mocha = await setupMocha({
+        lifecycle,
+        log: this.log,
+        config: this.config,
        providers,
-        this.esVersion,
+        esVersion: this.esVersion,
        reporter,
-        reporterOptions
-      );
-      if (abortSignal?.aborted) {
-        this.log.warning('run aborted');
-        return;
-      }
+        reporterOptions,
+      });

      // there's a bug in mocha's dry run, see https://github.com/mochajs/mocha/issues/4838
      // until we can update to a mocha version where this is fixed, we won't actually
      // execute the mocha dry run but simulate it by reading the suites and tests of
      // the mocha object and writing a report file with similar structure to the json report
      // (just leave out some execution details like timing, retry and erros)
-      if (config.get('mochaOpts.dryRun')) {
+      if (this.config.get('mochaOpts.dryRun')) {
        return this.simulateMochaDryRun(mocha);
      }

-      await this.lifecycle.beforeTests.trigger(mocha.suite);
-
+      if (abortSignal?.aborted) {
+        this.log.warning('run aborted');
+        return;
+      }
+
+      await lifecycle.beforeTests.trigger(mocha.suite);
      if (abortSignal?.aborted) {
        this.log.warning('run aborted');
        return;
      }

      this.log.info('Starting tests');
-      return await runTests(this.lifecycle, mocha, abortSignal);
+      return await runTests(lifecycle, mocha, abortSignal);
    });
  }

-  private async validateEsVersion(config: Config) {
-    const es = createEsClientForFtrConfig(config);
+  private async validateEsVersion() {
+    const es = createEsClientForFtrConfig(this.config);

    let esInfo;
    try {
@ -152,99 +149,112 @@ export class FunctionalTestRunner {
  }

  async getTestStats() {
-    return await this._run(async (config, coreProviders) => {
-      if (config.get('testRunner')) {
-        throw new Error('Unable to get test stats for config that uses a custom test runner');
+    return await this.runHarness(async (lifecycle, coreProviders) => {
+      if (this.config.get('testRunner')) {
+        return;
      }

-      // replace the function of custom service providers so that they return
-      // promise-like objects which never resolve, essentially disabling them
-      // allowing us to load the test files and populate the mocha suites
-      const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
-        readProviderSpec(type, providers).map((p) => ({
-          ...p,
-          fn: skip.includes(p.name)
-            ? (...args: unknown[]) => {
-                const result = p.fn(...args);
-                if ('then' in result) {
-                  throw new Error(
-                    `Provider [${p.name}] returns a promise so it can't loaded during test analysis`
-                  );
-                }
+      const providers = this.getStubProviderCollection(coreProviders);
+      const mocha = await setupMocha({
+        lifecycle,
+        log: this.log,
+        config: this.config,
+        providers,
+        esVersion: this.esVersion,
+      });

-                return result;
-              }
-            : () => ({
-                then: () => {},
-              }),
-        }));
-
-      const providers = new ProviderCollection(this.log, [
-        ...coreProviders,
-        ...readStubbedProviderSpec(
-          'Service',
-          config.get('services'),
-          config.get('servicesRequiredForTestAnalysis')
-        ),
-        ...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
-      ]);
-
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
-
-      const countTests = (suite: Suite): number =>
-        suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length);
+      const queue = new Set([mocha.suite]);
+      const allTests: Test[] = [];
+      for (const suite of queue) {
+        for (const test of suite.tests) {
+          allTests.push(test);
+        }
+        for (const childSuite of suite.suites) {
+          queue.add(childSuite);
+        }
+      }

      return {
-        testCount: countTests(mocha.suite),
+        testCount: allTests.length,
+        nonSkippedTestCount: allTests.filter((t) => !t.pending).length,
        testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()),
      };
    });
  }

-  async _run<T = any>(
-    handler: (config: Config, coreProvider: ReturnType<typeof readProviderSpec>) => Promise<T>
+  private getStubProviderCollection(coreProviders: Providers) {
+    // when we want to load the tests but not actually run anything we can
+    // use stubbed providers which allow mocha to do it's thing without taking
+    // too much time
+    const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
+      readProviderSpec(type, providers).map((p) => ({
+        ...p,
+        fn: skip.includes(p.name)
+          ? (ctx: any) => {
+              const result = ProviderCollection.callProviderFn(p.fn, ctx);
+
+              if ('then' in result) {
+                throw new Error(
+                  `Provider [${p.name}] returns a promise so it can't loaded during test analysis`
+                );
+              }
+
+              return result;
+            }
+          : () => ({
+              then: () => {},
+            }),
+      }));
+
+    return new ProviderCollection(this.log, [
+      ...coreProviders,
+      ...readStubbedProviderSpec(
+        'Service',
+        this.config.get('services'),
+        this.config.get('servicesRequiredForTestAnalysis')
+      ),
+      ...readStubbedProviderSpec('PageObject', this.config.get('pageObjects'), []),
+    ]);
+  }
+
+  private async runHarness<T = any>(
+    handler: (lifecycle: Lifecycle, coreProviders: Providers) => Promise<T>
  ): Promise<T> {
    let runErrorOccurred = false;
+    const lifecycle = new Lifecycle(this.log);
+    const testMetadata = new TestMetadata(lifecycle);

    try {
-      const config = await readConfigFile(
-        this.log,
-        this.esVersion,
-        this.configFile,
-        this.configOverrides
-      );
-      this.log.info('Config loaded');
-
      if (
-        (!config.get('testFiles') || config.get('testFiles').length === 0) &&
-        !config.get('testRunner')
+        (!this.config.get('testFiles') || this.config.get('testFiles').length === 0) &&
+        !this.config.get('testRunner')
      ) {
        throw new Error('No tests defined.');
      }

      const dockerServers = new DockerServersService(
-        config.get('dockerServers'),
+        this.config.get('dockerServers'),
        this.log,
-        this.lifecycle
+        lifecycle
      );

      // base level services that functional_test_runner exposes
      const coreProviders = readProviderSpec('Service', {
-        lifecycle: () => this.lifecycle,
+        lifecycle: () => lifecycle,
        log: () => this.log,
-        testMetadata: () => this.testMetadata,
-        config: () => config,
+        testMetadata: () => testMetadata,
+        config: () => this.config,
        dockerServers: () => dockerServers,
        esVersion: () => this.esVersion,
      });

-      return await handler(config, coreProviders);
+      return await handler(lifecycle, coreProviders);
    } catch (runError) {
      runErrorOccurred = true;
      throw runError;
    } finally {
      try {
-        await this.close();
+        await lifecycle.cleanup.trigger();
      } catch (closeError) {
        if (runErrorOccurred) {
          this.log.error('failed to close functional_test_runner');
@ -257,13 +267,6 @@ export class FunctionalTestRunner {
    }
  }

-  async close() {
-    if (this.closed) return;
-
-    this.closed = true;
-    await this.lifecycle.cleanup.trigger();
-  }
-
  simulateMochaDryRun(mocha: any) {
    interface TestEntry {
      file: string;
--- a/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts
@ -15,9 +15,15 @@ describe('Config', () => {
        services: {
          foo: () => 42,
        },
+        servers: {
+          elasticsearch: {
+            port: 1234,
+          },
+        },
      },
      primary: true,
      path: process.cwd(),
+      module: {} as any,
    });

    expect(config.has('services.foo')).toEqual(true);
--- a/packages/kbn-test/src/functional_test_runner/lib/config/config.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/config/config.ts
@ -10,6 +10,7 @@ import { Schema } from 'joi';
 import { cloneDeepWith, get, has, toPath } from 'lodash';

 import { schema } from './schema';
+import { ConfigModule } from './read_config_file';

 const $values = Symbol('values');

@ -17,25 +18,27 @@ interface Options {
  settings?: Record<string, any>;
  primary?: boolean;
  path: string;
+  module: ConfigModule;
 }

 export class Config {
  public readonly path: string;
+  public readonly module: ConfigModule;
  private [$values]: Record<string, any>;

  constructor(options: Options) {
-    const { settings = {}, primary = false, path = null } = options || {};
-
-    if (!path) {
+    if (!options.path) {
      throw new TypeError('path is a required option');
    }

-    this.path = path;
-    const { error, value } = schema.validate(settings, {
+    this.path = options.path;
+    this.module = options.module;
+
+    const { error, value } = schema.validate(options.settings, {
      abortEarly: false,
      context: {
-        primary: !!primary,
-        path,
+        primary: !!options?.primary,
+        path: options.path,
      },
    });

--- a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts
@ -6,59 +6,134 @@
 * Side Public License, v 1.
 */

-import { ToolingLog } from '@kbn/dev-utils';
+import Path from 'path';
+import { ToolingLog, createFlagError, createFailError } from '@kbn/dev-utils';
 import { defaultsDeep } from 'lodash';

+import { FtrConfigProvider } from '../../public_types';
 import { Config } from './config';
 import { EsVersion } from '../es_version';

-const cache = new WeakMap();
+interface LoadSettingsOptions {
+  path: string;
+  settingOverrides: any;
+  primary: boolean;
+}

-async function getSettingsFromFile(
-  log: ToolingLog,
-  esVersion: EsVersion,
-  path: string,
-  settingOverrides: any
-) {
-  const configModule = require(path); // eslint-disable-line @typescript-eslint/no-var-requires
-  const configProvider = configModule.__esModule ? configModule.default : configModule;
+export interface ConfigModule {
+  type: 'config';
+  path: string;
+  provider: FtrConfigProvider;
+}

-  if (!cache.has(configProvider)) {
-    log.debug('Loading config file from %j', path);
-    cache.set(
-      configProvider,
-      configProvider({
-        log,
-        esVersion,
-        async readConfigFile(p: string, o: any) {
-          return new Config({
-            settings: await getSettingsFromFile(log, esVersion, p, o),
-            primary: false,
-            path: p,
-          });
-        },
-      })
+async function getConfigModule({
+  path,
+  primary,
+}: {
+  path: string;
+  primary: boolean;
+}): Promise<ConfigModule> {
+  let resolvedPath;
+  try {
+    resolvedPath = require.resolve(path);
+  } catch (error) {
+    if (error.code === 'MODULE_NOT_FOUND') {
+      throw createFlagError(`Unable to find config file [${path}]`);
+    }
+    throw error;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const exports = require(resolvedPath);
+  const defaultExport = exports.__esModule ? exports.default : exports;
+  if (typeof defaultExport !== 'function') {
+    throw createFailError(
+      `default export must be a function in a ftr config otherwise is not a valid instance`
    );
  }

-  const settingsWithDefaults: any = defaultsDeep(
-    {},
-    settingOverrides,
-    await cache.get(configProvider)!
-  );
-
-  return settingsWithDefaults;
+  return {
+    type: 'config',
+    path: resolvedPath,
+    provider: defaultExport,
+  };
 }

+const cache = new WeakMap<FtrConfigProvider, Promise<any>>();
+async function executeConfigModule(
+  log: ToolingLog,
+  esVersion: EsVersion,
+  options: LoadSettingsOptions,
+  module: ConfigModule
+): Promise<any> {
+  const cached = cache.get(module.provider);
+
+  if (cached) {
+    return defaultsDeep({}, options.settingOverrides, await cached);
+  }
+
+  log.debug(`Loading config file from ${Path.relative(process.cwd(), options.path)}`);
+  const settings: Promise<any> = module.provider({
+    log,
+    esVersion,
+    async readConfigFile(p: string) {
+      const childModule = await getConfigModule({
+        primary: false,
+        path: p,
+      });
+
+      return new Config({
+        settings: await executeConfigModule(
+          log,
+          esVersion,
+          {
+            path: childModule.path,
+            settingOverrides: {},
+            primary: false,
+          },
+          childModule
+        ),
+        primary: false,
+        path: p,
+        module: childModule,
+      });
+    },
+  });
+
+  cache.set(module.provider, Promise.resolve(settings));
+
+  return defaultsDeep({}, options.settingOverrides, await settings);
+}
+
+const ident = <T>(vars: T) => vars;
+
 export async function readConfigFile(
  log: ToolingLog,
  esVersion: EsVersion,
  path: string,
-  settingOverrides: any = {}
+  settingOverrides: any = {},
+  extendSettings: (vars: any) => any = ident
 ) {
-  return new Config({
-    settings: await getSettingsFromFile(log, esVersion, path, settingOverrides),
+  const module = await getConfigModule({
    primary: true,
    path,
  });
+
+  return new Config({
+    settings: extendSettings(
+      await executeConfigModule(
+        log,
+        esVersion,
+        {
+          path,
+          settingOverrides,
+          primary: true,
+        },
+        module
+      )
+    ),
+    primary: true,
+    path,
+    module,
+  });
 }
--- a/packages/kbn-test/src/functional_test_runner/lib/index.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/index.ts
@ -16,5 +16,5 @@ export * from './test_metadata';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';

-export type { Provider } from './providers';
+export type { Provider, Providers } from './providers';
 export * from './es_version';
--- a/packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts
@ -6,29 +6,52 @@
 * Side Public License, v 1.
 */

+import * as Rx from 'rxjs';
+import { materialize } from 'rxjs/operators';
+import { ToolingLog } from '@kbn/dev-utils';
+
 import { LifecyclePhase } from './lifecycle_phase';

 import { Suite, Test } from '../fake_mocha_types';

 export class Lifecycle {
+  /** root subscription to cleanup lifecycle phases when lifecycle completes */
+  private readonly sub = new Rx.Subscription();
+
  /** lifecycle phase that will run handlers once before tests execute */
-  public readonly beforeTests = new LifecyclePhase<[Suite]>({
+  public readonly beforeTests = new LifecyclePhase<[Suite]>(this.sub, {
    singular: true,
  });
  /** lifecycle phase that runs handlers before each runnable (test and hooks) */
-  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
+  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>(this.sub);
  /** lifecycle phase that runs handlers before each suite */
-  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>(this.sub);
  /** lifecycle phase that runs handlers before each test */
-  public readonly beforeEachTest = new LifecyclePhase<[Test]>();
+  public readonly beforeEachTest = new LifecyclePhase<[Test]>(this.sub);
  /** lifecycle phase that runs handlers after each suite */
-  public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly afterTestSuite = new LifecyclePhase<[Suite]>(this.sub);
  /** lifecycle phase that runs handlers after a test fails */
-  public readonly testFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testFailure = new LifecyclePhase<[Error, Test]>(this.sub);
  /** lifecycle phase that runs handlers after a hook fails */
-  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>(this.sub);
  /** lifecycle phase that runs handlers at the very end of execution */
-  public readonly cleanup = new LifecyclePhase<[]>({
+  public readonly cleanup = new LifecyclePhase<[]>(this.sub, {
    singular: true,
  });
+
+  constructor(log: ToolingLog) {
+    for (const [name, phase] of Object.entries(this)) {
+      if (phase instanceof LifecyclePhase) {
+        phase.before$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+        phase.after$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+      }
+    }
+
+    // after the singular cleanup lifecycle phase completes unsubscribe from the root subscription
+    this.cleanup.after$.pipe(materialize()).subscribe((n) => {
+      if (n.kind === 'C') {
+        this.sub.unsubscribe();
+      }
+    });
+  }
 }
--- a/packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.test.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.test.ts
@ -26,7 +26,7 @@ describe('with randomness', () => {
  });

  it('calls handlers in random order', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
    const order: string[] = [];

    phase.add(
@ -69,7 +69,7 @@ describe('without randomness', () => {
  afterEach(() => jest.restoreAllMocks());

  it('calls all handlers and throws first error', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
    const fn1 = jest.fn();
    phase.add(fn1);

@ -88,7 +88,7 @@ describe('without randomness', () => {
  });

  it('triggers before$ just before calling handler and after$ once it resolves', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
    const order: string[] = [];

    const beforeSub = jest.fn(() => order.push('before'));
@ -116,7 +116,7 @@ describe('without randomness', () => {
  });

  it('completes before$ and after$ if phase is singular', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });

    const beforeNotifs: Array<Rx.Notification<unknown>> = [];
    phase.before$.pipe(materialize()).subscribe((n) => beforeNotifs.push(n));
@ -160,7 +160,7 @@ describe('without randomness', () => {
  });

  it('completes before$ subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
    await phase.trigger();

    await expect(phase.before$.pipe(materialize(), toArray()).toPromise()).resolves
@ -177,7 +177,7 @@ describe('without randomness', () => {
  });

  it('replays after$ event subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
    await phase.trigger();

    await expect(phase.after$.pipe(materialize(), toArray()).toPromise()).resolves
--- a/packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.ts
@ -26,6 +26,7 @@ export class LifecyclePhase<Args extends readonly any[]> {
  public readonly after$: Rx.Observable<void>;

  constructor(
+    sub: Rx.Subscription,
    private readonly options: {
      singular?: boolean;
    } = {}
@ -35,6 +36,12 @@ export class LifecyclePhase<Args extends readonly any[]> {

    this.afterSubj = this.options.singular ? new Rx.ReplaySubject<void>(1) : new Rx.Subject<void>();
    this.after$ = this.afterSubj.asObservable();
+
+    sub.add(() => {
+      this.beforeSubj.complete();
+      this.afterSubj.complete();
+      this.handlers.length = 0;
+    });
  }

  public add(fn: (...args: Args) => Promise<void> | void) {
--- a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js
+++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js
@ -25,15 +25,15 @@ import { validateCiGroupTags } from './validate_ci_group_tags';
 *  @param  {EsVersion} esVersion
 *  @return {Promise<Mocha>}
 */
-export async function setupMocha(
+export async function setupMocha({
  lifecycle,
  log,
  config,
  providers,
  esVersion,
  reporter,
-  reporterOptions
-) {
+  reporterOptions,
+}) {
  // configure mocha
  const mocha = new Mocha({
    ...config.get('mochaOpts'),
--- a/packages/kbn-test/src/functional_test_runner/lib/providers/index.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/providers/index.ts
@ -7,5 +7,6 @@
 */

 export { ProviderCollection } from './provider_collection';
-export { readProviderSpec } from './read_provider_spec';
+export * from './read_provider_spec';
+export { createAsyncInstance } from './async_instance';
 export type { Provider } from './read_provider_spec';
--- a/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts
@ -10,12 +10,19 @@ import { ToolingLog } from '@kbn/dev-utils';

 import { loadTracer } from '../load_tracer';
 import { createAsyncInstance, isAsyncInstance } from './async_instance';
-import { Providers } from './read_provider_spec';
+import { Providers, ProviderFn, isProviderConstructor } from './read_provider_spec';
 import { createVerboseInstance } from './verbose_instance';
-import { GenericFtrService } from '../../public_types';

 export class ProviderCollection {
-  private readonly instances = new Map();
+  static callProviderFn(providerFn: ProviderFn, ctx: any) {
+    if (isProviderConstructor(providerFn)) {
+      return new providerFn(ctx);
+    }
+
+    return providerFn(ctx);
+  }
+
+  private readonly instances = new Map<ProviderFn, any>();

  constructor(private readonly log: ToolingLog, private readonly providers: Providers) {}

@ -58,20 +65,13 @@ export class ProviderCollection {
    }
  }

-  public invokeProviderFn(provider: (args: any) => any) {
-    const ctx = {
+  public invokeProviderFn(provider: ProviderFn) {
+    return ProviderCollection.callProviderFn(provider, {
      getService: this.getService,
      hasService: this.hasService,
      getPageObject: this.getPageObject,
      getPageObjects: this.getPageObjects,
-    };
-
-    if (provider.prototype instanceof GenericFtrService) {
-      const Constructor = provider as any as new (ctx: any) => any;
-      return new Constructor(ctx);
-    }
-
-    return provider(ctx);
+    });
  }

  private findProvider(type: string, name: string) {
--- a/packages/kbn-test/src/functional_test_runner/lib/providers/read_provider_spec.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/providers/read_provider_spec.ts
@ -6,10 +6,20 @@
 * Side Public License, v 1.
 */

+import { GenericFtrService } from '../../public_types';
+
+export type ProviderConstructor = new (...args: any[]) => any;
+export type ProviderFactory = (...args: any[]) => any;
+
+export function isProviderConstructor(x: unknown): x is ProviderConstructor {
+  return typeof x === 'function' && x.prototype instanceof GenericFtrService;
+}
+
+export type ProviderFn = ProviderConstructor | ProviderFactory;
 export type Providers = ReturnType<typeof readProviderSpec>;
 export type Provider = Providers extends Array<infer X> ? X : unknown;

-export function readProviderSpec(type: string, providers: Record<string, (...args: any[]) => any>) {
+export function readProviderSpec(type: string, providers: Record<string, ProviderFn>) {
  return Object.keys(providers).map((name) => {
    return {
      type,
--- a/packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.test.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.test.ts
@ -11,6 +11,7 @@ import { Lifecycle } from '../lifecycle';
 import { decorateSnapshotUi, expectSnapshot } from './decorate_snapshot_ui';
 import path from 'path';
 import fs from 'fs';
+import { ToolingLog } from '@kbn/dev-utils';

 const createRootSuite = () => {
  const suite = {
@ -65,7 +66,7 @@ describe('decorateSnapshotUi', () => {
    let lifecycle: Lifecycle;
    let rootSuite: Suite;
    beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
      rootSuite = createRootSuite();
      decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });

@ -116,7 +117,7 @@ describe('decorateSnapshotUi', () => {
    let lifecycle: Lifecycle;
    let rootSuite: Suite;
    beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
      rootSuite = createRootSuite();
      decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });

@ -162,7 +163,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
    let lifecycle: Lifecycle;
    let rootSuite: Suite;
    beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
      rootSuite = createRootSuite();
      decorateSnapshotUi({ lifecycle, updateSnapshots: true, isCi: false });

@ -185,7 +186,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
        fs.writeFileSync(
          snapshotFile,
          `// Jest Snapshot v1, https://goo.gl/fbAQLP
-  
+
  exports[\`Test 1\`] = \`"foo"\`;
        `,
          { encoding: 'utf-8' }
@ -219,7 +220,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
    let lifecycle: Lifecycle;
    let rootSuite: Suite;
    beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
      rootSuite = createRootSuite();
      decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: true });

--- a/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts
+++ b/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts
@ -9,6 +9,8 @@
 import fs from 'fs';
 import { join, resolve } from 'path';

+import { ToolingLog } from '@kbn/dev-utils';
+
 jest.mock('fs');
 jest.mock('@kbn/utils', () => {
  return { REPO_ROOT: '/dev/null/root' };
@ -60,7 +62,7 @@ describe('SuiteTracker', () => {
  };

  const runLifecycleWithMocks = async (mocks: Suite[], fn: (objs: any) => any = () => {}) => {
-    const lifecycle = new Lifecycle();
+    const lifecycle = new Lifecycle(new ToolingLog());
    const suiteTracker = SuiteTracker.startTracking(
      lifecycle,
      resolve(REPO_ROOT, MOCK_CONFIG_PATH)
--- a/packages/kbn-test/src/functional_test_runner/public_types.ts
+++ b/packages/kbn-test/src/functional_test_runner/public_types.ts
@ -105,4 +105,6 @@ export interface FtrConfigProviderContext {
  readConfigFile(path: string): Promise<Config>;
 }

+export type FtrConfigProvider = <T>(ctx: FtrConfigProviderContext) => T | Promise<T>;
+
 export type { Test, Suite };
--- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
+++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts
@ -8,41 +8,138 @@

 import { resolve } from 'path';
 import type { ToolingLog } from '@kbn/dev-utils';
-import { KIBANA_ROOT } from './paths';
-import type { Config } from '../../functional_test_runner/';
+import getPort from 'get-port';
+import { REPO_ROOT } from '@kbn/dev-utils';
+import type { Config } from '../../functional_test_runner';
 import { createTestEsCluster } from '../../es';

 interface RunElasticsearchOptions {
  log: ToolingLog;
  esFrom?: string;
-}
-export async function runElasticsearch({
-  config,
-  options,
-  onEarlyExit,
-}: {
  config: Config;
-  options: RunElasticsearchOptions;
  onEarlyExit?: (msg: string) => void;
-}) {
-  const { log, esFrom } = options;
-  const ssl = config.get('esTestCluster.ssl');
-  const license = config.get('esTestCluster.license');
-  const esArgs = config.get('esTestCluster.serverArgs');
-  const esJavaOpts = config.get('esTestCluster.esJavaOpts');
+  logsDir?: string;
+  name?: string;
+}
+
+interface CcsConfig {
+  remoteClusterUrl: string;
+}
+
+type EsConfig = ReturnType<typeof getEsConfig>;
+
+function getEsConfig({
+  config,
+  esFrom = config.get('esTestCluster.from'),
+}: RunElasticsearchOptions) {
+  const ssl = !!config.get('esTestCluster.ssl');
+  const license: 'basic' | 'trial' | 'gold' = config.get('esTestCluster.license');
+  const esArgs: string[] = config.get('esTestCluster.serverArgs');
+  const esJavaOpts: string | undefined = config.get('esTestCluster.esJavaOpts');
  const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true');

-  const cluster = createTestEsCluster({
-    port: config.get('servers.elasticsearch.port'),
-    password: isSecurityEnabled ? 'changeme' : config.get('servers.elasticsearch.password'),
+  const port: number | undefined = config.get('servers.elasticsearch.port');
+  const ccsConfig: CcsConfig | undefined = undefined; // config.get('esTestCluster.ccs');
+
+  const password: string | undefined = isSecurityEnabled
+    ? 'changeme'
+    : config.get('servers.elasticsearch.password');
+
+  const dataArchive: string | undefined = config.get('esTestCluster.dataArchive');
+
+  return {
+    ssl,
    license,
-    log,
-    basePath: resolve(KIBANA_ROOT, '.es'),
-    esFrom: esFrom || config.get('esTestCluster.from'),
-    dataArchive: config.get('esTestCluster.dataArchive'),
    esArgs,
    esJavaOpts,
-    ssl,
+    isSecurityEnabled,
+    esFrom,
+    port,
+    password,
+    dataArchive,
+    ccsConfig,
+  };
+}
+
+export async function runElasticsearch(
+  options: RunElasticsearchOptions
+): Promise<() => Promise<void>> {
+  const { log, logsDir, name } = options;
+  const config = getEsConfig(options);
+
+  if (!config.ccsConfig) {
+    const node = await startEsNode({
+      log,
+      name: name ?? 'ftr',
+      logsDir,
+      config,
+    });
+    return async () => {
+      await node.cleanup();
+    };
+  }
+
+  const remotePort = await getPort();
+  const remoteNode = await startEsNode({
+    log,
+    name: name ?? 'ftr-remote',
+    logsDir,
+    config: {
+      ...config,
+      // @ts-expect-error
+      port: parseInt(new URL(config.ccsConfig.remoteClusterUrl).port, 10),
+      transportPort: remotePort,
+    },
+  });
+
+  const localNode = await startEsNode({
+    log,
+    name: name ?? 'ftr-local',
+    logsDir,
+    config: {
+      ...config,
+      esArgs: [...config.esArgs, `cluster.remote.ftr-remote.seeds=localhost:${remotePort}`],
+    },
+  });
+
+  return async () => {
+    await localNode.cleanup();
+    await remoteNode.cleanup();
+  };
+}
+
+async function startEsNode({
+  log,
+  name,
+  config,
+  onEarlyExit,
+  logsDir,
+}: {
+  log: ToolingLog;
+  name: string;
+  config: EsConfig & { transportPort?: number };
+  onEarlyExit?: (msg: string) => void;
+  logsDir?: string;
+}) {
+  const cluster = createTestEsCluster({
+    clusterName: `cluster-${name}`,
+    esArgs: config.esArgs,
+    esFrom: config.esFrom,
+    esJavaOpts: config.esJavaOpts,
+    license: config.license,
+    password: config.password,
+    port: config.port,
+    ssl: config.ssl,
+    log,
+    writeLogsToPath: logsDir ? resolve(logsDir, `es-cluster-${name}.log`) : undefined,
+    basePath: resolve(REPO_ROOT, '.es'),
+    nodes: [
+      {
+        name,
+        dataArchive: config.dataArchive,
+      },
+    ],
+    transportPort: config.transportPort,
    onEarlyExit,
  });

--- a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts
+++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts
@ -6,8 +6,9 @@
 * Side Public License, v 1.
 */
 import type { ToolingLog } from '@kbn/dev-utils';
-import { FunctionalTestRunner, readConfigFile, EsVersion } from '../../functional_test_runner';
-import { CliError } from './run_cli';
+import { createFailError } from '@kbn/dev-utils';
+
+import { EsVersion, readConfigFile, FunctionalTestRunner } from '../../functional_test_runner';

 export interface CreateFtrOptions {
  /** installation dir from which to run Kibana */
@ -33,6 +34,7 @@ export interface CreateFtrOptions {
 export interface CreateFtrParams {
  configPath: string;
  options: CreateFtrOptions;
+  signal?: AbortSignal;
 }
 async function createFtr({
  configPath,
@ -49,40 +51,35 @@ async function createFtr({
    dryRun,
  },
 }: CreateFtrParams) {
-  const config = await readConfigFile(log, esVersion, configPath);
+  const config = await readConfigFile(log, esVersion, configPath, {
+    mochaOpts: {
+      bail: !!bail,
+      grep,
+      dryRun: !!dryRun,
+    },
+    kbnTestServer: {
+      installDir,
+    },
+    suiteFiles: {
+      include: suiteFiles?.include || [],
+      exclude: suiteFiles?.exclude || [],
+    },
+    suiteTags: {
+      include: suiteTags?.include || [],
+      exclude: suiteTags?.exclude || [],
+    },
+    updateBaselines,
+    updateSnapshots,
+  });

  return {
    config,
-    ftr: new FunctionalTestRunner(
-      log,
-      configPath,
-      {
-        mochaOpts: {
-          bail: !!bail,
-          grep,
-          dryRun: !!dryRun,
-        },
-        kbnTestServer: {
-          installDir,
-        },
-        updateBaselines,
-        updateSnapshots,
-        suiteFiles: {
-          include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')],
-          exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')],
-        },
-        suiteTags: {
-          include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')],
-          exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')],
-        },
-      },
-      esVersion
-    ),
+    ftr: new FunctionalTestRunner(log, config, esVersion),
  };
 }

-export async function assertNoneExcluded(params: CreateFtrParams) {
-  const { config, ftr } = await createFtr(params);
+export async function assertNoneExcluded({ configPath, options }: CreateFtrParams) {
+  const { config, ftr } = await createFtr({ configPath, options });

  if (config.get('testRunner')) {
    // tests with custom test runners are not included in this check
@ -90,37 +87,37 @@ export async function assertNoneExcluded(params: CreateFtrParams) {
  }

  const stats = await ftr.getTestStats();
-  if (stats.testsExcludedByTag.length > 0) {
-    throw new CliError(`
-      ${stats.testsExcludedByTag.length} tests in the ${params.configPath} config
+  if (stats?.testsExcludedByTag.length > 0) {
+    throw createFailError(`
+      ${stats?.testsExcludedByTag.length} tests in the ${configPath} config
      are excluded when filtering by the tags run on CI. Make sure that all suites are
      tagged with one of the following tags:

-      ${JSON.stringify(params.options.suiteTags)}
+      ${JSON.stringify(options.suiteTags)}

-      - ${stats.testsExcludedByTag.join('\n      - ')}
+      - ${stats?.testsExcludedByTag.join('\n      - ')}
    `);
  }
 }

-export async function runFtr(params: CreateFtrParams, signal?: AbortSignal) {
-  const { ftr } = await createFtr(params);
+export async function runFtr({ configPath, options, signal }: CreateFtrParams) {
+  const { ftr } = await createFtr({ configPath, options });

  const failureCount = await ftr.run(signal);
  if (failureCount > 0) {
-    throw new CliError(
+    throw createFailError(
      `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}`
    );
  }
 }

-export async function hasTests(params: CreateFtrParams) {
-  const { ftr, config } = await createFtr(params);
+export async function hasTests({ configPath, options }: CreateFtrParams) {
+  const { ftr, config } = await createFtr({ configPath, options });

  if (config.get('testRunner')) {
    // configs with custom test runners are assumed to always have tests
    return true;
  }
  const stats = await ftr.getTestStats();
-  return stats.testCount > 0;
+  return stats?.testCount && stats?.testCount > 0;
 }
--- a/packages/kbn-test/src/functional_tests/tasks.ts
+++ b/packages/kbn-test/src/functional_tests/tasks.ts
@ -113,19 +113,26 @@ export async function runTests(options: RunTestsParams) {
          abortCtrl.abort();
        };

-        let es;
+        let shutdownEs;
        try {
          if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') {
-            es = await runElasticsearch({ config, options: { ...options, log }, onEarlyExit });
+            shutdownEs = await runElasticsearch({ ...options, config, log, onEarlyExit });
            if (abortCtrl.signal.aborted) {
              return;
            }
          }
-          await runKibanaServer({ procs, config, options, onEarlyExit });
+          await runKibanaServer({
+            procs,
+            config,
+            options: {
+              installDir: options.installDir,
+            },
+            onEarlyExit,
+          });
          if (abortCtrl.signal.aborted) {
            return;
          }
-          await runFtr({ configPath, options: { ...options, log } }, abortCtrl.signal);
+          await runFtr({ configPath, options: { ...options, log }, signal: abortCtrl.signal });
        } finally {
          try {
            const delay = config.get('kbnTestServer.delayShutdown');
@ -136,8 +143,8 @@ export async function runTests(options: RunTestsParams) {

            await procs.stop('kibana');
          } finally {
-            if (es) {
-              await es.cleanup();
+            if (shutdownEs) {
+              await shutdownEs();
            }
          }
        }
@ -177,7 +184,7 @@ export async function startServers({ ...options }: StartServerOptions) {
  await withProcRunner(log, async (procs) => {
    const config = await readConfigFile(log, options.esVersion, options.config);

-    const es = await runElasticsearch({ config, options: opts });
+    const shutdownEs = await runElasticsearch({ config, log, esFrom: options.esFrom });
    await runKibanaServer({
      procs,
      config,
@ -201,7 +208,7 @@ export async function startServers({ ...options }: StartServerOptions) {
    log.success(makeSuccessMessage(options));

    await procs.waitForAllToStop();
-    await es.cleanup();
+    await shutdownEs();
  });
 }

--- a/packages/kbn-test/src/index.ts
+++ b/packages/kbn-test/src/index.ts
@ -18,6 +18,8 @@ import {
 // @internal
 export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions };

+export { runElasticsearch, runKibanaServer } from './functional_tests/lib';
+
 // @ts-ignore not typed yet
 // @internal
 export { runTests, startServers } from './functional_tests/tasks';
@ -50,6 +52,8 @@ export { readConfigFile } from './functional_test_runner/lib/config/read_config_

 export { runFtrCli } from './functional_test_runner/cli';

+export { ProviderCollection, readProviderSpec } from './functional_test_runner/lib/providers';
+
 // @internal
 export { setupJUnitReportGeneration, escapeCdata } from './mocha';

--- a/x-pack/plugins/apm/ftr_e2e/cypress.config.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress.config.ts
@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineConfig } from 'cypress';
+// @ts-ignore
+import { plugin } from './cypress/plugins';
+
+module.exports = defineConfig({
+  fileServerFolder: './cypress',
+  fixturesFolder: './cypress/fixtures',
+  screenshotsFolder: './cypress/screenshots',
+  videosFolder: './cypress/videos',
+  requestTimeout: 10000,
+  responseTimeout: 40000,
+  defaultCommandTimeout: 30000,
+  execTimeout: 120000,
+  pageLoadTimeout: 120000,
+  viewportHeight: 900,
+  viewportWidth: 1440,
+  video: false,
+  screenshotOnRunFailure: false,
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      plugin(on, config);
+    },
+    baseUrl: 'http://localhost:5601',
+    supportFile: './cypress/support/e2e.ts',
+    specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}',
+  },
+});
--- a/x-pack/plugins/apm/ftr_e2e/cypress.json
+++ b/x-pack/plugins/apm/ftr_e2e/cypress.json
@ -1,16 +0,0 @@
-{
-  "fileServerFolder": "./cypress",
-  "fixturesFolder": "./cypress/fixtures",
-  "integrationFolder": "./cypress/integration",
-  "pluginsFile": "./cypress/plugins/index.js",
-  "screenshotsFolder": "./cypress/screenshots",
-  "supportFile": "./cypress/support/index.ts",
-  "videosFolder": "./cypress/videos",
-  "defaultCommandTimeout": 30000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 120000,
-  "viewportHeight": 900,
-  "viewportWidth": 1440,
-  "video": false,
-  "screenshotOnRunFailure": false
-}
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/no_data_screen.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/rules/error_count.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/rules/error_count.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/settings/custom_links.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/settings/custom_links.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/deep_links.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/deep_links.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/dependencies.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/dependencies.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/home.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/home.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/header_filters.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/header_filters.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/instances_table.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/instances_table.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/service_overview.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/service_overview.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/time_comparison.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/service_overview/time_comparison.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/transactions_overview/transactions_overview.spec.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/transactions_overview/transactions_overview.spec.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress/plugins/index.js
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/plugins/index.js
@ -22,7 +22,9 @@
 /**
 * @type {Cypress.PluginConfig}
 */
-module.exports = () => {
+const plugin = () => {
  // `on` is used to hook into various events Cypress emits
  // `config` is the resolved Cypress config
 };
+
+module.exports = { plugin };
--- a/x-pack/plugins/apm/ftr_e2e/cypress/support/commands.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/support/commands.ts
@ -18,10 +18,12 @@ Cypress.Commands.add('loginAsPowerUser', () => {
 Cypress.Commands.add(
  'loginAs',
  ({ username, password }: { username: string; password: string }) => {
-    cy.log(`Logging in as ${username}`);
+    // cy.session(username, () => {
    const kibanaUrl = Cypress.env('KIBANA_URL');
+    cy.log(`Logging in as ${username} on ${kibanaUrl}`);
+    cy.visit('/');
    cy.request({
-      log: false,
+      log: true,
      method: 'POST',
      url: `${kibanaUrl}/internal/security/login`,
      body: {
@ -33,7 +35,9 @@ Cypress.Commands.add(
      headers: {
        'kbn-xsrf': 'e2e_test',
      },
+      // });
    });
+    cy.visit('/');
  }
 );

--- a/x-pack/plugins/apm/ftr_e2e/cypress/support/index.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress/support/index.ts
--- a/x-pack/plugins/apm/ftr_e2e/cypress_start.ts
+++ b/x-pack/plugins/apm/ftr_e2e/cypress_start.ts
@ -61,7 +61,11 @@ async function cypressStart(

  const res = await cypressExecution({
    ...(spec !== undefined ? { spec } : {}),
-    config: { baseUrl: kibanaUrl },
+    config: {
+      e2e: {
+        baseUrl: kibanaUrl,
+      },
+    },
    env: {
      START_DATE: start,
      END_DATE: end,
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.config.js
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.config.js
@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { defineConfig } from 'cypress';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 120000,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    // eslint-disable-next-line no-unused-vars
+    setupNodeEvents(on, config) {},
+    supportFile: './cypress/support/commands.ts',
+  },
+  env: {
+    password: 'changeme',
+    username: 'elastic',
+  },
+  execTimeout: 120000,
+  pageLoadTimeout: 180000,
+  retries: {
+    runMode: 2,
+  },
+  screenshotsFolder: '../../../target/cypress/screenshots',
+  video: false,
+  videosFolder: '../../../target/cypress/videos',
+  viewportHeight: 1200,
+  viewportWidth: 1600,
+});
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress.json
@ -1,20 +0,0 @@
-{
-  "supportFile": "./cypress/support/commands.ts",
-  "pluginsFile": false,
-  "retries": {
-    "runMode": 2
-  },
-  "baseUrl": "http://localhost:5601",
-  "env": {
-    "username": "elastic",
-    "password": "changeme"
-  },
-  "screenshotsFolder": "../../../target/cypress/screenshots",
-  "videosFolder": "../../../target/cypress/videos",
-  "defaultCommandTimeout": 120000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 180000,
-  "viewportWidth": 1600,
-  "viewportHeight": 1200,
-  "video": false
-}
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.config.js
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.config.js
@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { defineConfig } from 'cypress';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 120000,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    // eslint-disable-next-line no-unused-vars
+    setupNodeEvents(on, config) {},
+    supportFile: false,
+  },
+  env: {
+    password: 'changeme',
+    username: 'elastic',
+  },
+  execTimeout: 120000,
+  fixturesFolder: false,
+  pageLoadTimeout: 180000,
+  retries: {
+    runMode: 2,
+  },
+  screenshotsFolder: '../../../target/cypress/screenshots',
+  video: false,
+  videosFolder: '../../../target/cypress/videos',
+  viewportHeight: 1200,
+  viewportWidth: 1600,
+});
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress.json
@ -1,21 +0,0 @@
-{
-  "supportFile": false,
-  "pluginsFile": false,
-  "retries": {
-    "runMode": 2
-  },
-  "baseUrl": "http://localhost:5601",
-  "env": {
-    "username": "elastic",
-    "password": "changeme"
-  },
-  "fixturesFolder": false,
-  "screenshotsFolder": "../../../target/cypress/screenshots",
-  "videosFolder": "../../../target/cypress/videos",
-  "defaultCommandTimeout": 120000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 180000,
-  "viewportWidth": 1600,
-  "viewportHeight": 1200,
-  "video": false
-}
--- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.config.js
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.config.js
@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { defineConfig } from 'cypress';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 120000,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    // eslint-disable-next-line no-unused-vars
+    setupNodeEvents(on, config) {},
+    supportFile: './cypress/support/commands.ts',
+  },
+  env: {
+    password: 'changeme',
+    username: 'elastic',
+  },
+  execTimeout: 120000,
+  pageLoadTimeout: 180000,
+  retries: {
+    runMode: 2,
+  },
+  screenshotsFolder: '../../../target/cypress/screenshots',
+  video: false,
+  videosFolder: '../../../target/cypress/videos',
+  viewportHeight: 1200,
+  viewportWidth: 1600,
+});
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress.json
@ -1,20 +0,0 @@
-{
-  "supportFile": "./cypress/support/commands.ts",
-  "pluginsFile": false,
-  "retries": {
-    "runMode": 2
-  },
-  "baseUrl": "http://localhost:5601",
-  "env": {
-    "username": "elastic",
-    "password": "changeme"
-  },
-  "screenshotsFolder": "../../../target/cypress/screenshots",
-  "videosFolder": "../../../target/cypress/videos",
-  "defaultCommandTimeout": 120000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 180000,
-  "viewportWidth": 1600,
-  "viewportHeight": 1200,
-  "video": false
-}
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts
--- a/x-pack/plugins/osquery/cypress/cypress.config.ts
+++ b/x-pack/plugins/osquery/cypress/cypress.config.ts
@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { defineConfig } from 'cypress';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 60000,
+  execTimeout: 120000,
+  pageLoadTimeout: 12000,
+
+  retries: {
+    runMode: 1,
+    openMode: 0,
+  },
+
+  screenshotsFolder: '../../../target/kibana-osquery/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  videosFolder: '../../../target/kibana-osquery/cypress/videos',
+  viewportHeight: 900,
+  viewportWidth: 1440,
+  experimentalStudio: true,
+
+  env: {
+    'cypress-react-selector': {
+      root: '#osquery-app',
+    },
+  },
+
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    // eslint-disable-next-line
+    setupNodeEvents(on, config) {
+      // implement node event listeners here
+    },
+  },
+});
--- a/x-pack/plugins/osquery/cypress/cypress.json
+++ b/x-pack/plugins/osquery/cypress/cypress.json
@ -1,14 +0,0 @@
-{
-  "baseUrl": "http://localhost:5620",
-  "defaultCommandTimeout": 60000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 120000,
-  "nodeVersion": "system",
-  "retries": {
-    "runMode": 2
-  },
-  "trashAssetsBeforeRuns": false,
-  "video": false,
-  "viewportHeight": 900,
-  "viewportWidth": 1440
-}
--- a/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts
+++ b/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts
--- a/x-pack/plugins/osquery/cypress/support/index.ts
+++ b/x-pack/plugins/osquery/cypress/support/index.ts
--- a/x-pack/plugins/osquery/package.json
+++ b/x-pack/plugins/osquery/package.json
@ -5,9 +5,9 @@
    "private": true,
    "license": "Elastic-License",
    "scripts": {
-      "cypress:open": "../../../node_modules/.bin/cypress open --config-file ./cypress/cypress.json",
+      "cypress:open": "../../../node_modules/.bin/cypress open --config-file ./cypress/cypress.config.ts",
      "cypress:open-as-ci": "node ../../../scripts/functional_tests --config ../../test/osquery_cypress/visual_config.ts",
-      "cypress:run": "../../../node_modules/.bin/cypress run --config-file ./cypress/cypress.json",
+      "cypress:run": "../../../node_modules/.bin/cypress run --config-file ./cypress/cypress.config.ts",
      "cypress:run-as-ci": "node ../../../scripts/functional_tests --config ../../test/osquery_cypress/cli_config.ts"
    }
 }
--- a/x-pack/plugins/security_solution/cypress/README.md
+++ b/x-pack/plugins/security_solution/cypress/README.md
@ -71,7 +71,7 @@ This configuration runs cypress tests against an arbitrary host.

 #### integration-test (CI)

-This configuration is driven by [elastic/integration-test](https://github.com/elastic/integration-test) which, as part of a bigger set of tests, provisions one VM with two instances configured in CCS mode and runs the [CCS Cypress test specs](./ccs_integration).
+This configuration is driven by [elastic/integration-test](https://github.com/elastic/integration-test) which, as part of a bigger set of tests, provisions one VM with two instances configured in CCS mode and runs the [CCS Cypress test specs](./ccs_e2e).

 The two clusters are named `admin` and `data` and are reachable as follows:

--- a/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts
--- a/x-pack/plugins/security_solution/cypress/ccs_integration/detection_rules/event_correlation_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/ccs_integration/detection_rules/event_correlation_rule.spec.ts
--- a/x-pack/plugins/security_solution/cypress/cypress.config.ts
+++ b/x-pack/plugins/security_solution/cypress/cypress.config.ts
@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineConfig } from 'cypress';
+import { esArchiver } from './support/es_archiver';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 60000,
+  execTimeout: 60000,
+  pageLoadTimeout: 60000,
+  responseTimeout: 60000,
+  screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  videosFolder: '../../../target/kibana-security-solution/cypress/videos',
+  viewportHeight: 946,
+  viewportWidth: 1680,
+  numTestsKeptInMemory: 10,
+  e2e: {
+    experimentalRunAllSpecs: true,
+    experimentalMemoryManagement: true,
+    setupNodeEvents(on, config) {
+      esArchiver(on, config);
+    },
+  },
+});
--- a/x-pack/plugins/security_solution/cypress/cypress.json
+++ b/x-pack/plugins/security_solution/cypress/cypress.json
@ -1,16 +0,0 @@
-{
-  "baseUrl": "http://localhost:5601",
-  "defaultCommandTimeout": 60000,
-  "execTimeout": 120000,
-  "pageLoadTimeout": 120000,
-  "nodeVersion": "system",
-  "retries": {
-    "runMode": 2
-  },
-  "screenshotsFolder": "../../../target/kibana-security-solution/cypress/screenshots",
-  "trashAssetsBeforeRuns": false,
-  "video": false,
-  "videosFolder": "../../../target/kibana-security-solution/cypress/videos",
-  "viewportHeight": 900,
-  "viewportWidth": 1440
-}
--- a/x-pack/plugins/security_solution/cypress/cypress_ci.config.ts
+++ b/x-pack/plugins/security_solution/cypress/cypress_ci.config.ts
@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineConfig } from 'cypress';
+import { esArchiver } from './support/es_archiver';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  defaultCommandTimeout: 150000,
+  execTimeout: 150000,
+  pageLoadTimeout: 150000,
+  numTestsKeptInMemory: 0,
+  retries: {
+    runMode: 1,
+  },
+  screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  videosFolder: '../../../target/kibana-security-solution/cypress/videos',
+  viewportHeight: 946,
+  viewportWidth: 1680,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    experimentalMemoryManagement: true,
+    numTestsKeptInMemory: 0,
+    specPattern: './cypress/e2e/**/*.cy.ts',
+    setupNodeEvents(on, config) {
+      esArchiver(on, config);
+    },
+  },
+});
--- a/x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson
+++ b/x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson
@ -0,0 +1,2 @@
+{"_version":"WzI3NywxXQ==","created_at":"2023-08-11T20:32:19.942Z","created_by":"system_indices_superuser","description":"Test exception list description","id":"2b5e0860-3886-11ee-9d6c-ddd8321de1dc","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"ecba8bb0-6ec2-48a0-96f3-c791bd00a338","type":"detection","updated_at":"2023-08-11T20:32:19.944Z","updated_by":"system_indices_superuser","version":1}
+{"exported_exception_list_count":1,"exported_exception_list_item_count":0,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0}
--- a/x-pack/plugins/security_solution/cypress/integration/cases/attach_timeline.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/cases/attach_timeline.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/cases/connector_options.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/cases/connector_options.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/cases/connectors.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/cases/connectors.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/cases/creation.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/cases/creation.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/cases/privileges.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/cases/privileges.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/data_sources/sourcerer.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/data_sources/sourcerer.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/acknowledged.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/acknowledged.spec.ts
@ -29,7 +29,8 @@ import { refreshPage } from '../../tasks/security_header';

 import { ALERTS_URL } from '../../urls/navigation';

-describe('Marking alerts as acknowledged', () => {
+// Also skipped on main branch
+describe.skip('Marking alerts as acknowledged', () => {
  beforeEach(() => {
    cleanKibana();
    loginAndWaitForPage(ALERTS_URL);
@ -69,7 +70,8 @@ describe('Marking alerts as acknowledged', () => {
  });
 });

-describe('Marking alerts as acknowledged with read only role', () => {
+// Also skipped on main branch
+describe.skip('Marking alerts as acknowledged with read only role', () => {
  beforeEach(() => {
    cleanKibana();
    loginAndWaitForPage(ALERTS_URL, ROLES.t2_analyst);
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_detection_callouts_index_outdated.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_detection_callouts_index_outdated.spec.ts
@ -19,9 +19,9 @@ import { waitForAlertsIndexToBeCreated } from '../../tasks/alerts';
 import { goToRuleDetails } from '../../tasks/alerts_detection_rules';
 import { createCustomRule, deleteCustomRule } from '../../tasks/api_calls/rules';
 import { getCallOut, waitForCallOutToBeShown } from '../../tasks/common/callouts';
-import { cleanKibana } from '../../tasks/common';

 const loadPageAsPlatformEngineerUser = (url: string) => {
+  login(ROLES.soc_manager);
  waitForPageWithoutDateRange(url, ROLES.soc_manager);
  waitForPageTitleToBeShown();
 };
@ -36,12 +36,9 @@ describe('Detections > Need Admin Callouts indicating an admin is needed to migr
  before(() => {
    // First, we have to open the app on behalf of a privileged user in order to initialize it.
    // Otherwise the app will be disabled and show a "welcome"-like page.
-    cleanKibana();
-    loginAndWaitForPageWithoutDateRange(ALERTS_URL, ROLES.platform_engineer);
+    loginAndWaitForPageWithoutDateRange(ALERTS_URL);
    waitForAlertsIndexToBeCreated();
-
-    // After that we can login as a soc manager.
-    login(ROLES.soc_manager);
+    waitForPageTitleToBeShown();
  });

  context(
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts
@ -25,8 +25,11 @@ const loadDetectionsPage = (role: ROLES) => {

 describe('Alerts timeline', () => {
  before(() => {
-    // First we login as a privileged user to create alerts.
    cleanKibana();
+  });
+
+  beforeEach(() => {
+    // First we login as a privileged user to create alerts.
    loginAndWaitForPage(ALERTS_URL, ROLES.platform_engineer);
    waitForAlertsPanelToBeLoaded();
    waitForAlertsIndexToBeCreated();
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/building_block_alerts.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/cti_enrichments.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/missing_privileges_callout.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/missing_privileges_callout.spec.ts
@ -19,14 +19,15 @@ import { waitForAlertsIndexToBeCreated } from '../../tasks/alerts';
 import { goToRuleDetails } from '../../tasks/alerts_detection_rules';
 import { createCustomRule, deleteCustomRule } from '../../tasks/api_calls/rules';
 import { getCallOut, waitForCallOutToBeShown, dismissCallOut } from '../../tasks/common/callouts';
-import { cleanKibana } from '../../tasks/common';

 const loadPageAsReadOnlyUser = (url: string) => {
+  login(ROLES.reader);
  waitForPageWithoutDateRange(url, ROLES.reader);
  waitForPageTitleToBeShown();
 };

 const loadPageAsPlatformEngineer = (url: string) => {
+  login(ROLES.platform_engineer);
  waitForPageWithoutDateRange(url, ROLES.platform_engineer);
  waitForPageTitleToBeShown();
 };
@ -46,12 +47,9 @@ describe('Detections > Callouts', () => {
  before(() => {
    // First, we have to open the app on behalf of a privileged user in order to initialize it.
    // Otherwise the app will be disabled and show a "welcome"-like page.
-    cleanKibana();
    loginAndWaitForPageWithoutDateRange(ALERTS_URL, ROLES.platform_engineer);
+    waitForPageTitleToBeShown();
    waitForAlertsIndexToBeCreated();
-
-    // After that we can login as a read-only user.
-    login(ROLES.reader);
  });

  context('indicating read-only access to resources', () => {
@ -135,6 +133,7 @@ describe('Detections > Callouts', () => {

    context('On Rules Management page', () => {
      beforeEach(() => {
+        login(ROLES.platform_engineer);
        loadPageAsPlatformEngineer(DETECTIONS_RULE_MANAGEMENT_URL);
      });

--- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts
@ -306,6 +306,7 @@ describe('Custom detection rules deletion and edition', () => {

          deleteRuleFromDetailsPage();

+          // @ts-expect-error
          cy.waitFor('@deleteRule').then(() => {
            cy.get(RULES_TABLE).should('exist');
            cy.get(RULES_TABLE).then(($table) => {
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/event_correlation_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/event_correlation_rule.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/export_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/export_rule.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/links.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/links.spec.ts
--- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts
--- a/Show more
+++ b/Show more