[ftr] add first-class support for playwrite journeys (#140680)

* [ftr] add first-class support for playwrite journeys

* [CI] Auto-commit changed files from 'node scripts/generate codeowners'

* fix jest test

* remove ability to customize kibana server args, if we need it we can add it back

* remove dev dir that doesn't exist

* fix typo

* prevent duplicated array converstion logic by sharing flag reader

* remove destructuring of option

* fix scalability config and config_path import

* fix start_servers args and tests

* include simple readme

* fix jest tests and support build re-use when changes are just to jest tests

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/ftr_configs.yml

@@ -56,7 +56,7 @@ disabled:
   - x-pack/test/fleet_packages/config.ts
 
   # Scalability testing config that we run in its own pipeline
-  - x-pack/test/performance/scalability/config.ts
+  - x-pack/test/scalability/config.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:
@@ -267,10 +267,10 @@ enabled:
   - x-pack/test/ui_capabilities/spaces_only/config.ts
   - x-pack/test/upgrade_assistant_integration/config.js
   - x-pack/test/usage_collection/config.ts
-  - x-pack/test/performance/journeys/ecommerce_dashboard/config.ts
+  - x-pack/performance/journeys/ecommerce_dashboard.ts
-  - x-pack/test/performance/journeys/flight_dashboard/config.ts
+  - x-pack/performance/journeys/flight_dashboard.ts
-  - x-pack/test/performance/journeys/login/config.ts
+  - x-pack/performance/journeys/login.ts
-  - x-pack/test/performance/journeys/many_fields_discover/config.ts
+  - x-pack/performance/journeys/many_fields_discover.ts
-  - x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts
+  - x-pack/performance/journeys/promotion_tracking_dashboard.ts
-  - x-pack/test/performance/journeys/web_logs_dashboard/config.ts
+  - x-pack/performance/journeys/web_logs_dashboard.ts
-  - x-pack/test/performance/journeys/data_stress_test_lens/config.ts
+  - x-pack/performance/journeys/data_stress_test_lens.ts

.buildkite/pipelines/performance/daily.yml

@@ -1,19 +1,19 @@
 steps:
-  - label: ':male-mechanic::skin-tone-2: Pre-Build'
+  - label: '👨‍🔧 Pre-Build'
     command: .buildkite/scripts/lifecycle/pre_build.sh
     agents:
       queue: kibana-default
 
   - wait
 
-  - label: ':factory_worker: Build Kibana Distribution and Plugins'
+  - label: '🧑‍🏭 Build Kibana Distribution and Plugins'
     command: .buildkite/scripts/steps/build_kibana.sh
     agents:
       queue: c2-16
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
-  - label: ':muscle: Performance Tests with Playwright config'
+  - label: '💪 Performance Tests with Playwright config'
     command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
       queue: kb-static-ubuntu
@@ -21,13 +21,13 @@ steps:
     key: tests
     timeout_in_minutes: 60
 
-  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+  - label: '🚢 Performance Tests dataset extraction for scalability benchmarking'
     command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
     agents:
       queue: n2-2
     depends_on: tests
 
-  - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats'
+  - label: '📈 Report performance metrics to ci-stats'
     command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
     agents:
       queue: n2-2
@@ -36,7 +36,7 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: ':male_superhero::skin-tone-2: Post-Build'
+  - label: '🦸 Post-Build'
     command: .buildkite/scripts/lifecycle/post_build.sh
     agents:
       queue: kibana-default

.buildkite/pull_requests.json

@@ -42,7 +42,12 @@
       "kibana_versions_check": true,
       "kibana_build_reuse": true,
       "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge"],
-      "kibana_build_reuse_regexes": ["^test/", "^x-pack/test/"]
+      "kibana_build_reuse_regexes": [
+        "^test/",
+        "^x-pack/test/",
+        "/__snapshots__/",
+        "\\.test\\.(ts|tsx|js|jsx)"
+      ]
     }
   ]
 }

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -4,16 +4,33 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
+is_test_execution_step
+
 .buildkite/scripts/bootstrap.sh
 # These tests are running on static workers so we have to make sure we delete previous build of Kibana
 rm -rf "$KIBANA_BUILD_LOCATION"
 .buildkite/scripts/download_build_artifacts.sh
 
-echo --- Run Performance Tests with Playwright config
+echo "--- 🦺 Starting Elasticsearch"
 
 node scripts/es snapshot&
+export esPid=$!
+trap 'kill ${esPid}' EXIT
 
-esPid=$!
+export TEST_ES_URL=http://elastic:changeme@localhost:9200
+export TEST_ES_DISABLE_STARTUP=true
+
+# Pings the es server every second for up to 2 minutes until it is green
+curl \
+  --fail \
+  --silent \
+  --retry 120 \
+  --retry-delay 1 \
+  --retry-connrefused \
+  -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" \
+  > /dev/null
+
+echo "✅ ES is ready and will continue to run in the background"
 
 # unset env vars defined in other parts of CI for automatic APM collection of
 # Kibana. We manage APM config in our FTR config and performance service, and
@@ -29,39 +46,27 @@ unset ELASTIC_APM_SERVER_URL
 unset ELASTIC_APM_SECRET_TOKEN
 unset ELASTIC_APM_GLOBAL_LABELS
 
+for journey in x-pack/performance/journeys/*; do
+  set +e
 
-export TEST_ES_URL=http://elastic:changeme@localhost:9200
+  phases=("WARMUP" "TEST")
-export TEST_ES_DISABLE_STARTUP=true
+  for phase in "${phases[@]}"; do
+    echo "--- $journey - $phase"
 
-# Pings the es server every seconds 2 mins until it is status is green
+    export TEST_PERFORMANCE_PHASE="$phase"
-curl --retry 120 \
+    node scripts/functional_tests \
-  --retry-delay 1 \
+      --config "$journey" \
-  --retry-all-errors \
+      --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  -I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow"
+      --debug \
+      --bail
 
-journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover" "data_stress_test_lens")
+    status=$?
+    if [ $status -ne 0 ]; then
+      echo "^^^ +++"
+      echo "❌ FTR failed with status code: $status"
+      exit 1
+    fi
+  done
 
-for i in "${journeys[@]}"; do
+  set -e
-    echo "JOURNEY[${i}] is running"
-
-    export TEST_PERFORMANCE_PHASE=WARMUP
-    export JOURNEY_NAME="${i}"
-
-    checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \
-      node scripts/functional_tests \
-        --config "x-pack/test/performance/journeys/${i}/config.ts" \
-        --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-        --debug \
-        --bail
-
-    export TEST_PERFORMANCE_PHASE=TEST
-
-    checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: TEST)" \
-      node scripts/functional_tests \
-        --config "x-pack/test/performance/journeys/${i}/config.ts" \
-        --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-        --debug \
-        --bail
 done
-
-kill "$esPid"

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -15,19 +15,16 @@ OUTPUT_DIR="${KIBANA_DIR}/${OUTPUT_REL}"
 .buildkite/scripts/bootstrap.sh
 
 echo "--- Extract APM metrics"
-scalabilityJourneys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
+for journey in x-pack/performance/journeys/*; do
+  echo "Looking for journey=${journey} and BUILD_ID=${BUILD_ID} in APM traces"
 
-for i in "${scalabilityJourneys[@]}"; do
+  node scripts/extract_performance_testing_dataset \
-    JOURNEY_NAME="${i}"
+    --config "${journey}" \
-    echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"
+    --buildId "${BUILD_ID}" \
-
+    --es-url "${ES_SERVER_URL}" \
-    node scripts/extract_performance_testing_dataset \
+    --es-username "${USER_FROM_VAULT}" \
-        --config "x-pack/test/performance/journeys/${i}/config.ts" \
+    --es-password "${PASS_FROM_VAULT}" \
-        --buildId "${BUILD_ID}" \
+    --without-static-resources
-        --es-url "${ES_SERVER_URL}" \
-        --es-username "${USER_FROM_VAULT}" \
-        --es-password "${PASS_FROM_VAULT}" \
-        --without-static-resources
 done
 
 echo "--- Creating scalability dataset in ${OUTPUT_REL}"

.eslintrc.js

@@ -178,6 +178,7 @@ const DEV_PATTERNS = [
   'x-pack/{dev-tools,tasks,scripts,test,build_chromium}/**/*',
   'x-pack/plugins/*/server/scripts/**/*',
   'x-pack/plugins/fleet/cypress',
+  'x-pack/performance/**/*',
 ];
 
 /** Restricted imports with suggested alternatives */

.github/CODEOWNERS

@@ -861,9 +861,12 @@ packages/kbn-eslint-plugin-disable @elastic/kibana-operations
 packages/kbn-eslint-plugin-eslint @elastic/kibana-operations
 packages/kbn-eslint-plugin-imports @elastic/kibana-operations
 packages/kbn-expect @elastic/kibana-operations
+packages/kbn-failed-test-reporter-cli @elastic/kibana-operations
 packages/kbn-field-types @elastic/kibana-app-services
 packages/kbn-find-used-node-modules @elastic/kibana-operations
 packages/kbn-flot-charts @elastic/kibana-operations
+packages/kbn-ftr-common-functional-services @elastic/kibana-operations
+packages/kbn-ftr-screenshot-filename @elastic/kibana-operations
 packages/kbn-generate @elastic/kibana-operations
 packages/kbn-get-repo-files @elastic/kibana-operations
 packages/kbn-handlebars @elastic/kibana-security
@@ -874,6 +877,7 @@ packages/kbn-import-resolver @elastic/kibana-operations
 packages/kbn-interpreter @elastic/kibana-app-services
 packages/kbn-io-ts-utils @elastic/apm-ui
 packages/kbn-jest-serializers @elastic/kibana-operations
+packages/kbn-journeys @elastic/kibana-operations
 packages/kbn-kibana-manifest-schema @elastic/kibana-operations
 packages/kbn-logging @elastic/kibana-core
 packages/kbn-logging-mocks @elastic/kibana-core

dev_docs/operations/operations_landing.mdx

@@ -26,6 +26,7 @@ layout: landing
     { pageId: "kibDevDocsOpsWritingStableFunctionalTests" },
     { pageId: "kibDevDocsOpsFlakyTestRunner" },
     { pageId: "kibDevDocsOpsCiStats" },
+    { pageId: "kibDevDocsOpsJourneys" },
   ]}
 />
 

package.json

@@ -702,11 +702,15 @@
     "@kbn/eslint-plugin-eslint": "link:bazel-bin/packages/kbn-eslint-plugin-eslint",
     "@kbn/eslint-plugin-imports": "link:bazel-bin/packages/kbn-eslint-plugin-imports",
     "@kbn/expect": "link:bazel-bin/packages/kbn-expect",
+    "@kbn/failed-test-reporter-cli": "link:bazel-bin/packages/kbn-failed-test-reporter-cli",
     "@kbn/find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules",
+    "@kbn/ftr-common-functional-services": "link:bazel-bin/packages/kbn-ftr-common-functional-services",
+    "@kbn/ftr-screenshot-filename": "link:bazel-bin/packages/kbn-ftr-screenshot-filename",
     "@kbn/generate": "link:bazel-bin/packages/kbn-generate",
     "@kbn/get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files",
     "@kbn/import-resolver": "link:bazel-bin/packages/kbn-import-resolver",
     "@kbn/jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers",
+    "@kbn/journeys": "link:bazel-bin/packages/kbn-journeys",
     "@kbn/kibana-manifest-schema": "link:bazel-bin/packages/kbn-kibana-manifest-schema",
     "@kbn/managed-vscode-config": "link:bazel-bin/packages/kbn-managed-vscode-config",
     "@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli",
@@ -1017,8 +1021,11 @@
     "@types/kbn__es-types": "link:bazel-bin/packages/kbn-es-types/npm_module_types",
     "@types/kbn__eslint-plugin-disable": "link:bazel-bin/packages/kbn-eslint-plugin-disable/npm_module_types",
     "@types/kbn__eslint-plugin-imports": "link:bazel-bin/packages/kbn-eslint-plugin-imports/npm_module_types",
+    "@types/kbn__failed-test-reporter-cli": "link:bazel-bin/packages/kbn-failed-test-reporter-cli/npm_module_types",
     "@types/kbn__field-types": "link:bazel-bin/packages/kbn-field-types/npm_module_types",
     "@types/kbn__find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules/npm_module_types",
+    "@types/kbn__ftr-common-functional-services": "link:bazel-bin/packages/kbn-ftr-common-functional-services/npm_module_types",
+    "@types/kbn__ftr-screenshot-filename": "link:bazel-bin/packages/kbn-ftr-screenshot-filename/npm_module_types",
     "@types/kbn__generate": "link:bazel-bin/packages/kbn-generate/npm_module_types",
     "@types/kbn__get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files/npm_module_types",
     "@types/kbn__handlebars": "link:bazel-bin/packages/kbn-handlebars/npm_module_types",
@@ -1032,6 +1039,7 @@
     "@types/kbn__interpreter": "link:bazel-bin/packages/kbn-interpreter/npm_module_types",
     "@types/kbn__io-ts-utils": "link:bazel-bin/packages/kbn-io-ts-utils/npm_module_types",
     "@types/kbn__jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers/npm_module_types",
+    "@types/kbn__journeys": "link:bazel-bin/packages/kbn-journeys/npm_module_types",
     "@types/kbn__kbn-ci-stats-performance-metrics": "link:bazel-bin/packages/kbn-kbn-ci-stats-performance-metrics/npm_module_types",
     "@types/kbn__kibana-manifest-schema": "link:bazel-bin/packages/kbn-kibana-manifest-schema/npm_module_types",
     "@types/kbn__logging": "link:bazel-bin/packages/kbn-logging/npm_module_types",

packages/BUILD.bazel

@@ -204,9 +204,12 @@ filegroup(
     "//packages/kbn-eslint-plugin-eslint:build",
     "//packages/kbn-eslint-plugin-imports:build",
     "//packages/kbn-expect:build",
+    "//packages/kbn-failed-test-reporter-cli:build",
     "//packages/kbn-field-types:build",
     "//packages/kbn-find-used-node-modules:build",
     "//packages/kbn-flot-charts:build",
+    "//packages/kbn-ftr-common-functional-services:build",
+    "//packages/kbn-ftr-screenshot-filename:build",
     "//packages/kbn-generate:build",
     "//packages/kbn-get-repo-files:build",
     "//packages/kbn-handlebars:build",
@@ -217,6 +220,7 @@ filegroup(
     "//packages/kbn-interpreter:build",
     "//packages/kbn-io-ts-utils:build",
     "//packages/kbn-jest-serializers:build",
+    "//packages/kbn-journeys:build",
     "//packages/kbn-kibana-manifest-schema:build",
     "//packages/kbn-logging:build",
     "//packages/kbn-logging-mocks:build",
@@ -514,8 +518,11 @@ filegroup(
     "//packages/kbn-es-types:build_types",
     "//packages/kbn-eslint-plugin-disable:build_types",
     "//packages/kbn-eslint-plugin-imports:build_types",
+    "//packages/kbn-failed-test-reporter-cli:build_types",
     "//packages/kbn-field-types:build_types",
     "//packages/kbn-find-used-node-modules:build_types",
+    "//packages/kbn-ftr-common-functional-services:build_types",
+    "//packages/kbn-ftr-screenshot-filename:build_types",
     "//packages/kbn-generate:build_types",
     "//packages/kbn-get-repo-files:build_types",
     "//packages/kbn-handlebars:build_types",
@@ -526,6 +533,7 @@ filegroup(
     "//packages/kbn-interpreter:build_types",
     "//packages/kbn-io-ts-utils:build_types",
     "//packages/kbn-jest-serializers:build_types",
+    "//packages/kbn-journeys:build_types",
     "//packages/kbn-kibana-manifest-schema:build_types",
     "//packages/kbn-logging:build_types",
     "//packages/kbn-logging-mocks:build_types",

packages/kbn-dev-cli-runner/index.ts

@@ -9,4 +9,5 @@
 export * from './src/run';
 export * from './src/run_with_commands';
 export * from './src/flags';
+export * from './src/flags_reader';
 export type { CleanupTask } from './src/cleanup';

packages/kbn-dev-cli-runner/src/flags.ts

@@ -53,6 +53,10 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
   };
 }
 
+export const DEFAULT_FLAG_ALIASES = {
+  v: 'verbose',
+};
+
 export function getFlags(
   argv: string[],
   flagOptions: RunOptions['flags'] = {},
@@ -67,7 +71,7 @@ export function getFlags(
     boolean: [...(flagOptions.boolean || []), ...logLevelFlags, 'help'],
     alias: {
       ...flagOptions.alias,
-      v: 'verbose',
+      ...DEFAULT_FLAG_ALIASES,
     },
     default: flagOptions.default,
     unknown: (name: string) => {

packages/kbn-dev-cli-runner/src/flags_reader.test.ts

@@ -0,0 +1,344 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createAbsolutePathSerializer } from '@kbn/jest-serializers';
+
+import { getFlags } from './flags';
+import { FlagsReader } from './flags_reader';
+
+const FLAGS = {
+  string: 'string',
+  astring: ['foo', 'bar'],
+  num: '1234',
+  bool: true,
+  missing: undefined,
+};
+
+const basic = new FlagsReader(FLAGS);
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
+
+describe('#string()', () => {
+  it('returns a single string, regardless of flag count', () => {
+    expect(basic.string('string')).toMatchInlineSnapshot(`"string"`);
+    expect(basic.string('astring')).toBe(FLAGS.astring.at(-1));
+  });
+
+  it('returns undefined when flag is missing', () => {
+    expect(basic.string('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  it('throws for non-string flags', () => {
+    expect(() => basic.string('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws when flag is missing', () => {
+      expect(() => basic.requiredString('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#arrayOfStrings()', () => {
+  it('returns an array of strings for string flags, regardless of count', () => {
+    expect(basic.arrayOfStrings('string')).toMatchInlineSnapshot(`
+      Array [
+        "string",
+      ]
+    `);
+    expect(basic.arrayOfStrings('astring')).toMatchInlineSnapshot(`
+      Array [
+        "foo",
+        "bar",
+      ]
+    `);
+  });
+
+  it('returns undefined when flag is missing', () => {
+    expect(basic.arrayOfStrings('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  it('throws for non-string flags', () => {
+    expect(() => basic.arrayOfStrings('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws when flag is missing', () => {
+      expect(() => basic.requiredArrayOfStrings('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#enum()', () => {
+  it('validates that values match options', () => {
+    expect(basic.enum('string', ['a', 'string', 'b'])).toMatchInlineSnapshot(`"string"`);
+    expect(basic.enum('missing', ['a', 'b'])).toMatchInlineSnapshot(`undefined`);
+    expect(() => basic.enum('string', ['a', 'b'])).toThrowErrorMatchingInlineSnapshot(
+      `"invalid --string, expected one of \\"a\\", \\"b\\""`
+    );
+  });
+});
+
+describe('#path()', () => {
+  it('parses the string to an absolute path based on CWD', () => {
+    expect(basic.path('string')).toMatchInlineSnapshot(`<absolute path>/string`);
+    expect(basic.path('missing')).toMatchInlineSnapshot(`undefined`);
+  });
+
+  describe('required version', () => {
+    it('throws if the flag is missing', () => {
+      expect(() => basic.requiredPath('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+
+  describe('array version', () => {
+    it('parses a list of paths', () => {
+      expect(basic.arrayOfPaths('astring')).toMatchInlineSnapshot(`
+        Array [
+          <absolute path>/foo,
+          <absolute path>/bar,
+        ]
+      `);
+    });
+
+    describe('required version', () => {
+      it('throws if the flag is missing', () => {
+        expect(() => basic.requiredArrayOfPaths('missing')).toThrowErrorMatchingInlineSnapshot(
+          `"missing required flag --missing"`
+        );
+      });
+    });
+  });
+});
+
+describe('#number()', () => {
+  it('parses strings as numbers', () => {
+    expect(basic.number('num')).toMatchInlineSnapshot(`1234`);
+    expect(basic.number('missing')).toMatchInlineSnapshot(`undefined`);
+    expect(() => basic.number('bool')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --bool to be a string"`
+    );
+    expect(() => basic.number('string')).toThrowErrorMatchingInlineSnapshot(
+      `"unable to parse --string value [string] as a number"`
+    );
+    expect(() => basic.number('astring')).toThrowErrorMatchingInlineSnapshot(
+      `"unable to parse --astring value [bar] as a number"`
+    );
+  });
+
+  describe('required version', () => {
+    it('throws if the flag is missing', () => {
+      expect(() => basic.requiredNumber('missing')).toThrowErrorMatchingInlineSnapshot(
+        `"missing required flag --missing"`
+      );
+    });
+  });
+});
+
+describe('#boolean()', () => {
+  it('ensures flag is boolean, requires value', () => {
+    expect(basic.boolean('bool')).toMatchInlineSnapshot(`true`);
+    expect(() => basic.boolean('missing')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --missing to be a boolean"`
+    );
+    expect(() => basic.boolean('string')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --string to be a boolean"`
+    );
+    expect(() => basic.boolean('astring')).toThrowErrorMatchingInlineSnapshot(
+      `"expected --astring to be a boolean"`
+    );
+  });
+});
+
+describe('#getPositionals()', () => {
+  it('returns all positional arguments in flags', () => {
+    const flags = new FlagsReader({
+      ...FLAGS,
+      _: ['a', 'b', 'c'],
+    });
+
+    expect(flags.getPositionals()).toMatchInlineSnapshot(`
+      Array [
+        "a",
+        "b",
+        "c",
+      ]
+    `);
+  });
+
+  it('handles missing _ flag', () => {
+    const flags = new FlagsReader({});
+    expect(flags.getPositionals()).toMatchInlineSnapshot(`Array []`);
+  });
+});
+
+describe('#getUnused()', () => {
+  it('returns a map of all unused flags', () => {
+    const flags = new FlagsReader({
+      a: '1',
+      b: '2',
+      c: '3',
+    });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+        "b" => "2",
+        "c" => "3",
+      }
+    `);
+
+    flags.number('a');
+    flags.number('b');
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "c" => "3",
+      }
+    `);
+  });
+
+  it('ignores the default flags which are forced on commands', () => {
+    const rawFlags = getFlags(['--a=1'], {
+      string: ['a'],
+    });
+
+    const flags = new FlagsReader(rawFlags, {
+      aliases: {
+        v: 'verbose',
+      },
+    });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+      }
+    `);
+    flags.number('a');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+
+  it('treats aliased flags as used', () => {
+    const flags = new FlagsReader(
+      {
+        f: true,
+        force: true,
+        v: true,
+        verbose: true,
+      },
+      {
+        aliases: {
+          f: 'force',
+          v: 'verbose',
+        },
+      }
+    );
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "f" => true,
+        "force" => true,
+      }
+    `);
+    flags.boolean('force');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+    flags.boolean('v');
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+
+  it('treats failed reads as "uses"', () => {
+    const flags = new FlagsReader({ a: 'b' });
+
+    expect(flags.getUnused()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "b",
+      }
+    `);
+    expect(() => flags.number('a')).toThrowError();
+    expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`);
+  });
+});
+
+describe('#getUsed()', () => {
+  it('returns a map of all used flags', () => {
+    const flags = new FlagsReader({
+      a: '1',
+      b: '2',
+      c: '3',
+    });
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+
+    flags.number('a');
+    flags.number('b');
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "1",
+        "b" => "2",
+      }
+    `);
+  });
+
+  it('treats aliases flags as used', () => {
+    const flags = new FlagsReader(
+      {
+        f: true,
+        force: true,
+        v: true,
+        verbose: true,
+      },
+      {
+        aliases: {
+          f: 'force',
+          v: 'verbose',
+        },
+      }
+    );
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+    flags.boolean('force');
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "force" => true,
+        "f" => true,
+      }
+    `);
+    flags.boolean('v');
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "force" => true,
+        "f" => true,
+        "v" => true,
+        "verbose" => true,
+      }
+    `);
+  });
+
+  it('treats failed reads as "uses"', () => {
+    const flags = new FlagsReader({ a: 'b' });
+
+    expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`);
+    expect(() => flags.number('a')).toThrowError();
+    expect(flags.getUsed()).toMatchInlineSnapshot(`
+      Map {
+        "a" => "b",
+      }
+    `);
+  });
+});

packages/kbn-dev-cli-runner/src/flags_reader.ts

@@ -0,0 +1,267 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { LOG_LEVEL_FLAGS } from '@kbn/tooling-log';
+
+type FlagValue = string | string[] | boolean;
+const FORCED_FLAGS = new Set([...LOG_LEVEL_FLAGS.map((l) => l.name), 'help']);
+
+const makeAbsolute = (rel: string) => Path.resolve(process.cwd(), rel);
+
+const nonUndefinedValues = (e: [string, FlagValue | undefined]): e is [string, FlagValue] =>
+  e[1] !== undefined;
+
+export class FlagsReader {
+  private readonly used: Map<string, FlagValue>;
+  private readonly unused: Map<string, FlagValue>;
+  private readonly _: string[];
+  private readonly aliasMap: Map<string, string>;
+
+  constructor(
+    flags: Record<string, FlagValue | undefined>,
+    private readonly opts?: { aliases?: Record<string, string> }
+  ) {
+    this.used = new Map();
+    this.unused = new Map(
+      Object.entries(flags)
+        .filter(nonUndefinedValues)
+        .filter((e) => e[0] !== 'unexpected')
+    );
+    this.aliasMap = new Map(
+      Object.entries(this.opts?.aliases ?? []).flatMap(([a, b]) => [
+        [a, b],
+        [b, a],
+      ])
+    );
+
+    this._ = this.arrayOfStrings('_') ?? [];
+  }
+
+  private use(key: string) {
+    const alias = this.aliasMap.get(key);
+
+    const used = this.used.get(key);
+    if (used !== undefined) {
+      return used;
+    }
+
+    const unused = this.unused.get(key);
+    if (unused !== undefined) {
+      this.used.set(key, unused);
+      this.unused.delete(key);
+
+      if (alias !== undefined) {
+        this.used.set(alias, unused);
+        this.unused.delete(alias);
+      }
+    }
+
+    return unused;
+  }
+
+  /**
+   * Read a string flag that supports multiple instances into an array of strings. If the
+   * flag is only passed once an array with a single item will be returned. If the flag is not
+   * passed then undefined will be returned.
+   */
+  arrayOfStrings(key: string) {
+    const value = this.use(key);
+
+    switch (typeof value) {
+      case 'boolean':
+        throw createFlagError(`expected --${key} to be a string`);
+      case 'string':
+        return value ? [value] : [];
+      default:
+        return value;
+    }
+  }
+
+  /**
+   * Same as #arrayOfStrings() except when the flag is not passed a "flag error" is thrown telling
+   * the user that the flag is required and shows them the help text.
+   */
+  requiredArrayOfStrings(key: string) {
+    const value = this.arrayOfStrings(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Read the value of a string flag. If the flag is passed multiple times the last value is returned. If
+   * the flag is not passed then undefined is returned.
+   */
+  string(key: string) {
+    const value = this.use(key);
+
+    switch (typeof value) {
+      case 'undefined':
+        return undefined;
+      case 'string':
+        return value || undefined; // convert "" to undefined
+      case 'object':
+        const last = value.at(-1);
+        if (last === undefined) {
+          throw createFlagError(`expected --${key} to be a string`);
+        }
+        return last || undefined; // convert "" to undefined
+      default:
+        throw createFlagError(`expected --${key} to be a string`);
+    }
+  }
+
+  /**
+   * Same as #string() except when the flag is passed it is validated against a list
+   * of valid values
+   */
+  enum<T extends string>(key: string, values: readonly T[]) {
+    const value = this.string(key);
+    if (value === undefined) {
+      return;
+    }
+
+    if (values.includes(value as T)) {
+      return value as T;
+    }
+
+    throw createFlagError(`invalid --${key}, expected one of "${values.join('", "')}"`);
+  }
+
+  /**
+   * Same as #string() except when a flag is not passed a "flag error" is thrown telling the user
+   * that the flag is required and shows them the help text.
+   */
+  requiredString(key: string) {
+    const value = this.string(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Same as #string(), except that when there is a value for the string it is resolved to an
+   * absolute path based on the current working directory
+   */
+  path(key: string) {
+    const value = this.string(key);
+    if (value !== undefined) {
+      return makeAbsolute(value);
+    }
+  }
+
+  /**
+   * Same as #requiredString() except that values are converted to absolute paths based on the
+   * current working directory
+   */
+  requiredPath(key: string) {
+    return makeAbsolute(this.requiredString(key));
+  }
+
+  /**
+   * Same as #arrayOfStrings(), except that when there are values they are resolved to
+   * absolute paths based on the current working directory
+   */
+  arrayOfPaths(key: string) {
+    const value = this.arrayOfStrings(key);
+    if (value !== undefined) {
+      return value.map(makeAbsolute);
+    }
+  }
+
+  /**
+   * Same as #requiredArrayOfStrings(), except that values are resolved to absolute paths
+   * based on the current working directory
+   */
+  requiredArrayOfPaths(key: string) {
+    return this.requiredArrayOfStrings(key).map(makeAbsolute);
+  }
+
+  /**
+   * Parsed the provided flag as a number, if the value does not parse to a valid number
+   * using Number.parseFloat() then a "flag error" is thrown. If the flag is not passed
+   * undefined is returned.
+   */
+  number(key: string) {
+    const value = this.string(key);
+    if (value === undefined) {
+      return;
+    }
+
+    const num = Number.parseFloat(value);
+    if (Number.isNaN(num)) {
+      throw createFlagError(`unable to parse --${key} value [${value}] as a number`);
+    }
+
+    return num;
+  }
+
+  /**
+   * Same as #number() except that when the flag is missing a "flag error" is thrown
+   */
+  requiredNumber(key: string) {
+    const value = this.number(key);
+    if (value === undefined) {
+      throw createFlagError(`missing required flag --${key}`);
+    }
+    return value;
+  }
+
+  /**
+   * Read a boolean flag value, if the flag is properly defined as a "boolean" in the run options
+   * then the value will always be a boolean, defaulting to `false`, so there is no need for an
+   * optional/requiredBoolean() method.
+   */
+  boolean(key: string) {
+    const value = this.use(key);
+    if (typeof value !== 'boolean') {
+      throw createFlagError(`expected --${key} to be a boolean`);
+    }
+    return value;
+  }
+
+  /**
+   * Get the positional arguments passed, includes any values that are not associated with
+   * a specific --flag
+   */
+  getPositionals() {
+    return this._.slice(0);
+  }
+
+  /**
+   * Returns all of the unused flags. When a flag is read via any of the key-specific methods
+   * the key is marked as "used" and this method will return a map of just the flags which
+   * have not been used yet (excluding the default flags like --debug, --verbose, and --help)
+   */
+  getUnused() {
+    return new Map(
+      [...this.unused.entries()].filter(([key]) => {
+        const alias = this.aliasMap.get(key);
+        if (alias !== undefined && FORCED_FLAGS.has(alias)) {
+          return false;
+        }
+
+        return !FORCED_FLAGS.has(key);
+      })
+    );
+  }
+
+  /**
+   * Returns all of the used flags. When a flag is read via any of the key-specific methods
+   * the key is marked as "used" and from then on this method will return a map including that
+   * and any other key used by these methods.
+   */
+  getUsed() {
+    return new Map(this.used);
+  }
+}

packages/kbn-dev-cli-runner/src/run.ts

@@ -10,7 +10,8 @@ import { pickLevelFromFlags, ToolingLog, LogLevel } from '@kbn/tooling-log';
 import { ProcRunner, withProcRunner } from '@kbn/dev-proc-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
 
-import { Flags, getFlags, FlagOptions } from './flags';
+import { Flags, getFlags, FlagOptions, DEFAULT_FLAG_ALIASES } from './flags';
+import { FlagsReader } from './flags_reader';
 import { getHelp } from './help';
 import { CleanupTask, Cleanup } from './cleanup';
 import { Metrics, MetricsMeta } from './metrics';
@@ -21,6 +22,7 @@ export interface RunContext {
   procRunner: ProcRunner;
   statsMeta: MetricsMeta;
   addCleanupTask: (task: CleanupTask) => void;
+  flagsReader: FlagsReader;
 }
 export type RunFn = (context: RunContext) => Promise<void> | void;
 
@@ -71,6 +73,12 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
         procRunner,
         statsMeta: metrics.meta,
         addCleanupTask: cleanup.add.bind(cleanup),
+        flagsReader: new FlagsReader(flags, {
+          aliases: {
+            ...options.flags?.alias,
+            ...DEFAULT_FLAG_ALIASES,
+          },
+        }),
       });
     });
   } catch (error) {

packages/kbn-dev-cli-runner/src/run_with_commands.test.ts

@@ -9,6 +9,7 @@
 import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
 import { ProcRunner } from '@kbn/dev-proc-runner';
 
+import { FlagsReader } from './flags_reader';
 import { RunWithCommands } from './run_with_commands';
 
 const testLog = new ToolingLog();
@@ -44,6 +45,7 @@ it('extends the context using extendContext()', async () => {
   expect(context).toEqual({
     log: expect.any(ToolingLog),
     flags: expect.any(Object),
+    flagsReader: expect.any(FlagsReader),
     addCleanupTask: expect.any(Function),
     procRunner: expect.any(ProcRunner),
     statsMeta: expect.any(Map),

packages/kbn-dev-cli-runner/src/run_with_commands.ts

@@ -11,7 +11,8 @@ import { withProcRunner } from '@kbn/dev-proc-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
 
 import { RunContext, RunOptions } from './run';
-import { getFlags, FlagOptions, mergeFlagOptions } from './flags';
+import { getFlags, FlagOptions, mergeFlagOptions, DEFAULT_FLAG_ALIASES } from './flags';
+import { FlagsReader } from './flags_reader';
 import { Cleanup } from './cleanup';
 import { getHelpForAllCommands, getCommandLevelHelp } from './help';
 import { Metrics } from './metrics';
@@ -116,6 +117,12 @@ export class RunWithCommands<T> {
           procRunner,
           statsMeta: metrics.meta,
           addCleanupTask: cleanup.add.bind(cleanup),
+          flagsReader: new FlagsReader(commandFlags, {
+            aliases: {
+              ...commandFlagOptions.alias,
+              ...DEFAULT_FLAG_ALIASES,
+            },
+          }),
         };
 
         const extendedContext = {

packages/kbn-failed-test-reporter-cli/BUILD.bazel

@@ -0,0 +1,145 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-failed-test-reporter-cli"
+PKG_REQUIRE_NAME = "@kbn/failed-test-reporter-cli"
+
+SOURCE_FILES = glob(
+  [
+    "**/*.ts",
+    "**/*.html",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "//packages/kbn-utils:npm_module_types",
+  "//packages/kbn-ci-stats-reporter:npm_module_types",
+  "//packages/kbn-dev-cli-runner:npm_module_types",
+  "//packages/kbn-dev-cli-errors:npm_module_types",
+  "//packages/kbn-dev-utils:npm_module_types",
+  "//packages/kbn-tooling-log:npm_module_types",
+  "//packages/kbn-ftr-screenshot-filename:npm_module_types",
+  "//packages/kbn-jest-serializers:npm_module_types",
+  "//packages/kbn-journeys:npm_module_types",
+  "@npm//@elastic/elasticsearch",
+  "@npm//@types/node",
+  "@npm//@types/he",
+  "@npm//@types/jest",
+  "@npm//@types/strip-ansi",
+  "@npm//@types/normalize-path",
+  "@npm//@types/xml2js",
+  "@npm//axios",
+  "@npm//dedent",
+  "@npm//globby",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+  additional_args = [
+    "--copy-files"
+  ],
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-failed-test-reporter-cli/README.md

@@ -0,0 +1,3 @@
+# @kbn/failed-test-reporter-cli
+
+Empty package generated by @kbn/generate

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts

@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { REPO_ROOT } from '@kbn/utils';
+import { run } from '@kbn/dev-cli-runner';
+import { createFailError, createFlagError } from '@kbn/dev-cli-errors';
+import { CiStatsReporter } from '@kbn/ci-stats-reporter';
+import globby from 'globby';
+import normalize from 'normalize-path';
+
+import { getFailures } from './get_failures';
+import { GithubApi } from './github_api';
+import { updateFailureIssue, createFailureIssue } from './report_failure';
+import { readTestReport, getRootMetadata } from './test_report';
+import { addMessagesToReport } from './add_messages_to_report';
+import { getReportMessageIter } from './report_metadata';
+import { reportFailuresToEs } from './report_failures_to_es';
+import { reportFailuresToFile } from './report_failures_to_file';
+import { getBuildkiteMetadata } from './buildkite_metadata';
+import { ExistingFailedTestIssues } from './existing_failed_test_issues';
+
+const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
+const DISABLE_MISSING_TEST_REPORT_ERRORS =
+  process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true';
+
+run(
+  async ({ log, flags }) => {
+    const indexInEs = flags['index-errors'];
+
+    let updateGithub = flags['github-update'];
+    if (updateGithub && !process.env.GITHUB_TOKEN) {
+      throw createFailError(
+        'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag'
+      );
+    }
+
+    let branch: string = '';
+    if (updateGithub) {
+      let isPr = false;
+
+      if (process.env.BUILDKITE === 'true') {
+        branch = process.env.BUILDKITE_BRANCH || '';
+        isPr = process.env.BUILDKITE_PULL_REQUEST === 'true';
+        updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true';
+      } else {
+        // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
+        const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
+        branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || '';
+        isPr = !!process.env.ghprbPullId;
+
+        const isMainOrVersion = branch === 'main' || branch.match(/^\d+\.(x|\d+)$/);
+        if (!isMainOrVersion || isPr) {
+          log.info('Failure issues only created on main/version branch jobs');
+          updateGithub = false;
+        }
+      }
+
+      if (!branch) {
+        throw createFailError(
+          'Unable to determine originating branch from job name or other environment variables'
+        );
+      }
+    }
+
+    const githubApi = new GithubApi({
+      log,
+      token: process.env.GITHUB_TOKEN,
+      dryRun: !updateGithub,
+    });
+
+    const bkMeta = getBuildkiteMetadata();
+
+    try {
+      const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
+      if (typeof buildUrl !== 'string' || !buildUrl) {
+        throw createFlagError('Missing --build-url or process.env.BUILD_URL');
+      }
+
+      const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
+        normalize(Path.resolve(p))
+      );
+      log.info('Searching for reports at', patterns);
+      const reportPaths = await globby(patterns, {
+        absolute: true,
+      });
+
+      if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) {
+        // it is fine for code coverage to not have test results
+        return;
+      }
+
+      if (!reportPaths.length) {
+        throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
+      }
+
+      log.info('found', reportPaths.length, 'junit reports', reportPaths);
+
+      const existingIssues = new ExistingFailedTestIssues(log);
+      for (const reportPath of reportPaths) {
+        const report = await readTestReport(reportPath);
+        const messages = Array.from(getReportMessageIter(report));
+        const failures = getFailures(report);
+
+        await existingIssues.loadForFailures(failures);
+
+        if (indexInEs) {
+          await reportFailuresToEs(log, failures);
+        }
+
+        for (const failure of failures) {
+          const pushMessage = (msg: string) => {
+            messages.push({
+              classname: failure.classname,
+              name: failure.name,
+              message: msg,
+            });
+          };
+
+          if (failure.likelyIrrelevant) {
+            pushMessage(
+              'Failure is likely irrelevant' +
+                (updateGithub ? ', so an issue was not created or updated' : '')
+            );
+            continue;
+          }
+
+          const existingIssue = existingIssues.getForFailure(failure);
+          if (existingIssue) {
+            const { newBody, newCount } = await updateFailureIssue(
+              buildUrl,
+              existingIssue,
+              githubApi,
+              branch
+            );
+            const url = existingIssue.github.htmlUrl;
+            existingIssue.github.body = newBody;
+            failure.githubIssue = url;
+            failure.failureCount = updateGithub ? newCount : newCount - 1;
+            pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`);
+            if (updateGithub) {
+              pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`);
+            }
+            continue;
+          }
+
+          const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
+          existingIssues.addNewlyCreated(failure, newIssue);
+          pushMessage('Test has not failed recently on tracked branches');
+          if (updateGithub) {
+            pushMessage(`Created new issue: ${newIssue.html_url}`);
+            failure.githubIssue = newIssue.html_url;
+          }
+          failure.failureCount = updateGithub ? 1 : 0;
+        }
+
+        // mutates report to include messages and writes updated report to disk
+        await addMessagesToReport({
+          report,
+          messages,
+          log,
+          reportPath,
+          dryRun: !flags['report-update'],
+        });
+
+        await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report));
+      }
+    } finally {
+      await CiStatsReporter.fromEnv(log).metrics([
+        {
+          group: 'github api request count',
+          id: `failed test reporter`,
+          value: githubApi.getRequestCount(),
+          meta: Object.fromEntries(
+            Object.entries(bkMeta).map(
+              ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const
+            )
+          ),
+        },
+      ]);
+    }
+  },
+  {
+    description: `a cli that opens issues or updates existing issues based on junit reports`,
+    flags: {
+      boolean: ['github-update', 'report-update'],
+      string: ['build-url'],
+      default: {
+        'github-update': true,
+        'report-update': true,
+        'index-errors': true,
+        'build-url': process.env.BUILD_URL,
+      },
+      help: `
+        --no-github-update Execute the CLI without writing to Github
+        --no-report-update Execute the CLI without writing to the JUnit reports
+        --no-index-errors  Execute the CLI without indexing failures into Elasticsearch
+        --build-url        URL of the failed build, defaults to process.env.BUILD_URL
+      `,
+    },
+  }
+);

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts

@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+import { createHash } from 'crypto';
+
+import globby from 'globby';
+import { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/utils';
+import { escape } from 'he';
+import { FtrScreenshotFilename } from '@kbn/ftr-screenshot-filename';
+import { JourneyScreenshots } from '@kbn/journeys';
+
+import { BuildkiteMetadata } from './buildkite_metadata';
+import { TestFailure } from './get_failures';
+
+interface JourneyMeta {
+  journeyName: string;
+}
+function getJourneyMetadata(rootMeta: Record<string, unknown>): JourneyMeta | undefined {
+  const { journeyName } = rootMeta;
+  if (typeof journeyName === 'string') {
+    return { journeyName };
+  }
+
+  return undefined;
+}
+
+async function getJourneySnapshotHtml(log: ToolingLog, journeyMeta: JourneyMeta) {
+  let screenshots;
+  try {
+    screenshots = await JourneyScreenshots.load(journeyMeta.journeyName);
+  } catch (error) {
+    log.error(`Failed to load journey screenshots: ${error.message}`);
+    return '';
+  }
+
+  return [
+    '<section>',
+    '<h5>Steps</h5>',
+    ...screenshots.get().flatMap(({ title, path }) => {
+      const base64 = Fs.readFileSync(path, 'base64');
+
+      return [
+        `<p><strong>${escape(title)}</strong></p>`,
+        `<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${base64}" />`,
+      ];
+    }),
+    '</section>',
+  ].join('\n');
+}
+
+let _allScreenshotsCache: Array<{ path: string; name: string }> | undefined;
+function getAllScreenshots(log: ToolingLog) {
+  return (_allScreenshotsCache ??= findAllScreenshots(log));
+}
+function findAllScreenshots(log: ToolingLog) {
+  try {
+    return globby
+      .sync(
+        [
+          'test/functional/**/screenshots/failure/*.png',
+          'x-pack/test/functional/**/screenshots/failure/*.png',
+        ],
+        {
+          cwd: REPO_ROOT,
+          absolute: true,
+        }
+      )
+      .map((path) => ({
+        path,
+        name: Path.basename(path, Path.extname(path)),
+      }));
+  } catch (error) {
+    log.error(`Failed to find screenshots: ${error.message}`);
+    return [];
+  }
+}
+
+function getFtrScreenshotHtml(log: ToolingLog, failureName: string) {
+  return getAllScreenshots(log)
+    .filter((s) => s.name.startsWith(FtrScreenshotFilename.create(failureName, { ext: false })))
+    .map((s) => {
+      const base64 = Fs.readFileSync(s.path).toString('base64');
+      return `<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${base64}" />`;
+    })
+    .join('\n');
+}
+
+export async function reportFailuresToFile(
+  log: ToolingLog,
+  failures: TestFailure[],
+  bkMeta: BuildkiteMetadata,
+  rootMeta: Record<string, unknown>
+) {
+  if (!failures?.length) {
+    return;
+  }
+
+  const journeyMeta = getJourneyMetadata(rootMeta);
+
+  // Jest could, in theory, fail 1000s of tests and write 1000s of failures
+  // So let's just write files for the first 20
+  for (const failure of failures.slice(0, 20)) {
+    const hash = createHash('md5').update(failure.name).digest('hex');
+    const filenameBase = `${
+      process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : ''
+    }${hash}`;
+    const dir = Path.join('target', 'test_failures');
+
+    const failureLog = [
+      ['Test:', '-----', failure.classname, failure.name, ''],
+      ['Failure:', '--------', failure.failure],
+      failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [],
+    ]
+      .flat()
+      .join('\n');
+
+    const failureJSON = JSON.stringify(
+      {
+        ...failure,
+        hash,
+        buildId: bkMeta.buildId,
+        jobId: bkMeta.jobId,
+        url: bkMeta.url,
+        jobUrl: bkMeta.jobUrl,
+        jobName: bkMeta.jobName,
+      },
+      null,
+      2
+    );
+
+    const failureHTML = Fs.readFileSync(
+      require.resolve('./report_failures_to_file_html_template.html')
+    )
+      .toString()
+      .replace('$TITLE', escape(failure.name))
+      .replace(
+        '$MAIN',
+        `
+        ${failure.classname
+          .split('.')
+          .map((part) => `<h5>${escape(part.replace('·', '.'))}</h5>`)
+          .join('')}
+        <hr />
+        <p><strong>${escape(failure.name)}</strong></p>
+        <p>
+          <small>
+            <strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
+              failure.failureCount || 0
+            }</span>
+            ${
+              failure.githubIssue
+                ? `<br /><a href="${escape(failure.githubIssue)}">${escape(
+                    failure.githubIssue
+                  )}</a>`
+                : ''
+            }
+          </small>
+        </p>
+        ${
+          bkMeta.jobUrl
+            ? `<p>
+              <small>
+                <strong>Buildkite Job</strong><br />
+                <a href="${escape(bkMeta.jobUrl)}">${escape(bkMeta.jobUrl)}</a>
+              </small>
+            </p>`
+            : ''
+        }
+        <pre>${escape(failure.failure)}</pre>
+        ${
+          journeyMeta
+            ? await getJourneySnapshotHtml(log, journeyMeta)
+            : getFtrScreenshotHtml(log, failure.name)
+        }
+        ${
+          failure['system-out']
+            ? `
+              <h5>Stdout</h5>
+              <pre>${escape(failure['system-out'] || '')}</pre>
+            `
+            : ''
+        }
+      `
+      );
+
+    Fs.mkdirSync(dir, { recursive: true });
+    Fs.writeFileSync(Path.join(dir, `${filenameBase}.log`), failureLog, 'utf8');
+    Fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8');
+    Fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8');
+  }
+}

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts

@@ -35,6 +35,8 @@ export interface TestSuite {
     failures: string;
     /* number of skipped tests as a string */
     skipped: string;
+    /* optional JSON encoded metadata */
+    'metadata-json'?: string;
   };
   testcase?: TestCase[];
 }
@@ -93,3 +95,22 @@ export function* makeFailedTestCaseIter(report: TestReport) {
     yield testCase as FailedTestCase;
   }
 }
+
+export function getRootMetadata(report: TestReport): Record<string, unknown> {
+  const json =
+    ('testsuites' in report
+      ? report.testsuites?.testsuite?.[0]?.$?.['metadata-json']
+      : report.testsuite?.$?.['metadata-json']) ?? '{}';
+
+  try {
+    const obj = JSON.parse(json);
+
+    if (typeof obj === 'object' && obj !== null) {
+      return obj;
+    }
+
+    return {};
+  } catch {
+    return {};
+  }
+}

packages/kbn-failed-test-reporter-cli/index.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { runFailedTestsReporterCli } from './run_failed_tests_reporter_cli';
+import './failed_tests_reporter/failed_tests_reporter_cli';

packages/kbn-failed-test-reporter-cli/jest.config.js

@@ -6,7 +6,8 @@
  * Side Public License, v 1.
  */
 
-export { runTestsCli } from './run_tests/cli';
+module.exports = {
-export { processOptions as processRunTestsCliOptions } from './run_tests/args';
+  preset: '@kbn/test/jest_node',
-export { startServersCli } from './start_servers/cli';
+  rootDir: '../..',
-export { processOptions as processStartServersCliOptions } from './start_servers/args';
+  roots: ['<rootDir>/packages/kbn-failed-test-reporter-cli'],
+};

packages/kbn-failed-test-reporter-cli/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/failed-test-reporter-cli",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true,
+  "runtimeDeps": [],
+  "typeDeps": [],
+}

packages/kbn-failed-test-reporter-cli/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/failed-test-reporter-cli",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-failed-test-reporter-cli/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ]
+}

packages/kbn-ftr-common-functional-services/BUILD.bazel

@@ -0,0 +1,127 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-ftr-common-functional-services"
+PKG_REQUIRE_NAME = "@kbn/ftr-common-functional-services"
+
+SOURCE_FILES = glob(
+  [
+    "**/*.ts",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "@npm//@types/node",
+  "@npm//@types/jest",
+  "//packages/kbn-tooling-log:npm_module_types",
+  "//packages/kbn-es-archiver:npm_module_types",
+  "//packages/kbn-test:npm_module_types",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-ftr-common-functional-services/README.md

@@ -0,0 +1,3 @@
+# @kbn/ftr-common-functional-services
+
+A collection of very common services used by all functional FTR configs, moved to a package so that we can start putting FTR configs in packages.

packages/kbn-ftr-common-functional-services/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ProvidedType } from '@kbn/test';
+export { services as commonFunctionalServices } from './services/all';
+
+import { KibanaServerProvider } from './services/kibana_server';
+export type KibanaServer = ProvidedType<typeof KibanaServerProvider>;
+
+export { RetryService } from './services/retry';
+
+import { EsArchiverProvider } from './services/es_archiver';
+export type EsArchiver = ProvidedType<typeof EsArchiverProvider>;
+
+import { EsProvider } from './services/es';
+export type Es = ProvidedType<typeof EsProvider>;

packages/kbn-ftr-common-functional-services/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-ftr-common-functional-services'],
+};

packages/kbn-ftr-common-functional-services/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/ftr-common-functional-services",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true,
+  "runtimeDeps": [],
+  "typeDeps": [],
+}

packages/kbn-ftr-common-functional-services/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/ftr-common-functional-services",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-ftr-common-functional-services/services/all.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EsArchiverProvider } from './es_archiver';
+import { EsProvider } from './es';
+import { KibanaServerProvider } from './kibana_server';
+import { RetryService } from './retry';
+
+export const services = {
+  es: EsProvider,
+  kibanaServer: KibanaServerProvider,
+  esArchiver: EsArchiverProvider,
+  retry: RetryService,
+};

packages/kbn-ftr-common-functional-services/services/es.ts

@@ -9,12 +9,9 @@
 import { Client } from '@elastic/elasticsearch';
 
 import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test';
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from './ftr_provider_context';
 
-/*
+export function EsProvider({ getService }: FtrProviderContext): Client {
- registers Kibana-specific @elastic/elasticsearch client instance.
- */
-export function ElasticsearchProvider({ getService }: FtrProviderContext): Client {
   const config = getService('config');
 
   return createEsClientForFtrConfig(config, {

packages/kbn-ftr-common-functional-services/services/es_archiver.ts

@@ -7,17 +7,15 @@
  */
 
 import { EsArchiver } from '@kbn/es-archiver';
-import { FtrProviderContext } from '../ftr_provider_context';
+import { FtrProviderContext } from './ftr_provider_context';
-import * as KibanaServer from './kibana_server';
+import { extendEsArchiver } from './kibana_server';
 
 export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver {
   const config = getService('config');
   const client = getService('es');
-  const lifecycle = getService('lifecycle');
   const log = getService('log');
   const kibanaServer = getService('kibanaServer');
   const retry = getService('retry');
-  const esArchives: string[] = config.get('testData.esArchives');
 
   const esArchiver = new EsArchiver({
     client,
@@ -25,26 +23,12 @@ export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiv
     kbnClient: kibanaServer,
   });
 
-  KibanaServer.extendEsArchiver({
+  extendEsArchiver({
     esArchiver,
     kibanaServer,
     retry,
     defaults: config.get('uiSettings.defaults'),
   });
 
-  if (esArchives.length) {
-    lifecycle.beforeTests.add(async () => {
-      for (const archive of esArchives) {
-        await esArchiver.load(archive);
-      }
-    });
-
-    lifecycle.cleanup.add(async () => {
-      for (const archive of esArchives) {
-        await esArchiver.unload(archive);
-      }
-    });
-  }
-
   return esArchiver;
 }

packages/kbn-ftr-common-functional-services/services/ftr_provider_context.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test';
+
+import type { services } from './all';
+
+type Services = typeof services;
+
+export type FtrProviderContext = GenericFtrProviderContext<Services, {}>;
+export class FtrService extends GenericFtrService<FtrProviderContext> {}

packages/kbn-ftr-common-functional-services/services/kibana_server/kibana_server.ts

@@ -9,7 +9,7 @@
 import Url from 'url';
 import { KbnClient } from '@kbn/test';
 
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 export function KibanaServerProvider({ getService }: FtrProviderContext): KbnClient {
   const log = getService('log');
@@ -17,7 +17,6 @@ export function KibanaServerProvider({ getService }: FtrProviderContext): KbnCli
   const lifecycle = getService('lifecycle');
   const url = Url.format(config.get('servers.kibana'));
   const defaults = config.get('uiSettings.defaults');
-  const kbnArchives: string[] = config.get('testData.kbnArchives');
 
   const kbn = new KbnClient({
     log,
@@ -32,18 +31,5 @@ export function KibanaServerProvider({ getService }: FtrProviderContext): KbnCli
     });
   }
 
-  if (kbnArchives.length) {
-    lifecycle.beforeTests.add(async () => {
-      for (const archive of kbnArchives) {
-        await kbn.importExport.load(archive);
-      }
-    });
-    lifecycle.cleanup.add(async () => {
-      for (const archive of kbnArchives) {
-        await kbn.importExport.unload(archive);
-      }
-    });
-  }
-
   return kbn;
 }

packages/kbn-ftr-common-functional-services/services/retry/retry.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { FtrService } from '../../ftr_provider_context';
+import { FtrService } from '../ftr_provider_context';
 import { retryForSuccess } from './retry_for_success';
 import { retryForTruthy } from './retry_for_truthy';
 

packages/kbn-ftr-common-functional-services/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ]
+}

packages/kbn-ftr-screenshot-filename/BUILD.bazel

@@ -0,0 +1,125 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-ftr-screenshot-filename"
+PKG_REQUIRE_NAME = "@kbn/ftr-screenshot-filename"
+
+SOURCE_FILES = glob(
+  [
+    "**/*.ts",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "@npm//@types/node",
+  "@npm//@types/jest",
+  "@npm//tslib",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-ftr-screenshot-filename/README.md

@@ -0,0 +1,3 @@
+# @kbn/ftr-screenshot-filename
+
+A simple package that exposes a helper function for generating a unique screenshot filename that can be found by `node scripts/failed_test_reporter`.

packages/kbn-ftr-screenshot-filename/ftr_screenshot_filename.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createHash } from 'crypto';
+
+export function create(fullTitle: string, opts?: { ext?: boolean }) {
+  const truncatedName = fullTitle.replaceAll(/[^ a-zA-Z0-9-]+/g, '').slice(0, 80);
+  const failureNameHash = createHash('sha256').update(fullTitle).digest('hex');
+  return `${truncatedName}-${failureNameHash}${opts?.ext === false ? '' : `.png`}`;
+}

packages/kbn-ftr-screenshot-filename/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as FtrScreenshotFilename from './ftr_screenshot_filename';
+
+export { FtrScreenshotFilename };

packages/kbn-ftr-screenshot-filename/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-ftr-screenshot-filename'],
+};

packages/kbn-ftr-screenshot-filename/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/ftr-screenshot-filename",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true,
+  "runtimeDeps": [],
+  "typeDeps": [],
+}

packages/kbn-ftr-screenshot-filename/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/ftr-screenshot-filename",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-ftr-screenshot-filename/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ]
+}

packages/kbn-journeys/BUILD.bazel

@@ -0,0 +1,134 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-journeys"
+PKG_REQUIRE_NAME = "@kbn/journeys"
+
+SOURCE_FILES = glob(
+  [
+    "**/*.ts",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "@npm//@types/node",
+  "@npm//@types/mocha",
+  "@npm//playwright",
+  "@npm//uuid",
+  "@npm//axios",
+  "@npm//callsites",
+  "@npm//rxjs",
+  "@npm//elastic-apm-node",
+  "//packages/kbn-ftr-common-functional-services:npm_module_types",
+  "//packages/kbn-ftr-screenshot-filename:npm_module_types",
+  "//packages/kbn-test:npm_module_types",
+  "//packages/kbn-utils:npm_module_types",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-journeys/README.mdx

@@ -0,0 +1,32 @@
+---
+id: kibDevDocsOpsJourneys
+slug: /kibana-dev-docs/ops/journeys
+title: Journeys
+description: A new style of functional test, focused on performance testing for now
+tags: ['kibana', 'dev', 'contributor', 'operations', 'performance', 'functional', 'testing']
+---
+
+Journeys are a slightly newer take on Functional Tests, currently powered by [playwright](https://playwright.dev/docs).
+
+A Journey is a single pathway through Kibana and looks something like this:
+
+```ts
+import { Journey } from '@kbn/journeys';
+import { subj } from '@kbn/test-subj-selector';
+
+export const journey = new Journey({
+  esArchives: [ ... ],
+  kbnArchives: [ ... ],
+  scalabilitySetup: { ... },
+})
+  .step('Go to Discover Page', async ({ page, kbnUrl }) => {
+    await page.goto(kbnUrl.get(`/app/discover`));
+    await page.waitForSelector(subj('discoverDocTable'));
+  })
+
+  .step('Expand the first document', async ({ page }) => {
+    const expandButtons = page.locator(subj('docTableExpandToggleColumn'));
+    await expandButtons.first().click();
+    await page.locator('text="Expanded document"');
+  });
+```

packages/kbn-journeys/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { JourneyConfig } from './journey/journey_config';
+export type { ScalabilityAction, ScalabilitySetup } from './journey/journey_config';
+
+export { Journey } from './journey/journey';
+export type { Step } from './journey/journey';
+
+export { JourneyScreenshots } from './journey/journey_screenshots';

packages/kbn-journeys/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-journeys'],
+};

packages/kbn-journeys/journey/journey.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { inspect } from 'util';
+
+import { Page } from 'playwright';
+import callsites from 'callsites';
+import { ToolingLog } from '@kbn/tooling-log';
+import { FtrConfigProvider } from '@kbn/test';
+import { FtrProviderContext } from '@kbn/ftr-common-functional-services';
+
+import { Auth } from '../services/auth';
+import { InputDelays } from '../services/input_delays';
+import { KibanaUrl } from '../services/kibana_url';
+
+import { JourneyFtrHarness } from './journey_ftr_harness';
+import { makeFtrConfigProvider } from './journey_ftr_config';
+import { JourneyConfig, JourneyConfigOptions } from './journey_config';
+
+export interface BaseStepCtx {
+  page: Page;
+  log: ToolingLog;
+  inputDelays: InputDelays;
+  kbnUrl: KibanaUrl;
+}
+
+export type AnyStep = Step<{}>;
+
+export interface Step<CtxExt extends object> {
+  name: string;
+  index: number;
+  fn(ctx: BaseStepCtx & CtxExt): Promise<void>;
+}
+
+const CONFIG_PROVIDER_CACHE = new WeakMap<Journey<any>, FtrConfigProvider>();
+
+export class Journey<CtxExt extends object> {
+  static convertToFtrConfigProvider(journey: Journey<any>) {
+    const cached = CONFIG_PROVIDER_CACHE.get(journey);
+    if (cached) {
+      return cached;
+    }
+
+    const provider = makeFtrConfigProvider(journey.config, journey.#steps);
+    CONFIG_PROVIDER_CACHE.set(journey, provider);
+    return provider;
+  }
+
+  /**
+   * Load a journey from a file path
+   */
+  static async load(path: string) {
+    let m;
+    try {
+      m = await import(path);
+    } catch (error) {
+      throw new Error(`Unable to load file: ${path}`);
+    }
+
+    if (!m || !m.journey) {
+      throw new Error(`[${path}] is not a journey`);
+    }
+
+    const journey = m.journey;
+    if (journey instanceof Journey) {
+      return journey;
+    }
+
+    const dbg = inspect(journey);
+    throw new Error(`[${path}] does not export a Journey like it should, received ${dbg}`);
+  }
+
+  #steps: Array<Step<CtxExt>> = [];
+
+  config: JourneyConfig<CtxExt>;
+
+  /**
+   * Create a Journey which should be exported from a file in the
+   * x-pack/performance/journeys directory.
+   */
+  constructor(opts?: JourneyConfigOptions<CtxExt>) {
+    const path = callsites().at(1)?.getFileName();
+
+    if (!path) {
+      throw new Error('unable to determine path of journey config file');
+    }
+
+    this.config = new JourneyConfig(path, opts);
+  }
+
+  /**
+   * Define a step of this Journey. Steps are only separated from each other
+   * to aid in reading/debuging the journey and reading it's logging output.
+   *
+   * If a journey fails, a failure report will be created with a screenshot
+   * at the point of failure as well as a screenshot at the end of every
+   * step.
+   */
+  step(name: string, fn: (ctx: BaseStepCtx & CtxExt) => Promise<void>) {
+    this.#steps.push({
+      name,
+      index: this.#steps.length,
+      fn,
+    });
+
+    return this;
+  }
+
+  /** called by FTR to setup tests */
+  protected testProvider({ getService }: FtrProviderContext) {
+    new JourneyFtrHarness(
+      getService('log'),
+      getService('config'),
+      getService('esArchiver'),
+      getService('kibanaServer'),
+      new Auth(getService('config'), getService('log'), getService('kibanaServer')),
+      this.config
+    ).initMochaSuite(this.#steps);
+  }
+}

packages/kbn-journeys/journey/journey_config.ts

@@ -0,0 +1,155 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { REPO_ROOT } from '@kbn/utils';
+
+import { BaseStepCtx } from './journey';
+
+export interface RampConcurrentUsersAction {
+  action: 'rampConcurrentUsers';
+  /**
+   * Duration strings must be formatted as string that starts with an integer and
+   * ends with either "m" or "s" for minutes and seconds, respectively
+   *
+   * eg: "1m" or "30s"
+   */
+  duration: string;
+  minUsersCount: number;
+  maxUsersCount: number;
+}
+
+export interface ConstantConcurrentUsersAction {
+  action: 'constantConcurrentUsers';
+  /**
+   * Duration strings must be formatted as string that starts with an integer and
+   * ends with either "m" or "s" for minutes and seconds, respectively
+   *
+   * eg: "1m" or "30s"
+   */
+  duration: string;
+  userCount: number;
+}
+
+export type ScalabilityAction = RampConcurrentUsersAction | ConstantConcurrentUsersAction;
+
+export interface ScalabilitySetup {
+  /**
+   * Duration strings must be formatted as string that starts with an integer and
+   * ends with either "m" or "s" for minutes and seconds, respectively
+   *
+   * eg: "1m" or "30s"
+   */
+  maxDuration: string;
+  warmup: ScalabilityAction[];
+  test: ScalabilityAction[];
+}
+
+export interface JourneyConfigOptions<CtxExt> {
+  /**
+   * Set to `true` to skip this journey. should probably be preceded
+   * by a link to a Github issue where the reasoning for why this was
+   * skipped and not just deleted is outlined.
+   */
+  skipped?: boolean;
+  /**
+   * Scalability configuration used to customize automatically generated
+   * scalability tests. For now chat with Dima/Operations if you want to
+   * use this option.
+   */
+  scalabilitySetup?: ScalabilitySetup;
+  /**
+   * These labels will be attached to all APM data created when running
+   * this journey.
+   */
+  extraApmLabels?: Record<string, string>;
+  /**
+   * A list of kbnArchives which will be automatically loaded/unloaded
+   * for this journey.
+   */
+  kbnArchives?: string[];
+  /**
+   * A list of esArchives which will be automatically loaded/unloaded
+   * for this journey.
+   */
+  esArchives?: string[];
+  /**
+   * By default the API is used to get a cookie that can be used for all
+   * navigation requests to Kibana, so that we don't ever see the login
+   * screen. Set this to `false` to disable this behavior.
+   */
+  skipAutoLogin?: boolean;
+  /**
+   * Use this to extend the context provided to each step. This function
+   * is called with the default context and returns an object that will
+   * be merged with the default context provided to each step function.
+   */
+  extendContext?: (ctx: BaseStepCtx) => CtxExt;
+}
+
+export class JourneyConfig<CtxExt extends object> {
+  #opts: JourneyConfigOptions<CtxExt>;
+  #path: string;
+  #name: string;
+
+  constructor(path: string, opts: JourneyConfigOptions<CtxExt> = {}) {
+    this.#path = path;
+    this.#name = Path.basename(this.#path, Path.extname(this.#path));
+    this.#opts = opts;
+  }
+
+  getEsArchives() {
+    return this.#opts.esArchives ?? [];
+  }
+
+  getKbnArchives() {
+    return this.#opts.kbnArchives ?? [];
+  }
+
+  isXpack() {
+    return this.getRepoRelPath().split(Path.sep).at(0) === 'x-pack';
+  }
+
+  getExtraApmLabels() {
+    return this.#opts.extraApmLabels ? { ...this.#opts.extraApmLabels } : {};
+  }
+
+  getRepoRelPath() {
+    return Path.relative(REPO_ROOT, this.getPath());
+  }
+
+  getPath() {
+    return this.#path;
+  }
+
+  getName() {
+    return this.#name;
+  }
+
+  shouldAutoLogin() {
+    return !this.#opts.skipAutoLogin;
+  }
+
+  isSkipped() {
+    return !!this.#opts.skipped;
+  }
+
+  getScalabilityConfig() {
+    return this.#opts.scalabilitySetup;
+  }
+
+  getExtendedStepCtx(ctx: BaseStepCtx): BaseStepCtx & CtxExt {
+    const ext = this.#opts.extendContext ?? (() => ({} as CtxExt));
+
+    return {
+      ...ctx,
+      ...ext(ctx),
+    };
+  }
+}

packages/kbn-journeys/journey/journey_ftr_config.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { v4 as uuidV4 } from 'uuid';
+import { REPO_ROOT } from '@kbn/utils';
+import { FtrConfigProviderContext, FtrConfigProvider } from '@kbn/test';
+import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
+
+import { AnyStep } from './journey';
+import { JourneyConfig } from './journey_config';
+
+// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
+const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443';
+const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB';
+
+export function makeFtrConfigProvider(
+  config: JourneyConfig<any>,
+  steps: AnyStep[]
+): FtrConfigProvider {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const baseConfig = (
+      await readConfigFile(
+        Path.resolve(
+          REPO_ROOT,
+          config.isXpack()
+            ? 'x-pack/test/functional/config.base.js'
+            : 'test/functional/config.base.js'
+        )
+      )
+    ).getAll();
+
+    const testBuildId = process.env.BUILDKITE_BUILD_ID ?? `local-${uuidV4()}`;
+    const testJobId = process.env.BUILDKITE_JOB_ID ?? `local-${uuidV4()}`;
+    const prId = process.env.GITHUB_PR_NUMBER
+      ? Number.parseInt(process.env.GITHUB_PR_NUMBER, 10)
+      : undefined;
+
+    if (Number.isNaN(prId)) {
+      throw new Error('invalid GITHUB_PR_NUMBER environment variable');
+    }
+
+    const telemetryLabels: Record<string, string | boolean | undefined | number> = {
+      branch: process.env.BUILDKITE_BRANCH,
+      ciBuildId: process.env.BUILDKITE_BUILD_ID,
+      ciBuildJobId: process.env.BUILDKITE_JOB_ID,
+      ciBuildNumber: Number(process.env.BUILDKITE_BUILD_NUMBER) || 0,
+      gitRev: process.env.BUILDKITE_COMMIT,
+      isPr: prId !== undefined,
+      ...(prId !== undefined ? { prId } : {}),
+      ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG,
+      journeyName: config.getName(),
+    };
+
+    return {
+      ...baseConfig,
+
+      mochaOpts: {
+        ...baseConfig.mochaOpts,
+        bail: true,
+      },
+
+      services: commonFunctionalServices,
+      pageObjects: {},
+
+      servicesRequiredForTestAnalysis: ['performance', 'journeyConfig'],
+
+      junit: {
+        reportName: `Journey: ${config.getName()}`,
+        metadata: {
+          journeyName: config.getName(),
+          stepNames: steps.map((s) => s.name),
+        },
+      },
+
+      kbnTestServer: {
+        ...baseConfig.kbnTestServer,
+        // delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution
+        delayShutdown: 15_000,
+
+        serverArgs: [
+          ...baseConfig.kbnTestServer.serverArgs,
+          `--telemetry.optIn=${process.env.TEST_PERFORMANCE_PHASE === 'TEST'}`,
+          `--telemetry.labels=${JSON.stringify(telemetryLabels)}`,
+          '--csp.strict=false',
+          '--csp.warnLegacyBrowsers=false',
+        ],
+
+        env: {
+          ELASTIC_APM_ACTIVE: process.env.TEST_PERFORMANCE_PHASE ? 'true' : 'false',
+          ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
+          ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
+          ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
+          ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
+          ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
+          // capture request body for both errors and request transactions
+          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body
+          ELASTIC_APM_CAPTURE_BODY: 'all',
+          // capture request headers
+          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-headers
+          ELASTIC_APM_CAPTURE_HEADERS: true,
+          // request body with bigger size will be trimmed.
+          // 300_000 is the default of the APM server.
+          // for a body with larger size, we might need to reconfigure the APM server to increase the limit.
+          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length
+          ELASTIC_APM_LONG_FIELD_MAX_LENGTH: 300_000,
+          ELASTIC_APM_GLOBAL_LABELS: Object.entries({
+            ...config.getExtraApmLabels(),
+            testJobId,
+            testBuildId,
+            journeyName: config.getName(),
+            ftrConfig: config.getRepoRelPath(),
+            performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+          })
+            .flatMap(([key, value]) => (value == null ? [] : `${key}=${value}`))
+            .join(','),
+        },
+      },
+    };
+  };
+}

packages/kbn-journeys/journey/journey_ftr_harness.ts

@@ -0,0 +1,410 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Url from 'url';
+import { inspect, format } from 'util';
+import { setTimeout } from 'timers/promises';
+
+import * as Rx from 'rxjs';
+import apmNode from 'elastic-apm-node';
+import playwright, { ChromiumBrowser, Page, BrowserContext, CDPSession, Request } from 'playwright';
+import { asyncMap, asyncForEach } from '@kbn/std';
+import { ToolingLog } from '@kbn/tooling-log';
+import { Config } from '@kbn/test';
+import { EsArchiver, KibanaServer } from '@kbn/ftr-common-functional-services';
+
+import { Auth } from '../services/auth';
+import { getInputDelays } from '../services/input_delays';
+import { KibanaUrl } from '../services/kibana_url';
+
+import type { Step, AnyStep } from './journey';
+import type { JourneyConfig } from './journey_config';
+import { JourneyScreenshots } from './journey_screenshots';
+
+export class JourneyFtrHarness {
+  private readonly screenshots: JourneyScreenshots;
+
+  constructor(
+    private readonly log: ToolingLog,
+    private readonly config: Config,
+    private readonly esArchiver: EsArchiver,
+    private readonly kibanaServer: KibanaServer,
+    private readonly auth: Auth,
+    private readonly journeyConfig: JourneyConfig<any>
+  ) {
+    this.screenshots = new JourneyScreenshots(this.journeyConfig.getName());
+  }
+
+  private browser: ChromiumBrowser | undefined;
+  private page: Page | undefined;
+  private client: CDPSession | undefined;
+  private context: BrowserContext | undefined;
+  private currentSpanStack: Array<apmNode.Span | null> = [];
+  private currentTransaction: apmNode.Transaction | undefined | null = undefined;
+
+  private pageTeardown$ = new Rx.Subject<Page>();
+  private telemetryTrackerSubs = new Map<Page, Rx.Subscription>();
+
+  private apm: apmNode.Agent | null = null;
+
+  private async setupApm() {
+    const kbnTestServerEnv = this.config.get(`kbnTestServer.env`);
+
+    this.apm = apmNode.start({
+      serviceName: 'functional test runner',
+      environment: process.env.CI ? 'ci' : 'development',
+      active: kbnTestServerEnv.ELASTIC_APM_ACTIVE !== 'false',
+      serverUrl: kbnTestServerEnv.ELASTIC_APM_SERVER_URL,
+      secretToken: kbnTestServerEnv.ELASTIC_APM_SECRET_TOKEN,
+      globalLabels: kbnTestServerEnv.ELASTIC_APM_GLOBAL_LABELS,
+      transactionSampleRate: kbnTestServerEnv.ELASTIC_APM_TRANSACTION_SAMPLE_RATE,
+      logger: {
+        warn: (...args: any[]) => {
+          this.log.warning('APM WARN', ...args);
+        },
+        info: (...args: any[]) => {
+          this.log.info('APM INFO', ...args);
+        },
+        fatal: (...args: any[]) => {
+          this.log.error(format('APM FATAL', ...args));
+        },
+        error: (...args: any[]) => {
+          this.log.error(format('APM ERROR', ...args));
+        },
+        debug: (...args: any[]) => {
+          this.log.debug('APM DEBUG', ...args);
+        },
+        trace: (...args: any[]) => {
+          this.log.verbose('APM TRACE', ...args);
+        },
+      },
+    });
+
+    if (this.currentTransaction) {
+      throw new Error(`Transaction exist, end prev transaction ${this.currentTransaction?.name}`);
+    }
+
+    this.currentTransaction = this.apm?.startTransaction(
+      `Journey: ${this.journeyConfig.getName()}`,
+      'performance'
+    );
+  }
+
+  private async setupBrowserAndPage() {
+    const browser = await this.getBrowserInstance();
+    this.context = await browser.newContext({ bypassCSP: true });
+
+    if (this.journeyConfig.shouldAutoLogin()) {
+      const cookie = await this.auth.login({ username: 'elastic', password: 'changeme' });
+      await this.context.addCookies([cookie]);
+    }
+
+    this.page = await this.context.newPage();
+
+    if (!process.env.NO_BROWSER_LOG) {
+      this.page.on('console', this.onConsoleEvent);
+    }
+
+    await this.sendCDPCommands(this.context, this.page);
+
+    this.trackTelemetryRequests(this.page);
+    await this.interceptBrowserRequests(this.page);
+  }
+
+  private async onSetup() {
+    await Promise.all([
+      this.setupApm(),
+      this.setupBrowserAndPage(),
+      asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => {
+        await this.esArchiver.load(esArchive);
+      }),
+      asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => {
+        await this.kibanaServer.importExport.load(kbnArchive);
+      }),
+    ]);
+  }
+
+  private async tearDownBrowserAndPage() {
+    if (this.page) {
+      const telemetryTracker = this.telemetryTrackerSubs.get(this.page);
+      this.telemetryTrackerSubs.delete(this.page);
+
+      if (telemetryTracker && !telemetryTracker.closed) {
+        this.log.info(`Waiting for telemetry requests, including starting within next 3 secs`);
+        this.pageTeardown$.next(this.page);
+        await new Promise<void>((resolve) => telemetryTracker.add(resolve));
+      }
+
+      this.log.info('destroying page');
+      await this.client?.detach();
+      await this.page.close();
+      await this.context?.close();
+    }
+
+    if (this.browser) {
+      this.log.info('closing browser');
+      await this.browser.close();
+    }
+  }
+
+  private async teardownApm() {
+    if (!this.apm) {
+      return;
+    }
+
+    if (this.currentTransaction) {
+      this.currentTransaction.end('Success');
+      this.currentTransaction = undefined;
+    }
+
+    const apmStarted = this.apm.isStarted();
+    // @ts-expect-error
+    const apmActive = apmStarted && this.apm._conf.active;
+
+    if (!apmActive) {
+      this.log.warning('APM is not active');
+      return;
+    }
+
+    this.log.info('Flushing APM');
+    await new Promise<void>((resolve) => this.apm?.flush(() => resolve()));
+    // wait for the HTTP request that apm.flush() starts, which we
+    // can't track but hope it is started within 3 seconds, node will stay
+    // alive for active requests
+    // https://github.com/elastic/apm-agent-nodejs/issues/2088
+    await setTimeout(3000);
+  }
+
+  private async onTeardown() {
+    await Promise.all([
+      this.tearDownBrowserAndPage(),
+      this.teardownApm(),
+      asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => {
+        await this.esArchiver.unload(esArchive);
+      }),
+      asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => {
+        await this.kibanaServer.importExport.unload(kbnArchive);
+      }),
+    ]);
+  }
+
+  private async onStepSuccess(step: AnyStep) {
+    if (!this.page) {
+      return;
+    }
+
+    await this.screenshots.addSuccess(step, await this.page.screenshot());
+  }
+
+  private async onStepError(step: AnyStep, err: Error) {
+    if (this.currentTransaction) {
+      this.currentTransaction.end(`Failure ${err.message}`);
+      this.currentTransaction = undefined;
+    }
+
+    if (this.page) {
+      await this.screenshots.addError(step, await this.page.screenshot());
+    }
+  }
+
+  private async withSpan<T>(name: string, type: string | undefined, block: () => Promise<T>) {
+    if (!this.currentTransaction) {
+      return await block();
+    }
+
+    const span = this.apm?.startSpan(name, type ?? null, {
+      childOf: this.currentTransaction,
+    });
+    if (!span) {
+      return await block();
+    }
+
+    try {
+      this.currentSpanStack.unshift(span);
+      const result = await block();
+      span.setOutcome('success');
+      span.end();
+      return result;
+    } catch (error) {
+      span.setOutcome('failure');
+      span.end();
+      throw error;
+    } finally {
+      if (span !== this.currentSpanStack.shift()) {
+        // eslint-disable-next-line no-unsafe-finally
+        throw new Error('span stack mismatch');
+      }
+    }
+  }
+
+  private getCurrentTraceparent() {
+    return (this.currentSpanStack.length ? this.currentSpanStack[0] : this.currentTransaction)
+      ?.traceparent;
+  }
+
+  private async getBrowserInstance() {
+    if (this.browser) {
+      return this.browser;
+    }
+    return await this.withSpan('Browser creation', 'setup', async () => {
+      const headless = !!(process.env.TEST_BROWSER_HEADLESS || process.env.CI);
+      this.browser = await playwright.chromium.launch({ headless, timeout: 60_000 });
+      return this.browser;
+    });
+  }
+
+  private async sendCDPCommands(context: BrowserContext, page: Page) {
+    const client = await context.newCDPSession(page);
+
+    await client.send('Network.clearBrowserCache');
+    await client.send('Network.setCacheDisabled', { cacheDisabled: true });
+    await client.send('Network.emulateNetworkConditions', {
+      latency: 100,
+      downloadThroughput: 750_000,
+      uploadThroughput: 750_000,
+      offline: false,
+    });
+
+    return client;
+  }
+
+  private telemetryTrackerCount = 0;
+
+  private trackTelemetryRequests(page: Page) {
+    const id = ++this.telemetryTrackerCount;
+
+    const requestFailure$ = Rx.fromEvent<Request>(page, 'requestfailed');
+    const requestSuccess$ = Rx.fromEvent<Request>(page, 'requestfinished');
+    const request$ = Rx.fromEvent<Request>(page, 'request').pipe(
+      Rx.takeUntil(
+        this.pageTeardown$.pipe(
+          Rx.first((p) => p === page),
+          Rx.delay(3000)
+          // If EBT client buffers:
+          // Rx.mergeMap(async () => {
+          //  await page.waitForFunction(() => {
+          //    // return window.kibana_ebt_client.buffer_size == 0
+          //  });
+          // })
+        )
+      ),
+      Rx.mergeMap((request) => {
+        if (!request.url().includes('telemetry-staging.elastic.co')) {
+          return Rx.EMPTY;
+        }
+
+        this.log.debug(`Waiting for telemetry request #${id} to complete`);
+        return Rx.merge(requestFailure$, requestSuccess$).pipe(
+          Rx.first((r) => r === request),
+          Rx.tap({
+            complete: () => this.log.debug(`Telemetry request #${id} complete`),
+          })
+        );
+      })
+    );
+
+    this.telemetryTrackerSubs.set(page, request$.subscribe());
+  }
+
+  private async interceptBrowserRequests(page: Page) {
+    await page.route('**', async (route, request) => {
+      const headers = await request.allHeaders();
+      const traceparent = this.getCurrentTraceparent();
+      if (traceparent && request.isNavigationRequest()) {
+        await route.continue({ headers: { traceparent, ...headers } });
+      } else {
+        await route.continue();
+      }
+    });
+  }
+
+  #_ctx?: Record<string, unknown>;
+  private getCtx() {
+    if (this.#_ctx) {
+      return this.#_ctx;
+    }
+
+    const page = this.page;
+
+    if (!page) {
+      throw new Error('performance service is not properly initialized');
+    }
+
+    this.#_ctx = this.journeyConfig.getExtendedStepCtx({
+      page,
+      log: this.log,
+      inputDelays: getInputDelays(),
+      kbnUrl: new KibanaUrl(
+        new URL(
+          Url.format({
+            protocol: this.config.get('servers.kibana.protocol'),
+            hostname: this.config.get('servers.kibana.hostname'),
+            port: this.config.get('servers.kibana.port'),
+          })
+        )
+      ),
+    });
+
+    return this.#_ctx;
+  }
+
+  public initMochaSuite(steps: Array<Step<any>>) {
+    const journeyName = this.journeyConfig.getName();
+
+    (this.journeyConfig.isSkipped() ? describe.skip : describe)(`Journey[${journeyName}]`, () => {
+      before(async () => await this.onSetup());
+      after(async () => await this.onTeardown());
+
+      for (const step of steps) {
+        it(step.name, async () => {
+          await this.withSpan(`step: ${step.name}`, 'step', async () => {
+            try {
+              await step.fn(this.getCtx());
+              await this.onStepSuccess(step);
+            } catch (e) {
+              const error = new Error(`Step [${step.name}] failed: ${e.message}`);
+              error.stack = e.stack;
+              await this.onStepError(step, error);
+              throw error; // Rethrow error if step fails otherwise it is silently passing
+            }
+          });
+        });
+      }
+    });
+  }
+
+  private onConsoleEvent = async (message: playwright.ConsoleMessage) => {
+    try {
+      const { url, lineNumber, columnNumber } = message.location();
+      const location = `${url}:${lineNumber}:${columnNumber}`;
+
+      const args = await asyncMap(message.args(), (handle) => handle.jsonValue());
+      const text = args.length
+        ? args.map((arg) => (typeof arg === 'string' ? arg : inspect(arg, false, null))).join(' ')
+        : message.text();
+
+      if (
+        url.includes('kbn-ui-shared-deps-npm.dll.js') &&
+        text.includes('moment construction falls')
+      ) {
+        // ignore errors from moment about constructing dates with invalid formats
+        return;
+      }
+
+      const type = message.type();
+      const method = type === 'debug' ? type : type === 'warning' ? 'error' : 'info';
+      const name = type === 'warning' ? 'error' : 'log';
+      this.log[method](`[console.${name}] @ ${location}:\n${text}`);
+    } catch (error) {
+      const dbg = inspect(message);
+      this.log.error(
+        `Error interpreting browser console.log:\nerror:${error.message}\nmessage:\n${dbg}`
+      );
+    }
+  };
+}

packages/kbn-journeys/journey/journey_screenshots.ts

@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fsp from 'fs/promises';
+
+import * as Rx from 'rxjs';
+import { REPO_ROOT } from '@kbn/utils';
+import { FtrScreenshotFilename } from '@kbn/ftr-screenshot-filename';
+
+import type { AnyStep } from './journey';
+
+interface StepShot {
+  type: 'success' | 'failure';
+  title: string;
+  filename: string;
+}
+
+interface Manifest {
+  steps: StepShot[];
+}
+
+const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null;
+const isString = (v: unknown): v is string => typeof v === 'string';
+const isStepShot = (v: unknown): v is StepShot =>
+  isObj(v) &&
+  (v.type === 'success' || v.type === 'failure') &&
+  isString(v.title) &&
+  isString(v.filename);
+
+const write = async (path: string, content: string | Buffer) => {
+  await Fsp.mkdir(Path.dirname(path), { recursive: true });
+  await Fsp.writeFile(path, content);
+};
+
+export class JourneyScreenshots {
+  static async load(journeyName: string) {
+    const screenshots = new JourneyScreenshots(journeyName);
+
+    const json = await Fsp.readFile(screenshots.#manifestPath, 'utf8');
+    const manifest = JSON.parse(json);
+
+    if (!isObj(manifest)) {
+      throw new Error('invalid manifest, json parsed but not to an object');
+    }
+
+    const { steps } = manifest;
+
+    if (!Array.isArray(steps) || !steps.every(isStepShot)) {
+      throw new Error('invalid manifest, steps must be an array of StepShot objects');
+    }
+
+    screenshots.#manifest = { steps };
+    return screenshots;
+  }
+
+  readonly #dir: string;
+  readonly #manifestPath: string;
+
+  #manifest: Manifest = {
+    steps: [],
+  };
+
+  constructor(journeyName: string) {
+    this.#dir = Path.resolve(REPO_ROOT, 'data/journey_screenshots', journeyName);
+    this.#manifestPath = Path.resolve(this.#dir, 'manifest.json');
+  }
+
+  readonly #isLocked = new Rx.BehaviorSubject<boolean>(false);
+  async lock(fn: () => Promise<void>) {
+    if (this.#isLocked.getValue()) {
+      do {
+        await Rx.firstValueFrom(this.#isLocked.pipe(Rx.skip(1)));
+      } while (this.#isLocked.getValue());
+    }
+
+    try {
+      this.#isLocked.next(true);
+      await fn();
+    } finally {
+      this.#isLocked.next(false);
+    }
+  }
+
+  async addError(step: AnyStep, screenshot: Buffer) {
+    await this.lock(async () => {
+      const filename = FtrScreenshotFilename.create(`${step.index}-${step.name}-failure`);
+      this.#manifest.steps.push({
+        type: 'failure',
+        title: `Step #${step.index + 1}: ${step.name} - FAILED`,
+        filename,
+      });
+
+      await Promise.all([
+        write(Path.resolve(this.#dir, 'manifest.json'), JSON.stringify(this.#manifest)),
+        write(Path.resolve(this.#dir, filename), screenshot),
+      ]);
+    });
+  }
+
+  async addSuccess(step: AnyStep, screenshot: Buffer) {
+    await this.lock(async () => {
+      const filename = FtrScreenshotFilename.create(`${step.index}-${step.name}`);
+      this.#manifest.steps.push({
+        type: 'success',
+        title: `Step #${step.index + 1}: ${step.name} - DONE`,
+        filename,
+      });
+
+      await Promise.all([
+        write(Path.resolve(this.#dir, 'manifest.json'), JSON.stringify(this.#manifest)),
+        write(Path.resolve(this.#dir, filename), screenshot),
+      ]);
+    });
+  }
+
+  get() {
+    return this.#manifest.steps.map((stepShot) => ({
+      ...stepShot,
+      path: Path.resolve(this.#dir, stepShot.filename),
+    }));
+  }
+}

packages/kbn-journeys/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/journeys",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true,
+  "runtimeDeps": [],
+  "typeDeps": [],
+}

packages/kbn-journeys/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/journeys",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-journeys/services/auth.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Url from 'url';
+import { format } from 'util';
+
+import axios, { AxiosResponse } from 'axios';
+import { ToolingLog } from '@kbn/tooling-log';
+import { Config } from '@kbn/test';
+import { KibanaServer } from '@kbn/ftr-common-functional-services';
+
+export interface Credentials {
+  username: string;
+  password: string;
+}
+
+function extractCookieValue(authResponse: AxiosResponse) {
+  return authResponse.headers['set-cookie']?.[0].toString().split(';')[0].split('sid=')[1] ?? '';
+}
+export class Auth {
+  constructor(
+    private readonly config: Config,
+    private readonly log: ToolingLog,
+    private readonly kibanaServer: KibanaServer
+  ) {}
+
+  public async login({ username, password }: Credentials) {
+    const baseUrl = new URL(
+      Url.format({
+        protocol: this.config.get('servers.kibana.protocol'),
+        hostname: this.config.get('servers.kibana.hostname'),
+        port: this.config.get('servers.kibana.port'),
+      })
+    );
+
+    const loginUrl = new URL('/internal/security/login', baseUrl);
+    const provider = baseUrl.hostname === 'localhost' ? 'basic' : 'cloud-basic';
+
+    this.log.info('fetching auth cookie from', loginUrl.href);
+    const authResponse = await axios.request({
+      url: loginUrl.href,
+      method: 'post',
+      data: {
+        providerType: 'basic',
+        providerName: provider,
+        currentURL: new URL('/login?next=%2F', baseUrl).href,
+        params: { username, password },
+      },
+      headers: {
+        'content-type': 'application/json',
+        'kbn-version': await this.kibanaServer.version.get(),
+        'sec-fetch-mode': 'cors',
+        'sec-fetch-site': 'same-origin',
+      },
+      validateStatus: () => true,
+      maxRedirects: 0,
+    });
+
+    const cookie = extractCookieValue(authResponse);
+    if (cookie) {
+      this.log.info('captured auth cookie');
+    } else {
+      this.log.error(
+        format('unable to determine auth cookie from response', {
+          status: `${authResponse.status} ${authResponse.statusText}`,
+          body: authResponse.data,
+          headers: authResponse.headers,
+        })
+      );
+
+      throw new Error(`failed to determine auth cookie`);
+    }
+
+    return {
+      name: 'sid',
+      value: cookie,
+      url: baseUrl.href,
+    };
+  }
+}

packages/kbn-journeys/services/input_delays.ts

@@ -1,10 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * 2.0.
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
-interface InputDelays {
+
+export interface InputDelays {
   TYPING: number;
   MOUSE_CLICK: number;
 }
@@ -20,7 +22,7 @@ const PROFILES: Record<string, InputDelays> = {
   },
 };
 
-export function InputDelaysProvider(): InputDelays {
+export function getInputDelays(): InputDelays {
   const profile = PROFILES[process.env.INPUT_DELAY_PROFILE ?? 'user'];
 
   if (!profile) {

packages/kbn-journeys/services/kibana_url.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export interface PathOptions {
+  /**
+   * Query string parameters
+   */
+  params?: Record<string, string>;
+  /**
+   * The hash value of the URL
+   */
+  hash?: string;
+}
+
+export class KibanaUrl {
+  #baseUrl: URL;
+
+  constructor(baseUrl: URL) {
+    this.#baseUrl = baseUrl;
+  }
+
+  /**
+   * Get an absolute URL based on Kibana's URL
+   * @param rel relative url, resolved relative to Kibana's url
+   * @param options optional modifications to apply to the URL
+   */
+  get(rel?: string, options?: PathOptions) {
+    const url = new URL(rel ?? '/', this.#baseUrl);
+
+    if (options?.params) {
+      for (const [key, value] of Object.entries(options.params)) {
+        url.searchParams.set(key, value);
+      }
+    }
+
+    if (options?.hash !== undefined) {
+      url.hash = options.hash;
+    }
+
+    return url.href;
+  }
+
+  /**
+   * Get the URL for an app
+   * @param appName name of the app to get the URL for
+   * @param options optional modifications to apply to the URL
+   */
+  app(appName: string, options?: PathOptions) {
+    return this.get(`/app/${appName}`, options);
+  }
+
+  toString() {
+    return this.#baseUrl.href;
+  }
+}

packages/kbn-journeys/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "stripInternal": false,
+    "types": [
+      "mocha",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ]
+}

packages/kbn-performance-testing-dataset-extractor/BUILD.bazel

@@ -66,8 +66,8 @@ RUNTIME_DEPS = [
 TYPES_DEPS = [
   "//packages/kbn-dev-cli-errors:npm_module_types",
   "//packages/kbn-dev-cli-runner:npm_module_types",
-  "//packages/kbn-test:npm_module_types",
   "//packages/kbn-tooling-log:npm_module_types",
+  "//packages/kbn-journeys:npm_module_types",
   "@npm//@elastic/elasticsearch",
   "@npm//@types/node",
   "@npm//@types/jest",

packages/kbn-performance-testing-dataset-extractor/src/cli.ts

@@ -12,16 +12,13 @@
  *
  *************************************************************/
 
+import path from 'path';
+
 import { run } from '@kbn/dev-cli-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
-import { EsVersion, readConfigFile } from '@kbn/test';
+import { Journey } from '@kbn/journeys';
-import path from 'path';
-import { extractor } from './extractor';
-import { ScalabilitySetup, TestData } from './types';
 
-interface Vars {
+import { extractor } from './extractor';
-  [key: string]: string;
-}
 
 export async function runExtractor() {
   run(
@@ -50,50 +47,7 @@ export async function runExtractor() {
         throw createFlagError('--es-password must be defined');
       }
 
-      const configPath = flags.config;
+      const withoutStaticResources = !!flags['without-static-resources'] || false;
-      if (typeof configPath !== 'string') {
-        throw createFlagError('--config must be a string');
-      }
-      const config = await readConfigFile(log, EsVersion.getDefault(), path.resolve(configPath));
-
-      const scalabilitySetup: ScalabilitySetup = config.get('scalabilitySetup');
-
-      if (!scalabilitySetup) {
-        log.warning(
-          `'scalabilitySetup' is not defined in config file, output file for Kibana scalability run won't be generated`
-        );
-      }
-
-      const testData: TestData = config.get('testData');
-
-      const env = config.get(`kbnTestServer.env`);
-      if (
-        typeof env !== 'object' ||
-        typeof env.ELASTIC_APM_GLOBAL_LABELS !== 'string' ||
-        !env.ELASTIC_APM_GLOBAL_LABELS.includes('journeyName=')
-      ) {
-        log.error(
-          `'journeyName' must be defined in config file:
-
-      env: {
-        ...config.kbnTestServer.env,
-        ELASTIC_APM_GLOBAL_LABELS: Object.entries({
-          journeyName: <journey name>,
-        })
-      },`
-        );
-        return;
-      }
-
-      const envVars: Vars = env.ELASTIC_APM_GLOBAL_LABELS.split(',').reduce(
-        (acc: Vars, pair: string) => {
-          const [key, value] = pair.split('=');
-          return { ...acc, [key]: value };
-        },
-        {}
-      );
-      const journeyName = envVars.journeyName;
-
       const buildId = flags.buildId;
       if (buildId && typeof buildId !== 'string') {
         throw createFlagError('--buildId must be a string');
@@ -102,11 +56,37 @@ export async function runExtractor() {
         throw createFlagError('--buildId must be defined');
       }
 
-      const withoutStaticResources = !!flags['without-static-resources'] || false;
+      const configPath = flags.config;
+      if (typeof configPath !== 'string') {
+        throw createFlagError('--config must be a string');
+      }
+      const journey = await Journey.load(path.resolve(configPath));
+
+      const scalabilitySetup = journey.config.getScalabilityConfig();
+      if (!scalabilitySetup) {
+        log.warning(
+          `'scalabilitySetup' is not defined in config file, output file for Kibana scalability run won't be generated`
+        );
+      }
+
+      const testData = {
+        esArchives: journey.config.getEsArchives(),
+        kbnArchives: journey.config.getKbnArchives(),
+      };
 
       return extractor({
-        param: { journeyName, scalabilitySetup, testData, buildId, withoutStaticResources },
+        param: {
-        client: { baseURL, username, password },
+          journeyName: journey.config.getName(),
+          scalabilitySetup,
+          testData,
+          buildId,
+          withoutStaticResources,
+        },
+        client: {
+          baseURL,
+          username,
+          password,
+        },
         log,
       });
     },

packages/kbn-performance-testing-dataset-extractor/src/types.ts

@@ -7,6 +7,7 @@
  */
 
 import { ToolingLog } from '@kbn/tooling-log';
+import { ScalabilitySetup } from '@kbn/journeys';
 
 export interface Request {
   transactionId: string;
@@ -31,19 +32,6 @@ export interface Stream<T extends Request> {
   requests: T[];
 }
 
-export interface InjectionStep {
-  action: string;
-  minUsersCount?: number;
-  maxUsersCount: number;
-  duration: string;
-}
-
-export interface ScalabilitySetup {
-  warmup: InjectionStep[];
-  test: InjectionStep[];
-  maxDuration: string;
-}
-
 export interface TestData {
   kbnArchives?: string[];
   esArchives?: string[];
@@ -52,7 +40,7 @@ export interface TestData {
 export interface CLIParams {
   param: {
     journeyName: string;
-    scalabilitySetup: ScalabilitySetup;
+    scalabilitySetup?: ScalabilitySetup;
     testData: TestData;
     buildId: string;
     withoutStaticResources: boolean;

packages/kbn-test/BUILD.bazel

@@ -99,6 +99,7 @@ TYPES_DEPS = [
   "//packages/kbn-tooling-log:npm_module_types",
   "//packages/kbn-bazel-packages:npm_module_types",
   "//packages/kbn-get-repo-files:npm_module_types",
+  "//packages/kbn-ftr-screenshot-filename:npm_module_types",
   "@npm//@elastic/elasticsearch",
   "@npm//@jest/console",
   "@npm//@jest/reporters",
@@ -116,6 +117,7 @@ TYPES_DEPS = [
   "@npm//jest-snapshot",
   "@npm//redux",
   "@npm//rxjs",
+  "@npm//playwright",
   "@npm//xmlbuilder",
   "@npm//@types/archiver",
   "@npm//@types/chance",

packages/kbn-test/index.ts

@@ -6,23 +6,13 @@
  * Side Public License, v 1.
  */
 
-// @internal
-import {
-  runTestsCli,
-  processRunTestsCliOptions,
-  startServersCli,
-  processStartServersCliOptions,
-  // @ts-ignore not typed yet
-} from './src/functional_tests/cli';
-
 export { KbnClientRequesterError } from './src/kbn_client/kbn_client_requester_error';
 
 // @internal
-export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions };
+export { startServersCli, startServers } from './src/functional_tests/start_servers';
 
-// @ts-ignore not typed yet
 // @internal
-export { runTests, startServers } from './src/functional_tests/tasks';
+export { runTestsCli, runTests } from './src/functional_tests/run_tests';
 
 export { getKibanaCliArg, getKibanaCliLoggers } from './src/functional_tests/lib/kibana_cli_args';
 
@@ -48,15 +38,9 @@ export {
   systemIndicesSuperuser,
 } from './src/kbn';
 
-export { readConfigFile } from './src/functional_test_runner/lib/config/read_config_file';
-
-export { runFtrCli } from './src/functional_test_runner/cli';
-
 // @internal
 export { setupJUnitReportGeneration, escapeCdata } from './src/mocha';
 
-export { runFailedTestsReporterCli } from './src/failed_tests_reporter';
-
 export { CI_PARALLEL_PROCESS_PREFIX } from './src/ci_parallel_process_prefix';
 
 export * from './src/functional_test_runner';

packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts

@@ -1,167 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { createHash } from 'crypto';
-import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs';
-import { join, basename, resolve } from 'path';
-
-import { ToolingLog } from '@kbn/tooling-log';
-import { REPO_ROOT } from '@kbn/utils';
-import { escape } from 'he';
-
-import { BuildkiteMetadata } from './buildkite_metadata';
-import { TestFailure } from './get_failures';
-
-const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
-  const files = readdirSync(dirPath);
-
-  for (const file of files) {
-    if (statSync(join(dirPath, file)).isDirectory()) {
-      if (file.match(/node_modules/)) {
-        continue;
-      }
-
-      allScreenshots = findScreenshots(join(dirPath, file), allScreenshots);
-    } else {
-      const fullPath = join(dirPath, file);
-      if (fullPath.match(/screenshots\/failure\/.+\.png$/)) {
-        allScreenshots.push(fullPath);
-      }
-    }
-  }
-
-  return allScreenshots;
-};
-
-export function reportFailuresToFile(
-  log: ToolingLog,
-  failures: TestFailure[],
-  bkMeta: BuildkiteMetadata
-) {
-  if (!failures?.length) {
-    return;
-  }
-
-  let screenshots: string[];
-  try {
-    screenshots = [
-      ...findScreenshots(join(REPO_ROOT, 'test', 'functional')),
-      ...findScreenshots(join(REPO_ROOT, 'x-pack', 'test', 'functional')),
-    ];
-  } catch (e) {
-    log.error(e as Error);
-    screenshots = [];
-  }
-
-  const screenshotsByName: Record<string, string> = {};
-  for (const screenshot of screenshots) {
-    const [name] = basename(screenshot).split('.');
-    screenshotsByName[name] = screenshot;
-  }
-
-  // Jest could, in theory, fail 1000s of tests and write 1000s of failures
-  // So let's just write files for the first 20
-  for (const failure of failures.slice(0, 20)) {
-    const hash = createHash('md5').update(failure.name).digest('hex');
-    const filenameBase = `${
-      process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : ''
-    }${hash}`;
-    const dir = join('target', 'test_failures');
-
-    const failureLog = [
-      ['Test:', '-----', failure.classname, failure.name, ''],
-      ['Failure:', '--------', failure.failure],
-      failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [],
-    ]
-      .flat()
-      .join('\n');
-
-    const failureJSON = JSON.stringify(
-      {
-        ...failure,
-        hash,
-        buildId: bkMeta.buildId,
-        jobId: bkMeta.jobId,
-        url: bkMeta.url,
-        jobUrl: bkMeta.jobUrl,
-        jobName: bkMeta.jobName,
-      },
-      null,
-      2
-    );
-
-    let screenshot = '';
-    const truncatedName = failure.name.replace(/([^ a-zA-Z0-9-]+)/g, '_').slice(0, 80);
-    const failureNameHash = createHash('sha256').update(failure.name).digest('hex');
-    const screenshotName = `${truncatedName}-${failureNameHash}`;
-
-    if (screenshotsByName[screenshotName]) {
-      try {
-        screenshot = readFileSync(screenshotsByName[screenshotName]).toString('base64');
-      } catch (e) {
-        log.error(e as Error);
-      }
-    }
-
-    const screenshotHtml = screenshot
-      ? `<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${screenshot}" />`
-      : '';
-
-    const failureHTML = readFileSync(
-      resolve(
-        REPO_ROOT,
-        'packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html'
-      )
-    )
-      .toString()
-      .replace('$TITLE', escape(failure.name))
-      .replace(
-        '$MAIN',
-        `
-        ${failure.classname
-          .split('.')
-          .map((part) => `<h5>${escape(part.replace('·', '.'))}</h5>`)
-          .join('')}
-        <hr />
-        <p><strong>${escape(failure.name)}</strong></p>
-        <p>
-          <small>
-            <strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
-              failure.failureCount || 0
-            }</span>
-            ${
-              failure.githubIssue
-                ? `<br /><a href="${escape(failure.githubIssue)}">${escape(
-                    failure.githubIssue
-                  )}</a>`
-                : ''
-            }
-          </small>
-        </p>
-        ${
-          bkMeta.jobUrl
-            ? `<p>
-              <small>
-                <strong>Buildkite Job</strong><br />
-                <a href="${escape(bkMeta.jobUrl)}">${escape(bkMeta.jobUrl)}</a>
-              </small>
-            </p>`
-            : ''
-        }
-        <pre>${escape(failure.failure)}</pre>
-        ${screenshotHtml}
-        <pre>${escape(failure['system-out'] || '')}</pre>
-      `
-      );
-
-    mkdirSync(dir, { recursive: true });
-    writeFileSync(join(dir, `${filenameBase}.log`), failureLog, 'utf8');
-    writeFileSync(join(dir, `${filenameBase}.html`), failureHTML, 'utf8');
-    writeFileSync(join(dir, `${filenameBase}.json`), failureJSON, 'utf8');
-  }
-}

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -1,210 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Path from 'path';
-
-import { REPO_ROOT } from '@kbn/utils';
-import { run } from '@kbn/dev-cli-runner';
-import { createFailError, createFlagError } from '@kbn/dev-cli-errors';
-import { CiStatsReporter } from '@kbn/ci-stats-reporter';
-import globby from 'globby';
-import normalize from 'normalize-path';
-
-import { getFailures } from './get_failures';
-import { GithubApi } from './github_api';
-import { updateFailureIssue, createFailureIssue } from './report_failure';
-import { readTestReport } from './test_report';
-import { addMessagesToReport } from './add_messages_to_report';
-import { getReportMessageIter } from './report_metadata';
-import { reportFailuresToEs } from './report_failures_to_es';
-import { reportFailuresToFile } from './report_failures_to_file';
-import { getBuildkiteMetadata } from './buildkite_metadata';
-import { ExistingFailedTestIssues } from './existing_failed_test_issues';
-
-const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
-const DISABLE_MISSING_TEST_REPORT_ERRORS =
-  process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true';
-
-export function runFailedTestsReporterCli() {
-  run(
-    async ({ log, flags }) => {
-      const indexInEs = flags['index-errors'];
-
-      let updateGithub = flags['github-update'];
-      if (updateGithub && !process.env.GITHUB_TOKEN) {
-        throw createFailError(
-          'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag'
-        );
-      }
-
-      let branch: string = '';
-      if (updateGithub) {
-        let isPr = false;
-
-        if (process.env.BUILDKITE === 'true') {
-          branch = process.env.BUILDKITE_BRANCH || '';
-          isPr = process.env.BUILDKITE_PULL_REQUEST === 'true';
-          updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true';
-        } else {
-          // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
-          const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
-          branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || '';
-          isPr = !!process.env.ghprbPullId;
-
-          const isMainOrVersion = branch === 'main' || branch.match(/^\d+\.(x|\d+)$/);
-          if (!isMainOrVersion || isPr) {
-            log.info('Failure issues only created on main/version branch jobs');
-            updateGithub = false;
-          }
-        }
-
-        if (!branch) {
-          throw createFailError(
-            'Unable to determine originating branch from job name or other environment variables'
-          );
-        }
-      }
-
-      const githubApi = new GithubApi({
-        log,
-        token: process.env.GITHUB_TOKEN,
-        dryRun: !updateGithub,
-      });
-
-      const bkMeta = getBuildkiteMetadata();
-
-      try {
-        const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
-        if (typeof buildUrl !== 'string' || !buildUrl) {
-          throw createFlagError('Missing --build-url or process.env.BUILD_URL');
-        }
-
-        const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
-          normalize(Path.resolve(p))
-        );
-        log.info('Searching for reports at', patterns);
-        const reportPaths = await globby(patterns, {
-          absolute: true,
-        });
-
-        if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) {
-          // it is fine for code coverage to not have test results
-          return;
-        }
-
-        if (!reportPaths.length) {
-          throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
-        }
-
-        log.info('found', reportPaths.length, 'junit reports', reportPaths);
-
-        const existingIssues = new ExistingFailedTestIssues(log);
-        for (const reportPath of reportPaths) {
-          const report = await readTestReport(reportPath);
-          const messages = Array.from(getReportMessageIter(report));
-          const failures = getFailures(report);
-
-          await existingIssues.loadForFailures(failures);
-
-          if (indexInEs) {
-            await reportFailuresToEs(log, failures);
-          }
-
-          for (const failure of failures) {
-            const pushMessage = (msg: string) => {
-              messages.push({
-                classname: failure.classname,
-                name: failure.name,
-                message: msg,
-              });
-            };
-
-            if (failure.likelyIrrelevant) {
-              pushMessage(
-                'Failure is likely irrelevant' +
-                  (updateGithub ? ', so an issue was not created or updated' : '')
-              );
-              continue;
-            }
-
-            const existingIssue = existingIssues.getForFailure(failure);
-            if (existingIssue) {
-              const { newBody, newCount } = await updateFailureIssue(
-                buildUrl,
-                existingIssue,
-                githubApi,
-                branch
-              );
-              const url = existingIssue.github.htmlUrl;
-              existingIssue.github.body = newBody;
-              failure.githubIssue = url;
-              failure.failureCount = updateGithub ? newCount : newCount - 1;
-              pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`);
-              if (updateGithub) {
-                pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`);
-              }
-              continue;
-            }
-
-            const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
-            existingIssues.addNewlyCreated(failure, newIssue);
-            pushMessage('Test has not failed recently on tracked branches');
-            if (updateGithub) {
-              pushMessage(`Created new issue: ${newIssue.html_url}`);
-              failure.githubIssue = newIssue.html_url;
-            }
-            failure.failureCount = updateGithub ? 1 : 0;
-          }
-
-          // mutates report to include messages and writes updated report to disk
-          await addMessagesToReport({
-            report,
-            messages,
-            log,
-            reportPath,
-            dryRun: !flags['report-update'],
-          });
-
-          reportFailuresToFile(log, failures, bkMeta);
-        }
-      } finally {
-        await CiStatsReporter.fromEnv(log).metrics([
-          {
-            group: 'github api request count',
-            id: `failed test reporter`,
-            value: githubApi.getRequestCount(),
-            meta: Object.fromEntries(
-              Object.entries(bkMeta).map(
-                ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const
-              )
-            ),
-          },
-        ]);
-      }
-    },
-    {
-      description: `a cli that opens issues or updates existing issues based on junit reports`,
-      flags: {
-        boolean: ['github-update', 'report-update'],
-        string: ['build-url'],
-        default: {
-          'github-update': true,
-          'report-update': true,
-          'index-errors': true,
-          'build-url': process.env.BUILD_URL,
-        },
-        help: `
-          --no-github-update Execute the CLI without writing to Github
-          --no-report-update Execute the CLI without writing to the JUnit reports
-          --no-index-errors  Execute the CLI without indexing failures into Elasticsearch
-          --build-url        URL of the failed build, defaults to process.env.BUILD_URL
-        `,
-      },
-    }
-  );
-}

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -9,26 +9,15 @@
 import Path from 'path';
 import { inspect } from 'util';
 
-import { run, Flags } from '@kbn/dev-cli-runner';
+import { run } from '@kbn/dev-cli-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
 import { ToolingLog } from '@kbn/tooling-log';
 import { getTimeReporter } from '@kbn/ci-stats-reporter';
 import exitHook from 'exit-hook';
 
+import { readConfigFile, EsVersion } from './lib';
 import { FunctionalTestRunner } from './functional_test_runner';
 
-const makeAbsolutePath = (v: string) => Path.resolve(process.cwd(), v);
-const toArray = (v: string | string[]) => ([] as string[]).concat(v || []);
-const parseInstallDir = (flags: Flags) => {
-  const flag = flags['kibana-install-dir'];
-
-  if (typeof flag !== 'string' && flag !== undefined) {
-    throw createFlagError('--kibana-install-dir must be a string or not defined');
-  }
-
-  return flag ? makeAbsolutePath(flag) : undefined;
-};
-
 export function runFtrCli() {
   const runStartTime = Date.now();
   const toolingLog = new ToolingLog({
@@ -37,52 +26,49 @@ export function runFtrCli() {
   });
   const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner');
   run(
-    async ({ flags, log }) => {
+    async ({ flagsReader, log }) => {
-      const esVersion = flags['es-version'] || undefined; // convert "" to undefined
+      const esVersionInput = flagsReader.string('es-version');
-      if (esVersion !== undefined && typeof esVersion !== 'string') {
+
-        throw createFlagError('expected --es-version to be a string');
+      const configPaths = [
+        ...(flagsReader.arrayOfStrings('config') ?? []),
+        ...(flagsReader.arrayOfStrings('journey') ?? []),
+      ].map((rel) => Path.resolve(rel));
+      if (configPaths.length !== 1) {
+        throw createFlagError(`Expected there to be exactly one --config/--journey flag`);
       }
 
-      const configRel = flags.config;
+      const esVersion = esVersionInput ? new EsVersion(esVersionInput) : EsVersion.getDefault();
-      if (typeof configRel !== 'string' || !configRel) {
+      const settingOverrides = {
-        throw createFlagError('--config is required');
+        mochaOpts: {
-      }
+          bail: flagsReader.boolean('bail'),
-      const configPath = makeAbsolutePath(configRel);
+          dryRun: flagsReader.boolean('dry-run'),
-
+          grep: flagsReader.string('grep'),
-      const functionalTestRunner = new FunctionalTestRunner(
+          invert: flagsReader.boolean('invert'),
-        log,
-        configPath,
-        {
-          mochaOpts: {
-            bail: flags.bail,
-            dryRun: flags['dry-run'],
-            grep: flags.grep || undefined,
-            invert: flags.invert,
-          },
-          kbnTestServer: {
-            installDir: parseInstallDir(flags),
-          },
-          suiteFiles: {
-            include: toArray(flags.include as string | string[]).map(makeAbsolutePath),
-            exclude: toArray(flags.exclude as string | string[]).map(makeAbsolutePath),
-          },
-          suiteTags: {
-            include: toArray(flags['include-tag'] as string | string[]),
-            exclude: toArray(flags['exclude-tag'] as string | string[]),
-          },
-          updateBaselines: flags.updateBaselines || flags.u,
-          updateSnapshots: flags.updateSnapshots || flags.u,
         },
-        esVersion
+        kbnTestServer: {
-      );
+          installDir: flagsReader.path('kibana-install-dir'),
+        },
+        suiteFiles: {
+          include: flagsReader.arrayOfPaths('include') ?? [],
+          exclude: flagsReader.arrayOfPaths('exclude') ?? [],
+        },
+        suiteTags: {
+          include: flagsReader.arrayOfStrings('include-tag') ?? [],
+          exclude: flagsReader.arrayOfStrings('exclude-tag') ?? [],
+        },
+        updateBaselines: flagsReader.boolean('updateBaselines') || flagsReader.boolean('u'),
+        updateSnapshots: flagsReader.boolean('updateSnapshots') || flagsReader.boolean('u'),
+      };
 
-      await functionalTestRunner.readConfigFile();
+      const config = await readConfigFile(log, esVersion, configPaths[0], settingOverrides);
 
-      if (flags.throttle) {
+      const functionalTestRunner = new FunctionalTestRunner(log, config, esVersion);
+
+      if (flagsReader.boolean('throttle')) {
         process.env.TEST_THROTTLE_NETWORK = '1';
       }
 
-      if (flags.headless) {
+      if (flagsReader.boolean('headless')) {
         process.env.TEST_BROWSER_HEADLESS = '1';
       }
 
@@ -95,7 +81,7 @@ export function runFtrCli() {
           await reportTime(runStartTime, 'total', {
             success: false,
             err: err.message,
-            ...flags,
+            ...Object.fromEntries(flagsReader.getUsed().entries()),
           });
           log.indent(-log.getIndent());
           log.error(err);
@@ -103,7 +89,7 @@ export function runFtrCli() {
         } else {
           await reportTime(runStartTime, 'total', {
             success: true,
-            ...flags,
+            ...Object.fromEntries(flagsReader.getUsed().entries()),
           });
         }
 
@@ -118,7 +104,7 @@ export function runFtrCli() {
       exitHook(teardown);
 
       try {
-        if (flags['test-stats']) {
+        if (flagsReader.boolean('test-stats')) {
           process.stderr.write(
             JSON.stringify(await functionalTestRunner.getTestStats(), null, 2) + '\n'
           );
@@ -139,6 +125,7 @@ export function runFtrCli() {
       flags: {
         string: [
           'config',
+          'journey',
           'grep',
           'include',
           'exclude',
@@ -159,7 +146,8 @@ export function runFtrCli() {
           'dry-run',
         ],
         help: `
-          --config=path      path to a config file
+          --config=path      path to a config file (either this or --journey is required)
+          --journey=path     path to a journey file (either this or --config is required)
           --bail             stop tests after the first failure
           --grep <pattern>   pattern used to select which tests to run
           --invert           invert grep to exclude tests

packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts

@@ -15,6 +15,7 @@
 import { EventEmitter } from 'events';
 
 export interface Suite {
+  currentTest?: Test;
   suites: Suite[];
   tests: Test[];
   title: string;