[kbn-scout] Custom event-oriented test reporter & persistence (#202906)

.github/CODEOWNERS

@@ -465,6 +465,8 @@ packages/kbn-safer-lodash-set @elastic/kibana-security
 packages/kbn-saved-objects-settings @elastic/appex-sharedux
 packages/kbn-saved-search-component @elastic/obs-ux-logs-team
 packages/kbn-scout @elastic/appex-qa
+packages/kbn-scout-info @elastic/appex-qa
+packages/kbn-scout-reporting @elastic/appex-qa
 packages/kbn-screenshotting-server @elastic/appex-sharedux
 packages/kbn-search-api-keys-components @elastic/search-kibana
 packages/kbn-search-api-keys-server @elastic/search-kibana

package.json

@@ -1493,6 +1493,8 @@
     "@kbn/repo-source-classifier": "link:packages/kbn-repo-source-classifier",
     "@kbn/repo-source-classifier-cli": "link:packages/kbn-repo-source-classifier-cli",
     "@kbn/scout": "link:packages/kbn-scout",
+    "@kbn/scout-info": "link:packages/kbn-scout-info",
+    "@kbn/scout-reporting": "link:packages/kbn-scout-reporting",
     "@kbn/security-api-integration-helpers": "link:x-pack/test/security_api_integration/packages/helpers",
     "@kbn/serverless-storybook-config": "link:packages/serverless/storybook/config",
     "@kbn/some-dev-log": "link:packages/kbn-some-dev-log",

packages/kbn-scout-info/README.md

@@ -0,0 +1,6 @@
+# @kbn/scout-info
+
+This package stores information that's commonly used by packages in the `@kbn/scout*` namespace, and any other
+ package that wishes to extend Scout functionality.
+
+Check out the `@kbn/scout` package if you want to learn more about Scout.

packages/kbn-scout-info/index.ts

@@ -7,5 +7,5 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-require('../src/setup_node_env');
+export * from './src/paths';
-require('@kbn/scout').runTestsCli();
+export * from './src/reporting';

packages/kbn-scout-info/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-scout-info'],
+};

packages/kbn-scout-info/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/scout-info",
+  "owner": "@elastic/appex-qa",
+  "devOnly": true
+}

packages/kbn-scout-info/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/scout-info",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

packages/kbn-scout-info/src/paths.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import path from 'node:path';
+import { REPO_ROOT } from '@kbn/repo-info';
+
+export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout');
+
+// Servers
+export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers');
+
+// Reporting
+export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports');

packages/kbn-scout-info/src/reporting.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export const SCOUT_TEST_EVENTS_TEMPLATE_NAME = 'scout-test-events';
+export const SCOUT_TEST_EVENTS_INDEX_PATTERN = `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-*`;
+export const SCOUT_TEST_EVENTS_DATA_STREAM_NAME = `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-kibana`;

packages/kbn-scout-info/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/repo-info"
+  ]
+}

packages/kbn-scout-reporting/README.md

@@ -0,0 +1,5 @@
+# @kbn/scout-reporting
+
+This package contains reporting functionality for Scout.
+
+Check out the `@kbn/scout` package if you want to learn more about Scout.

packages/kbn-scout-reporting/index.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * as cli from './src/cli';
+export * as datasources from './src/datasources';
+export * from './src/reporting';

packages/kbn-scout-reporting/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-scout-reporting'],
+};

packages/kbn-scout-reporting/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/scout-reporting",
+  "owner": "@elastic/appex-qa",
+  "devOnly": true
+}

packages/kbn-scout-reporting/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/scout-reporting",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

packages/kbn-scout-reporting/src/cli/common.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Client as ESClient, ClientOptions as ESClientOptions } from '@elastic/elasticsearch';
+import { ToolingLog } from '@kbn/tooling-log';
+import { createFailError } from '@kbn/dev-cli-errors';
+
+/**
+ * Get an Elasticsearch client for which connectivity has been validated
+ *
+ * @param options Elasticsearch client options
+ * @param log Logger instance
+ * @throws FailError if cluster information cannot be read from the target Elasticsearch instance
+ */
+export async function getValidatedESClient(
+  options: ESClientOptions,
+  log: ToolingLog
+): Promise<ESClient> {
+  const es = new ESClient(options);
+
+  await es.info().then(
+    (esInfo) => {
+      log.info(`Connected to Elasticsearch node '${esInfo.name}'`);
+    },
+    (err) => {
+      throw createFailError(`Failed to connect to Elasticsearch\n${err}`);
+    }
+  );
+
+  return es;
+}

packages/kbn-scout-reporting/src/cli/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export { initializeReportDatastream } from './initialize_report_datastream';
+export { uploadEvents } from './upload_events';

packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Command } from '@kbn/dev-cli-runner';
+import { ScoutReportDataStream } from '../reporting/report';
+import { getValidatedESClient } from './common';
+
+export const initializeReportDatastream: Command<void> = {
+  name: 'initialize-report-datastream',
+  description: 'Initialize a Scout report datastream in Elasticsearch',
+  flags: {
+    string: ['esURL', 'esAPIKey'],
+    boolean: ['verifyTLSCerts'],
+    default: {
+      esURL: process.env.ES_URL,
+      esAPIKey: process.env.ES_API_KEY,
+    },
+    help: `
+    --esURL           (required)  Elasticsearch URL [env: ES_URL]
+    --esAPIKey        (required)  Elasticsearch API Key [env: ES_API_KEY]
+    --verifyTLSCerts  (optional)  Verify TLS certificates
+    `,
+  },
+  run: async ({ flagsReader, log }) => {
+    const esURL = flagsReader.requiredString('esURL');
+    const esAPIKey = flagsReader.requiredString('esAPIKey');
+
+    // ES connection
+    log.info(`Connecting to Elasticsearch at ${esURL}`);
+    const es = await getValidatedESClient(
+      {
+        node: esURL,
+        auth: { apiKey: esAPIKey },
+        tls: {
+          rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'),
+        },
+      },
+      log
+    );
+
+    // Initialize the report datastream
+    const reportDataStream = new ScoutReportDataStream(es, log);
+    await reportDataStream.initialize();
+
+    log.success('Scout report data stream initialized');
+  },
+};

packages/kbn-scout-reporting/src/cli/upload_events.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import fs from 'node:fs';
+import { Command } from '@kbn/dev-cli-runner';
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { ScoutReportDataStream } from '../reporting/report';
+import { getValidatedESClient } from './common';
+
+export const uploadEvents: Command<void> = {
+  name: 'upload-events',
+  description: 'Upload events recorded by the Scout reporter to Elasticsearch',
+  flags: {
+    string: ['eventLogPath', 'esURL', 'esAPIKey'],
+    boolean: ['verifyTLSCerts'],
+    default: {
+      esURL: process.env.ES_URL,
+      esAPIKey: process.env.ES_API_KEY,
+    },
+    help: `
+    --eventLogPath    (required)  Path to the event log to upload
+    --esURL           (required)  Elasticsearch URL [env: ES_URL]
+    --esAPIKey        (required)  Elasticsearch API Key [env: ES_API_KEY]
+    --verifyTLSCerts  (optional)  Verify TLS certificates
+    `,
+  },
+  run: async ({ flagsReader, log }) => {
+    // Read & validate CLI options
+    const eventLogPath = flagsReader.requiredString('eventLogPath');
+
+    if (!fs.existsSync(eventLogPath)) {
+      throw createFlagError(`Event log path '${eventLogPath}' does not exist.`);
+    }
+
+    const esURL = flagsReader.requiredString('esURL');
+    const esAPIKey = flagsReader.requiredString('esAPIKey');
+
+    // ES connection
+    log.info(`Connecting to Elasticsearch at ${esURL}`);
+    const es = await getValidatedESClient(
+      {
+        node: esURL,
+        auth: { apiKey: esAPIKey },
+        tls: {
+          rejectUnauthorized: flagsReader.boolean('verifyTLSCerts'),
+        },
+      },
+      log
+    );
+
+    // Event log upload
+    const reportDataStream = new ScoutReportDataStream(es, log);
+    await reportDataStream.addEventsFromFile(eventLogPath);
+  },
+};

packages/kbn-scout-reporting/src/datasources/buildkite.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * Buildkite info
+ */
+export interface BuildkiteMetadata {
+  branch?: string;
+  commit?: string;
+  job_id?: string;
+  message?: string;
+  build: {
+    id?: string;
+    number?: string;
+    url?: string;
+  };
+  pipeline: {
+    id?: string;
+    name?: string;
+    slug?: string;
+  };
+  agent: {
+    name?: string;
+  };
+  group: {
+    id?: string;
+    key?: string;
+    label?: string;
+  };
+  step: {
+    id?: string;
+    key?: string;
+    label?: string;
+  };
+  command?: string;
+}
+
+/**
+ * Buildkite information extracted from environment variables
+ *
+ * This object is empty if the process is not running in a Buildkite pipeline.
+ */
+export const buildkite: BuildkiteMetadata =
+  process.env.BUILDKITE === 'true'
+    ? {
+        branch: process.env.BUILDKITE_BRANCH,
+        commit: process.env.BUILDKITE_COMMIT,
+        job_id: process.env.BUILDKITE_JOB_ID,
+        message: process.env.BUILDKITE_MESSAGE,
+        build: {
+          id: process.env.BUILDKITE_BUILD_ID,
+          number: process.env.BUILDKITE_BUILD_NUMBER,
+          url: process.env.BUILDKITE_BUILD_URL,
+        },
+        pipeline: {
+          id: process.env.BUILDKITE_PIPELINE_ID,
+          name: process.env.BUILDKITE_PIPELINE_NAME,
+          slug: process.env.BUILDKITE_PIPELINE_SLUG,
+        },
+        agent: {
+          name: process.env.BUILDKITE_AGENT_NAME,
+        },
+        group: {
+          id: process.env.BUILDKITE_GROUP_ID,
+          key: process.env.BUILDKITE_GROUP_KEY,
+          label: process.env.BUILDKITE_GROUP_LABEL,
+        },
+        step: {
+          id: process.env.BUILDKITE_STEP_ID,
+          key: process.env.BUILDKITE_STEP_KEY,
+          label: process.env.BUILDKITE_LABEL,
+        },
+        command: process.env.BUILDKITE_COMMAND,
+      }
+    : {
+        build: {},
+        pipeline: {},
+        agent: {},
+        group: {},
+        step: {},
+      };

packages/kbn-scout-reporting/src/datasources/host.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import os from 'node:os';
+
+/**
+ * Host info
+ */
+export interface HostMetadata {
+  architecture: string;
+  hostname: string;
+  os: OSMetadata;
+}
+
+/**
+ * Operating system info
+ */
+export interface OSMetadata {
+  platform: string;
+  version: string;
+  family: string;
+}
+
+/**
+ * Information about the host this process is running on
+ */
+export const host: HostMetadata = {
+  architecture: os.arch(),
+  hostname: os.hostname(),
+  os: {
+    platform: os.platform(),
+    version: os.release(),
+    family: os.type(),
+  },
+};

packages/kbn-scout-reporting/src/datasources/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { buildkite } from './buildkite';
+import { host } from './host';
+
+export * from './buildkite';
+export * from './host';
+
+export const environmentMetadata = {
+  buildkite,
+  host,
+};

packages/kbn-scout-reporting/src/reporting/index.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { createHash, randomBytes } from 'node:crypto';
+import type { ReporterDescription } from 'playwright/test';
+import type { ScoutPlaywrightReporterOptions } from './playwright';
+
+export * from './report';
+
+// ID helpers
+export function generateTestRunId() {
+  return randomBytes(8).toString('hex');
+}
+
+export function getTestIDForTitle(title: string) {
+  return createHash('sha256').update(title).digest('hex').slice(0, 31);
+}
+
+// Playwright reporting
+export const scoutPlaywrightReporter = (
+  options?: ScoutPlaywrightReporterOptions
+): ReporterDescription => {
+  return ['@kbn/scout-reporting/src/reporting/playwright.ts', options];
+};

packages/kbn-scout-reporting/src/reporting/playwright.ts

@@ -0,0 +1,300 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type {
+  FullConfig,
+  FullResult,
+  Reporter,
+  Suite,
+  TestCase,
+  TestError,
+  TestResult,
+  TestStep,
+} from '@playwright/test/reporter';
+
+import path from 'node:path';
+import { ToolingLog } from '@kbn/tooling-log';
+import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info';
+import stripANSI from 'strip-ansi';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { PathWithOwners, getPathsWithOwnersReversed, getCodeOwnersForFile } from '@kbn/code-owners';
+import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '.';
+import { environmentMetadata } from '../datasources';
+
+/**
+ * Configuration options for the Scout Playwright reporter
+ */
+export interface ScoutPlaywrightReporterOptions {
+  name?: string;
+  outputPath?: string;
+}
+
+/**
+ * Scout Playwright reporter
+ */
+export class ScoutPlaywrightReporter implements Reporter {
+  readonly log: ToolingLog;
+  readonly name: string;
+  readonly runId: string;
+  private report: ScoutReport;
+  private readonly pathsWithOwners: PathWithOwners[];
+
+  constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) {
+    this.log = new ToolingLog({
+      level: 'info',
+      writeTo: process.stdout,
+    });
+
+    this.name = this.reporterOptions.name || 'unknown';
+    this.runId = generateTestRunId();
+    this.log.info(`Scout test run ID: ${this.runId}`);
+
+    this.report = new ScoutReport(this.log);
+    this.pathsWithOwners = getPathsWithOwnersReversed();
+  }
+
+  private getFileOwners(filePath: string): string[] {
+    const concatenatedOwners = getCodeOwnersForFile(filePath, this.pathsWithOwners);
+
+    if (concatenatedOwners === undefined) {
+      return [];
+    }
+
+    return concatenatedOwners
+      .replace(/#.+$/, '')
+      .split(',')
+      .filter((value) => value.length > 0);
+  }
+
+  /**
+   * Root path of this reporter's output
+   */
+  public get reportRootPath(): string {
+    const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT;
+    return path.join(outputPath, `scout-playwright-${this.runId}`);
+  }
+
+  printsToStdio(): boolean {
+    // Don't take over console output
+    return false;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.report.logEvent({
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      event: {
+        action: ScoutReportEventAction.RUN_BEGIN,
+      },
+    });
+  }
+
+  onTestBegin(test: TestCase, result: TestResult) {
+    this.report.logEvent({
+      '@timestamp': result.startTime,
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent.titlePath().join(' '),
+        type: test.parent.type,
+      },
+      test: {
+        id: getTestIDForTitle(test.titlePath().join(' ')),
+        title: test.title,
+        tags: test.tags,
+        annotations: test.annotations,
+        expected_status: test.expectedStatus,
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_BEGIN,
+      },
+      file: {
+        path: path.relative(REPO_ROOT, test.location.file),
+        owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)),
+      },
+    });
+  }
+
+  onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
+    this.report.logEvent({
+      '@timestamp': step.startTime,
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent.titlePath().join(' '),
+        type: test.parent.type,
+      },
+      test: {
+        id: getTestIDForTitle(test.titlePath().join(' ')),
+        title: test.title,
+        tags: test.tags,
+        annotations: test.annotations,
+        expected_status: test.expectedStatus,
+        step: {
+          title: step.titlePath().join(' '),
+          category: step.category,
+        },
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_STEP_BEGIN,
+      },
+      file: {
+        path: path.relative(REPO_ROOT, test.location.file),
+        owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)),
+      },
+    });
+  }
+
+  onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
+    this.report.logEvent({
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent.titlePath().join(' '),
+        type: test.parent.type,
+      },
+      test: {
+        id: getTestIDForTitle(test.titlePath().join(' ')),
+        title: test.title,
+        tags: test.tags,
+        annotations: test.annotations,
+        expected_status: test.expectedStatus,
+        step: {
+          title: step.titlePath().join(' '),
+          category: step.category,
+          duration: step.duration,
+        },
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_STEP_END,
+        error: {
+          message: step.error?.message ? stripANSI(step.error.message) : undefined,
+          stack_trace: step.error?.stack ? stripANSI(step.error.stack) : undefined,
+        },
+      },
+      file: {
+        path: path.relative(REPO_ROOT, test.location.file),
+        owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)),
+      },
+    });
+  }
+
+  onTestEnd(test: TestCase, result: TestResult) {
+    this.report.logEvent({
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent.titlePath().join(' '),
+        type: test.parent.type,
+      },
+      test: {
+        id: getTestIDForTitle(test.titlePath().join(' ')),
+        title: test.title,
+        tags: test.tags,
+        annotations: test.annotations,
+        expected_status: test.expectedStatus,
+        status: result.status,
+        duration: result.duration,
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_END,
+        error: {
+          message: result.error?.message ? stripANSI(result.error.message) : undefined,
+          stack_trace: result.error?.stack ? stripANSI(result.error.stack) : undefined,
+        },
+      },
+      file: {
+        path: path.relative(REPO_ROOT, test.location.file),
+        owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)),
+      },
+    });
+  }
+
+  onEnd(result: FullResult) {
+    this.report.logEvent({
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+        status: result.status,
+        duration: result.duration,
+      },
+      event: {
+        action: ScoutReportEventAction.RUN_END,
+      },
+    });
+
+    // Save & conclude the report
+    try {
+      this.report.save(this.reportRootPath);
+    } finally {
+      this.report.conclude();
+    }
+  }
+
+  async onExit() {
+    // noop
+  }
+
+  onError(error: TestError) {
+    this.report.logEvent({
+      ...environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'playwright',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      event: {
+        action: ScoutReportEventAction.ERROR,
+        error: {
+          message: error.message ? stripANSI(error.message) : undefined,
+          stack_trace: error.stack ? stripANSI(error.stack) : undefined,
+        },
+      },
+    });
+  }
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ScoutPlaywrightReporter;

packages/kbn-scout-reporting/src/reporting/report/event.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { BuildkiteMetadata, HostMetadata } from '../../datasources';
+
+/**
+ * Scout reporter event type
+ */
+export enum ScoutReportEventAction {
+  RUN_BEGIN = 'run-begin',
+  RUN_END = 'run-end',
+  TEST_BEGIN = 'test-begin',
+  TEST_END = 'test-end',
+  TEST_STEP_BEGIN = 'test-step-begin',
+  TEST_STEP_END = 'test-step-end',
+  ERROR = 'error',
+}
+
+/**
+ * Scout report event info
+ */
+export interface ScoutReportEventInfo {
+  action: ScoutReportEventAction;
+  outcome?: 'failure' | 'success' | 'unknown';
+  error?: {
+    message?: string;
+    id?: string;
+    code?: string;
+    stack_trace?: string;
+    type?: string;
+  };
+}
+
+/**
+ * Scout reporter info
+ */
+export interface ScoutReporterInfo {
+  name: string;
+  type: 'jest' | 'ftr' | 'playwright';
+}
+
+/**
+ * Scout test run info
+ */
+export interface ScoutTestRunInfo {
+  id: string;
+  status?: string;
+  duration?: number;
+}
+
+/**
+ * Scout suite info
+ */
+export interface ScoutSuiteInfo {
+  title: string;
+  type: string;
+}
+
+/**
+ * Scout test info
+ */
+export interface ScoutTestInfo {
+  id: string;
+  title: string;
+  tags: string[];
+  annotations?: Array<{
+    type: string;
+    description?: string;
+  }>;
+  expected_status?: string;
+  duration?: number;
+  status?: string;
+  step?: {
+    title: string;
+    category?: string;
+    duration?: number;
+  };
+}
+
+/**
+ * Scout file info
+ */
+export interface ScoutFileInfo {
+  path: string;
+  owner: string | string[];
+}
+
+/**
+ * Document that records an event to be logged by the Scout reporter
+ */
+export interface ScoutReportEvent {
+  '@timestamp'?: Date;
+  buildkite?: BuildkiteMetadata;
+  host?: HostMetadata;
+  event: ScoutReportEventInfo;
+  file?: ScoutFileInfo;
+  labels?: { [id: string]: string };
+  reporter: ScoutReporterInfo;
+  test_run: ScoutTestRunInfo;
+  suite?: ScoutSuiteInfo;
+  test?: ScoutTestInfo;
+}

packages/kbn-scout-reporting/src/reporting/report/index.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+// eslint-disable-next-line max-classes-per-file
+import os from 'node:os';
+import path from 'node:path';
+import fs from 'node:fs';
+import { ToolingLog } from '@kbn/tooling-log';
+import { ScoutReportEvent } from './event';
+
+/**
+ * Generic error raised by a Scout report
+ */
+export class ScoutReportError extends Error {}
+
+/**
+ *
+ */
+export class ScoutReport {
+  log: ToolingLog;
+  workDir: string;
+  concluded = false;
+
+  constructor(log?: ToolingLog) {
+    this.log = log || new ToolingLog();
+    this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-'));
+  }
+
+  public get eventLogPath(): string {
+    return path.join(this.workDir, `event-log.ndjson`);
+  }
+
+  private raiseIfConcluded(additionalInfo?: string) {
+    if (this.concluded) {
+      let message = `Report at ${this.workDir} was concluded`;
+
+      if (additionalInfo) {
+        message += `: ${additionalInfo}`;
+      }
+
+      throw new ScoutReportError(message);
+    }
+  }
+
+  /**
+   * Logs an event to be processed by this reporter
+   *
+   * @param event {ScoutReportEvent} - Event to record
+   */
+  logEvent(event: ScoutReportEvent) {
+    this.raiseIfConcluded('logging new events is no longer allowed');
+
+    if (event['@timestamp'] === undefined) {
+      event['@timestamp'] = new Date();
+    }
+
+    fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n');
+  }
+
+  /**
+   * Save the report to a non-ephemeral location
+   *
+   * @param destination - Full path to the save location. Must not exist.
+   */
+  save(destination: string) {
+    this.raiseIfConcluded('nothing to save because workdir has been cleared');
+
+    if (fs.existsSync(destination)) {
+      throw new ScoutReportError(`Save destination path '${destination}' already exists`);
+    }
+
+    // Create the destination directory
+    this.log.info(`Saving Scout report to ${destination}`);
+    fs.mkdirSync(destination, { recursive: true });
+
+    // Copy the workdir data to the destination
+    fs.cpSync(this.workDir, destination, { recursive: true });
+  }
+
+  /**
+   * Call this when you're done adding information to this report.
+   *
+   * ⚠️**This will delete all the contents of the report's working directory**
+   */
+  conclude() {
+    // Remove the working directory
+    this.log.info(`Removing Scout report working directory ${this.workDir}`);
+    fs.rmSync(this.workDir, { recursive: true, force: true });
+
+    // Mark this report as concluded
+    this.concluded = true;
+    this.log.success('Scout report has concluded.');
+  }
+}
+
+export * from './event';
+export * from './persistence';

packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import {
+  buildkiteProperties,
+  reporterProperties,
+  testRunProperties,
+  suiteProperties,
+  testProperties,
+} from './mappings';
+
+export const buildkiteMappings: ClusterPutComponentTemplateRequest = {
+  name: 'scout-test-event.mappings.buildkite',
+  version: 1,
+  template: {
+    mappings: {
+      properties: {
+        buildkite: {
+          type: 'object',
+          properties: buildkiteProperties,
+        },
+      },
+    },
+  },
+};
+
+export const reporterMappings: ClusterPutComponentTemplateRequest = {
+  name: 'scout-test-event.mappings.reporter',
+  version: 1,
+  template: {
+    mappings: {
+      properties: {
+        reporter: {
+          type: 'object',
+          properties: reporterProperties,
+        },
+      },
+    },
+  },
+};
+
+export const testRunMappings: ClusterPutComponentTemplateRequest = {
+  name: 'scout-test-event.mappings.test-run',
+  version: 1,
+  template: {
+    mappings: {
+      properties: {
+        test_run: {
+          type: 'object',
+          properties: testRunProperties,
+        },
+      },
+    },
+  },
+};
+
+export const suiteMappings: ClusterPutComponentTemplateRequest = {
+  name: 'scout-test-event.mappings.suite',
+  version: 1,
+  template: {
+    mappings: {
+      properties: {
+        suite: {
+          type: 'object',
+          properties: suiteProperties,
+        },
+      },
+    },
+  },
+};
+
+export const testMappings: ClusterPutComponentTemplateRequest = {
+  name: 'scout-test-event.mappings.test',
+  version: 1,
+  template: {
+    mappings: {
+      properties: {
+        test: {
+          type: 'object',
+          properties: testProperties,
+        },
+      },
+    },
+  },
+};

packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts

@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline';
+import { ToolingLog } from '@kbn/tooling-log';
+import { Client as ESClient } from '@elastic/elasticsearch';
+import { SCOUT_TEST_EVENTS_DATA_STREAM_NAME } from '@kbn/scout-info';
+import { ScoutReportEvent } from '../event';
+import * as componentTemplates from './component_templates';
+import * as indexTemplates from './index_templates';
+
+export class ScoutReportDataStream {
+  private log: ToolingLog;
+
+  constructor(private es: ESClient, log?: ToolingLog) {
+    this.log = log || new ToolingLog();
+  }
+
+  async exists() {
+    return await this.es.indices.exists({ index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME });
+  }
+
+  async initialize() {
+    await this.setupComponentTemplates();
+    await this.setupIndexTemplate();
+
+    if (await this.exists()) {
+      return;
+    }
+
+    this.log.info(`Creating data stream '${SCOUT_TEST_EVENTS_DATA_STREAM_NAME}'`);
+    await this.es.indices.createDataStream({
+      name: SCOUT_TEST_EVENTS_DATA_STREAM_NAME,
+    });
+  }
+
+  async setupComponentTemplates() {
+    for (const template of [
+      componentTemplates.buildkiteMappings,
+      componentTemplates.reporterMappings,
+      componentTemplates.testRunMappings,
+      componentTemplates.suiteMappings,
+      componentTemplates.testMappings,
+    ]) {
+      const templateExists = await this.es.cluster.existsComponentTemplate({ name: template.name });
+      if (!templateExists) {
+        this.log.info(`Creating component template '${template.name}'`);
+        await this.es.cluster.putComponentTemplate(template);
+        continue;
+      }
+
+      // Template exists but might need to be updated
+      const newTemplateVersion = template.version || 0;
+      const existingTemplateVersion =
+        (await this.es.cluster.getComponentTemplate({ name: template.name })).component_templates[0]
+          .component_template.version || 0;
+
+      if (existingTemplateVersion >= newTemplateVersion) {
+        this.log.info(`Component template '${template.name} exists and is up to date.`);
+        continue;
+      }
+
+      this.log.info(
+        `Updating component template '${template.name}' (version ${existingTemplateVersion} -> ${newTemplateVersion})`
+      );
+      await this.es.cluster.putComponentTemplate(template);
+    }
+  }
+
+  async setupIndexTemplate() {
+    const template = indexTemplates.testEvents;
+    const templateExists: boolean = await this.es.indices.existsIndexTemplate({
+      name: template.name,
+    });
+
+    if (!templateExists) {
+      this.log.info(`Creating index template '${template.name}'`);
+      await this.es.indices.putIndexTemplate(template);
+      return;
+    }
+
+    // Template exists but might need to be updated
+    const newTemplateVersion = template.version || 0;
+    const existingTemplateVersion =
+      (await this.es.indices.getIndexTemplate({ name: template.name })).index_templates[0]
+        .index_template.version || 0;
+
+    if (existingTemplateVersion >= newTemplateVersion) {
+      this.log.info(`Index template '${template.name} exists and is up to date.`);
+      return;
+    }
+
+    this.log.info(
+      `Updating index template '${template.name}' (version ${existingTemplateVersion} -> ${newTemplateVersion})`
+    );
+    await this.es.indices.putIndexTemplate(template);
+  }
+
+  async addEvent(event: ScoutReportEvent) {
+    await this.es.index({ index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME, document: event });
+  }
+
+  async addEventsFromFile(eventLogPath: string) {
+    // Make the given event log path absolute
+    eventLogPath = path.resolve(eventLogPath);
+
+    const events = async function* () {
+      const lineReader = readline.createInterface({
+        input: fs.createReadStream(eventLogPath),
+        crlfDelay: Infinity,
+      });
+
+      for await (const line of lineReader) {
+        yield line;
+      }
+    };
+
+    this.log.info(
+      `Uploading events from file ${eventLogPath} to data stream '${SCOUT_TEST_EVENTS_DATA_STREAM_NAME}'`
+    );
+
+    const stats = await this.es.helpers.bulk({
+      datasource: events(),
+      onDocument: () => {
+        return { create: { _index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME } };
+      },
+    });
+
+    this.log.info(`Uploaded ${stats.total} events in ${stats.time / 1000}s.`);
+
+    if (stats.failed > 0) {
+      this.log.warning(`Failed to upload ${stats.failed} events`);
+    }
+  }
+}

packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import { SCOUT_TEST_EVENTS_TEMPLATE_NAME, SCOUT_TEST_EVENTS_INDEX_PATTERN } from '@kbn/scout-info';
+import * as componentTemplates from './component_templates';
+
+export const testEvents: IndicesPutIndexTemplateRequest = {
+  name: SCOUT_TEST_EVENTS_TEMPLATE_NAME,
+  version: 1,
+  data_stream: {},
+  index_patterns: SCOUT_TEST_EVENTS_INDEX_PATTERN,
+  composed_of: [
+    'ecs@mappings',
+    componentTemplates.buildkiteMappings.name,
+    componentTemplates.reporterMappings.name,
+    componentTemplates.testRunMappings.name,
+    componentTemplates.suiteMappings.name,
+    componentTemplates.testMappings.name,
+  ],
+};

packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts

@@ -0,0 +1,172 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { PropertyName, MappingProperty } from '@elastic/elasticsearch/lib/api/types';
+
+export const buildkiteProperties: Record<PropertyName, MappingProperty> = {
+  branch: {
+    type: 'keyword',
+  },
+  commit: {
+    type: 'wildcard',
+  },
+  job_id: {
+    type: 'wildcard',
+  },
+  message: {
+    type: 'text',
+  },
+  build: {
+    type: 'object',
+    properties: {
+      id: {
+        type: 'wildcard',
+      },
+      number: {
+        type: 'integer',
+      },
+      url: {
+        type: 'wildcard',
+      },
+    },
+  },
+  pipeline: {
+    type: 'object',
+    properties: {
+      id: {
+        type: 'wildcard',
+      },
+      name: {
+        type: 'text',
+      },
+      slug: {
+        type: 'wildcard',
+      },
+    },
+  },
+  agent: {
+    type: 'object',
+    properties: {
+      name: {
+        type: 'wildcard',
+      },
+    },
+  },
+  group: {
+    type: 'object',
+    properties: {
+      id: {
+        type: 'wildcard',
+      },
+      key: {
+        type: 'wildcard',
+      },
+      label: {
+        type: 'keyword',
+      },
+    },
+  },
+  step: {
+    type: 'object',
+    properties: {
+      id: {
+        type: 'wildcard',
+      },
+      key: {
+        type: 'wildcard',
+      },
+      label: {
+        type: 'keyword',
+      },
+    },
+  },
+  command: {
+    type: 'wildcard',
+    fields: {
+      text: {
+        type: 'match_only_text',
+      },
+    },
+  },
+};
+
+export const reporterProperties: Record<PropertyName, MappingProperty> = {
+  name: {
+    type: 'text',
+  },
+  type: {
+    type: 'keyword',
+  },
+};
+
+export const testRunProperties: Record<PropertyName, MappingProperty> = {
+  id: {
+    type: 'wildcard',
+  },
+  status: {
+    type: 'keyword',
+  },
+  duration: {
+    type: 'long',
+  },
+};
+
+export const suiteProperties: Record<PropertyName, MappingProperty> = {
+  title: {
+    type: 'text',
+  },
+  type: {
+    type: 'keyword',
+  },
+};
+
+export const testProperties: Record<PropertyName, MappingProperty> = {
+  id: {
+    type: 'wildcard',
+  },
+  title: {
+    type: 'text',
+  },
+  tags: {
+    type: 'keyword',
+  },
+  annotations: {
+    type: 'object',
+    properties: {
+      type: {
+        type: 'keyword',
+      },
+      description: {
+        type: 'text',
+      },
+    },
+  },
+  expected_status: {
+    type: 'keyword',
+  },
+  duration: {
+    type: 'long',
+  },
+  status: {
+    type: 'keyword',
+  },
+  step: {
+    type: 'object',
+    properties: {
+      title: {
+        type: 'text',
+      },
+      category: {
+        type: 'keyword',
+      },
+      duration: {
+        type: 'long',
+      },
+    },
+  },
+};

packages/kbn-scout-reporting/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/tooling-log",
+    "@kbn/dev-cli-runner",
+    "@kbn/dev-cli-errors",
+    "@kbn/scout-info",
+    "@kbn/repo-info",
+    "@kbn/code-owners",
+  ]
+}

packages/kbn-scout/index.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export { startServersCli, runTestsCli } from './src/cli';
+export * as cli from './src/cli';
 export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
 export type {
   ScoutPage,

packages/kbn-scout/src/cli/index.ts

@@ -6,6 +6,16 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+import { RunWithCommands } from '@kbn/dev-cli-runner';
+import { cli as reportingCLI } from '@kbn/scout-reporting';
+import { startServer } from './start_server';
+import { runTests } from './run_tests';
 
-export { runTestsCli } from './run_tests_cli';
+export async function run() {
-export { startServersCli } from './start_servers_cli';
+  await new RunWithCommands(
+    {
+      description: 'Scout CLI',
+    },
+    [startServer, runTests, reportingCLI.initializeReportDatastream, reportingCLI.uploadEvents]
+  ).execute();
+}

packages/kbn-scout/src/cli/run_tests.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Command } from '@kbn/dev-cli-runner';
+import { initLogsDir } from '@kbn/test';
+import { TEST_FLAG_OPTIONS } from '../playwright/runner';
+import { parseTestFlags, runTests as runTestsFn } from '../playwright/runner';
+
+/**
+ * Start servers and run the tests
+ */
+export const runTests: Command<void> = {
+  name: 'run-tests',
+  description: `
+  Run a Scout Playwright config.
+
+  Note:
+    This also handles server starts. Make sure a Scout test server is not already running before invoking this command.
+
+  Common usage:
+    node scripts/scout run-tests --stateful --config <playwright_config_path>
+    node scripts/scout run-tests --serverless=es --headed --config <playwright_config_path>
+  `,
+  flags: TEST_FLAG_OPTIONS,
+  run: async ({ flagsReader, log }) => {
+    const options = await parseTestFlags(flagsReader);
+
+    if (options.logsDir) {
+      await initLogsDir(log, options.logsDir);
+    }
+
+    await runTestsFn(log, options);
+  },
+};

packages/kbn-scout/src/cli/run_tests_cli.ts

@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { run } from '@kbn/dev-cli-runner';
-import { initLogsDir } from '@kbn/test';
-import { TEST_FLAG_OPTIONS, parseTestFlags, runTests } from '../playwright/runner';
-
-/**
- * Start servers and run the tests
- */
-export function runTestsCli() {
-  run(
-    async ({ flagsReader, log }) => {
-      const options = await parseTestFlags(flagsReader);
-
-      if (options.logsDir) {
-        initLogsDir(log, options.logsDir);
-      }
-
-      await runTests(log, options);
-    },
-    {
-      description: `Run Scout UI Tests`,
-      usage: `
-      Usage:
-        node scripts/scout_test --help
-        node scripts/scout_test --stateful --config <playwright_config_path>
-        node scripts/scout_test --serverless=es --headed --config <playwright_config_path>
-      `,
-      flags: TEST_FLAG_OPTIONS,
-    }
-  );
-}

packages/kbn-scout/src/cli/start_server.ts

@@ -7,8 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { run } from '@kbn/dev-cli-runner';
+import { Command } from '@kbn/dev-cli-runner';
-
 import { initLogsDir } from '@kbn/test';
 
 import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers';
@@ -16,19 +15,16 @@ import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers'
 /**
  * Start servers
  */
-export function startServersCli() {
+export const startServer: Command<void> = {
-  run(
+  name: 'start-server',
-    async ({ flagsReader: flags, log }) => {
+  description: 'Start Elasticsearch & Kibana for testing purposes',
-      const options = parseServerFlags(flags);
+  flags: SERVER_FLAG_OPTIONS,
+  run: async ({ flagsReader, log }) => {
+    const options = parseServerFlags(flagsReader);
 
     if (options.logsDir) {
-        initLogsDir(log, options.logsDir);
+      await initLogsDir(log, options.logsDir);
     }
-
     await startServers(log, options);
   },
-    {
+};
-      flags: SERVER_FLAG_OPTIONS,
-    }
-  );
-}

packages/kbn-scout/src/config/utils.ts

@@ -12,7 +12,7 @@ import getopts from 'getopts';
 import path from 'path';
 import { ToolingLog } from '@kbn/tooling-log';
 import { ServerlessProjectType } from '@kbn/es';
-import { REPO_ROOT } from '@kbn/repo-info';
+import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
 import { CliSupportedServerModes, ScoutServerConfig } from '../types';
 import { getConfigFilePath } from './get_config_file';
 import { loadConfig } from './loader/config_load';
@@ -30,15 +30,14 @@ export const formatCurrentDate = () => {
 };
 
 const saveTestServersConfigOnDisk = (testServersConfig: ScoutServerConfig, log: ToolingLog) => {
-  const configDirPath = path.resolve(REPO_ROOT, '.scout', 'servers');
+  const configFilePath = path.join(SCOUT_SERVERS_ROOT, `local.json`);
-  const configFilePath = path.join(configDirPath, `local.json`);
 
   try {
     const jsonData = JSON.stringify(testServersConfig, null, 2);
 
-    if (!Fs.existsSync(configDirPath)) {
+    if (!Fs.existsSync(SCOUT_SERVERS_ROOT)) {
-      log.debug(`scout: creating configuration directory: ${configDirPath}`);
+      log.debug(`scout: creating configuration directory: ${SCOUT_SERVERS_ROOT}`);
-      Fs.mkdirSync(configDirPath, { recursive: true });
+      Fs.mkdirSync(SCOUT_SERVERS_ROOT, { recursive: true });
     }
 
     Fs.writeFileSync(configFilePath, jsonData, 'utf-8');

packages/kbn-scout/src/playwright/config/index.ts

@@ -8,8 +8,8 @@
  */
 
 import { defineConfig, PlaywrightTestConfig, devices } from '@playwright/test';
-import * as Path from 'path';
+import { scoutPlaywrightReporter } from '@kbn/scout-reporting';
-import { REPO_ROOT } from '@kbn/repo-info';
+import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
 import { ScoutPlaywrightOptions, ScoutTestOptions, VALID_CONFIG_MARKER } from '../types';
 
 export function createPlaywrightConfig(options: ScoutPlaywrightOptions): PlaywrightTestConfig {
@@ -27,10 +27,11 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
     reporter: [
       ['html', { outputFolder: './output/reports', open: 'never' }], // HTML report configuration
       ['json', { outputFile: './output/reports/test-results.json' }], // JSON report
+      scoutPlaywrightReporter({ name: 'scout-playwright' }), // Scout report
     ],
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
-      serversConfigDir: Path.resolve(REPO_ROOT, '.scout', 'servers'),
+      serversConfigDir: SCOUT_SERVERS_ROOT,
       [VALID_CONFIG_MARKER]: true,
       /* Base URL to use in actions like `await page.goto('/')`. */
       // baseURL: 'http://127.0.0.1:3000',

packages/kbn-scout/src/types/config.d.ts

@@ -29,6 +29,6 @@ export interface ScoutLoaderConfig {
     buildArgs?: string[];
     sourceArgs?: string[];
     serverArgs: string[];
-    useDedicatedTastRunner?: boolean;
+    useDedicatedTestRunner?: boolean;
   };
 }

packages/kbn-scout/tsconfig.json

@@ -27,5 +27,7 @@
     "@kbn/mock-idp-utils",
     "@kbn/test-suites-xpack",
     "@kbn/test-subj-selector",
+    "@kbn/scout-info",
+    "@kbn/scout-reporting"
   ]
 }

packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts

@@ -25,6 +25,7 @@ export interface Suite extends Runnable {
   suites: Suite[];
   tests: Test[];
   title: string;
+  fullTitle(): string;
   file: string;
   parent?: Suite;
   eachTest: (cb: (test: Test) => void) => void;
@@ -39,6 +40,7 @@ export interface Test extends Runnable {
   parent?: Suite;
   isPassed: () => boolean;
   pending?: boolean;
+  err?: Error;
 }
 
 export interface Runnable {
@@ -51,10 +53,22 @@ export interface Runnable {
   parent?: Suite;
 }
 
+interface Stats {
+  suites: number;
+  tests: number;
+  passes: number;
+  pending: number;
+  failures: number;
+  start?: Date;
+  end?: Date;
+  duration?: number;
+}
+
 export interface Runner extends EventEmitter {
   abort(): void;
   failures: any[];
   uncaught: (error: Error) => void;
+  stats?: Stats;
 }
 
 export interface Mocha {

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -178,6 +178,12 @@ export const schema = Joi.object()
       })
       .default(),
 
+    scoutReporter: Joi.object()
+      .keys({
+        enabled: Joi.boolean().default(process.env.ENABLE_SCOUT_REPORTER || false),
+      })
+      .default(),
+
     users: Joi.object().pattern(
       ID_PATTERN,
       Joi.object()

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -20,6 +20,7 @@ import * as symbols from './symbols';
 import { ms } from './ms';
 import { writeEpilogue } from './write_epilogue';
 import { setupCiStatsFtrTestGroupReporter } from './ci_stats_ftr_reporter';
+import { ScoutFTRReporter } from './scout_ftr_reporter';
 
 export function MochaReporterProvider({ getService }) {
   const log = getService('log');
@@ -65,6 +66,10 @@ export function MochaReporterProvider({ getService }) {
           });
         }
       }
+
+      if (config.get('scoutReporter.enabled')) {
+        new ScoutFTRReporter(runner);
+      }
     }
 
     onStart = () => {

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts

@@ -0,0 +1,203 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import path from 'node:path';
+import { ToolingLog } from '@kbn/tooling-log';
+import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info';
+import { REPO_ROOT } from '@kbn/repo-info';
+import {
+  generateTestRunId,
+  getTestIDForTitle,
+  ScoutReport,
+  ScoutReportEventAction,
+  datasources,
+} from '@kbn/scout-reporting';
+import { getCodeOwnersForFile, getPathsWithOwnersReversed, PathWithOwners } from '@kbn/code-owners';
+import { Runner, Test } from '../../../fake_mocha_types';
+
+/**
+ * Configuration options for the Scout Mocha reporter
+ */
+export interface ScoutFTRReporterOptions {
+  name?: string;
+  outputPath?: string;
+}
+
+/**
+ * Scout Mocha reporter
+ */
+export class ScoutFTRReporter {
+  readonly log: ToolingLog;
+  readonly name: string;
+  readonly runId: string;
+  private report: ScoutReport;
+  private readonly pathsWithOwners: PathWithOwners[];
+
+  constructor(private runner: Runner, private reporterOptions: ScoutFTRReporterOptions = {}) {
+    this.log = new ToolingLog({
+      level: 'info',
+      writeTo: process.stdout,
+    });
+
+    this.name = this.reporterOptions.name || 'ftr';
+    this.runId = generateTestRunId();
+    this.log.info(`Scout test run ID: ${this.runId}`);
+
+    this.report = new ScoutReport(this.log);
+    this.pathsWithOwners = getPathsWithOwnersReversed();
+
+    // Register event listeners
+    for (const [eventName, listener] of Object.entries({
+      start: this.onRunStart,
+      end: this.onRunEnd,
+      test: this.onTestStart,
+      'test end': this.onTestEnd,
+    })) {
+      runner.on(eventName, listener);
+    }
+  }
+
+  private getFileOwners(filePath: string): string[] {
+    const concatenatedOwners = getCodeOwnersForFile(filePath, this.pathsWithOwners);
+
+    if (concatenatedOwners === undefined) {
+      return [];
+    }
+
+    return concatenatedOwners
+      .replace(/#.+$/, '')
+      .split(',')
+      .filter((value) => value.length > 0);
+  }
+
+  /**
+   * Root path of this reporter's output
+   */
+  public get reportRootPath(): string {
+    const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT;
+    return path.join(outputPath, `scout-ftr-${this.runId}`);
+  }
+
+  onRunStart = () => {
+    /**
+     * Root suite execution began (all files have been parsed and hooks/tests are ready for execution)
+     */
+    this.report.logEvent({
+      ...datasources.environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'ftr',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      event: {
+        action: ScoutReportEventAction.RUN_BEGIN,
+      },
+    });
+  };
+
+  onTestStart = (test: Test) => {
+    /**
+     * Test execution started
+     */
+    this.report.logEvent({
+      ...datasources.environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'ftr',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent?.fullTitle() || 'unknown',
+        type: test.parent?.root ? 'root' : 'suite',
+      },
+      test: {
+        id: getTestIDForTitle(test.fullTitle()),
+        title: test.title,
+        tags: [],
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_BEGIN,
+      },
+      file: {
+        path: test.file ? path.relative(REPO_ROOT, test.file) : 'unknown',
+        owner: test.file ? this.getFileOwners(path.relative(REPO_ROOT, test.file)) : 'unknown',
+      },
+    });
+  };
+
+  onTestEnd = (test: Test) => {
+    /**
+     * Test execution ended
+     */
+    this.report.logEvent({
+      ...datasources.environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'ftr',
+      },
+      test_run: {
+        id: this.runId,
+      },
+      suite: {
+        title: test.parent?.fullTitle() || 'unknown',
+        type: test.parent?.root ? 'root' : 'suite',
+      },
+      test: {
+        id: getTestIDForTitle(test.fullTitle()),
+        title: test.title,
+        tags: [],
+        status: test.isPending() ? 'skipped' : test.isPassed() ? 'passed' : 'failed',
+        duration: test.duration,
+      },
+      event: {
+        action: ScoutReportEventAction.TEST_END,
+        error: {
+          message: test.err?.message,
+          stack_trace: test.err?.stack,
+        },
+      },
+      file: {
+        path: test.file ? path.relative(REPO_ROOT, test.file) : 'unknown',
+        owner: test.file ? this.getFileOwners(path.relative(REPO_ROOT, test.file)) : 'unknown',
+      },
+    });
+  };
+
+  onRunEnd = () => {
+    /**
+     * Root suite execution has ended
+     */
+    this.report.logEvent({
+      ...datasources.environmentMetadata,
+      reporter: {
+        name: this.name,
+        type: 'ftr',
+      },
+      test_run: {
+        id: this.runId,
+        status: this.runner.stats?.failures === 0 ? 'passed' : 'failed',
+        duration: this.runner.stats?.duration || 0,
+      },
+      event: {
+        action: ScoutReportEventAction.RUN_END,
+      },
+    });
+
+    // Save & conclude the report
+    try {
+      this.report.save(this.reportRootPath);
+    } finally {
+      this.report.conclude();
+    }
+  };
+}

packages/kbn-test/tsconfig.json

@@ -37,6 +37,8 @@
     "@kbn/core-saved-objects-api-server",
     "@kbn/mock-idp-utils",
     "@kbn/code-owners",
+    "@kbn/scout-reporting",
+    "@kbn/scout-info",
     "@kbn/react-mute-legacy-root-warning",
   ]
 }

scripts/scout.js

@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the "Elastic License
@@ -8,4 +10,4 @@
  */
 
 require('../src/setup_node_env');
-require('@kbn/scout').startServersCli();
+void require('@kbn/scout').cli.run();

tsconfig.base.json

@@ -1548,6 +1548,10 @@
       "@kbn/saved-search-plugin/*": ["src/plugins/saved_search/*"],
       "@kbn/scout": ["packages/kbn-scout"],
       "@kbn/scout/*": ["packages/kbn-scout/*"],
+      "@kbn/scout-info": ["packages/kbn-scout-info"],
+      "@kbn/scout-info/*": ["packages/kbn-scout-info/*"],
+      "@kbn/scout-reporting": ["packages/kbn-scout-reporting"],
+      "@kbn/scout-reporting/*": ["packages/kbn-scout-reporting/*"],
       "@kbn/screenshot-mode-example-plugin": ["examples/screenshot_mode_example"],
       "@kbn/screenshot-mode-example-plugin/*": ["examples/screenshot_mode_example/*"],
       "@kbn/screenshot-mode-plugin": ["src/plugins/screenshot_mode"],

yarn.lock

@@ -6908,6 +6908,14 @@
   version "0.0.0"
   uid ""
 
+"@kbn/scout-info@link:packages/kbn-scout-info":
+  version "0.0.0"
+  uid ""
+
+"@kbn/scout-reporting@link:packages/kbn-scout-reporting":
+  version "0.0.0"
+  uid ""
+
 "@kbn/scout@link:packages/kbn-scout":
   version "0.0.0"
   uid ""