[ci-stats] send test results to ci-stats service (#123740) (#124634)

* [ci-stats] send test results to ci-stats service

* move export to export type

(cherry picked from commit cc0380a461)

Co-authored-by: Spencer <email@spalger.com>

package.json

@@ -455,6 +455,7 @@
     "@emotion/babel-preset-css-prop": "^11.2.0",
     "@emotion/jest": "^11.3.0",
     "@istanbuljs/schema": "^0.1.2",
+    "@jest/console": "^26.6.2",
     "@jest/reporters": "^26.6.2",
     "@kbn/babel-code-parser": "link:bazel-bin/packages/kbn-babel-code-parser",
     "@kbn/babel-preset": "link:bazel-bin/packages/kbn-babel-preset",

packages/kbn-dev-utils/BUILD.bazel

@@ -45,6 +45,7 @@ NPM_MODULE_EXTRA_FILES = [
 
 RUNTIME_DEPS = [
   "//packages/kbn-utils",
+  "//packages/kbn-std",
   "@npm//@babel/core",
   "@npm//axios",
   "@npm//chalk",
@@ -67,6 +68,7 @@ RUNTIME_DEPS = [
 
 TYPES_DEPS = [
   "//packages/kbn-utils:npm_module_types",
+  "//packages/kbn-std:npm_module_types",
   "@npm//@babel/parser",
   "@npm//@babel/types",
   "@npm//@types/babel__core",

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -11,16 +11,28 @@ import Os from 'os';
 import Fs from 'fs';
 import Path from 'path';
 import crypto from 'crypto';
+
 import execa from 'execa';
-import Axios from 'axios';
+import Axios, { AxiosRequestConfig } from 'axios';
 // @ts-expect-error not "public", but necessary to prevent Jest shimming from breaking things
 import httpAdapter from 'axios/lib/adapters/http';
 
 import { ToolingLog } from '../tooling_log';
 import { parseConfig, Config } from './ci_stats_config';
+import type { CiStatsTestGroupInfo, CiStatsTestRun } from './ci_stats_test_group_types';
 
 const BASE_URL = 'https://ci-stats.kibana.dev';
 
+/** Container for metadata that can be attached to different ci-stats objects */
+export interface CiStatsMetadata {
+  /**
+   * Arbitrary key-value pairs which can be attached to CiStatsTiming and CiStatsMetric
+   * objects stored in the ci-stats service
+   */
+  [key: string]: string | string[] | number | boolean | undefined;
+}
+
+/** A ci-stats metric record */
 export interface CiStatsMetric {
   /** Top-level categorization for the metric, e.g. "page load bundle size" */
   group: string;
@@ -40,13 +52,7 @@ export interface CiStatsMetric {
   meta?: CiStatsMetadata;
 }
 
-export interface CiStatsMetadata {
-  /**
-   * Arbitrary key-value pairs which can be attached to CiStatsTiming and CiStatsMetric
-   * objects stored in the ci-stats service
-   */
-  [key: string]: string | string[] | number | boolean | undefined;
-}
+/** A ci-stats timing event */
 export interface CiStatsTiming {
   /** Top-level categorization for the timing, e.g. "scripts/foo", process type, etc. */
   group: string;
@@ -58,13 +64,7 @@ export interface CiStatsTiming {
   meta?: CiStatsMetadata;
 }
 
-interface ReqOptions {
-  auth: boolean;
-  path: string;
-  body: any;
-  bodyDesc: string;
-}
-
+/** Options for reporting timings to ci-stats */
 export interface TimingsOptions {
   /** list of timings to record */
   timings: CiStatsTiming[];
@@ -74,10 +74,41 @@ export interface TimingsOptions {
   kibanaUuid?: string | null;
 }
 
+/** Options for reporting metrics to ci-stats */
 export interface MetricsOptions {
   /** Default metadata to add to each metric */
   defaultMeta?: CiStatsMetadata;
 }
+
+/** Options for reporting tests to ci-stats */
+export interface CiStatsReportTestsOptions {
+  /**
+   * Information about the group of tests that were run
+   */
+  group: CiStatsTestGroupInfo;
+  /**
+   * Information about each test that ran, including failure information
+   */
+  testRuns: CiStatsTestRun[];
+}
+
+/* @internal */
+interface ReportTestsResponse {
+  buildId: string;
+  groupId: string;
+  testRunCount: number;
+}
+
+/* @internal */
+interface ReqOptions {
+  auth: boolean;
+  path: string;
+  body: any;
+  bodyDesc: string;
+  query?: AxiosRequestConfig['params'];
+}
+
+/** Object that helps report data to the ci-stats service */
 export class CiStatsReporter {
   /**
    * Create a CiStatsReporter by inspecting the ENV for the necessary config
@@ -86,7 +117,7 @@ export class CiStatsReporter {
     return new CiStatsReporter(parseConfig(log), log);
   }
 
-  constructor(private config: Config | undefined, private log: ToolingLog) {}
+  constructor(private readonly config: Config | undefined, private readonly log: ToolingLog) {}
 
   /**
    * Determine if CI_STATS is explicitly disabled by the environment. To determine
@@ -165,7 +196,7 @@ export class CiStatsReporter {
 
     this.log.debug('CIStatsReporter committerHash: %s', defaultMeta.committerHash);
 
-    return await this.req({
+    return !!(await this.req({
       auth: !!buildId,
       path: '/v1/timings',
       body: {
@@ -175,7 +206,7 @@ export class CiStatsReporter {
         timings,
       },
       bodyDesc: timings.length === 1 ? `${timings.length} timing` : `${timings.length} timings`,
-    });
+    }));
   }
 
   /**
@@ -188,12 +219,11 @@ export class CiStatsReporter {
     }
 
     const buildId = this.config?.buildId;
-
     if (!buildId) {
-      throw new Error(`CiStatsReporter can't be authorized without a buildId`);
+      throw new Error(`metrics can't be reported without a buildId`);
     }
 
-    return await this.req({
+    return !!(await this.req({
       auth: true,
       path: '/v1/metrics',
       body: {
@@ -204,6 +234,30 @@ export class CiStatsReporter {
       bodyDesc: `metrics: ${metrics
         .map(({ group, id, value }) => `[${group}/${id}=${value}]`)
         .join(' ')}`,
+    }));
+  }
+
+  /**
+   * Send test reports to ci-stats
+   */
+  async reportTests({ group, testRuns }: CiStatsReportTestsOptions) {
+    if (!this.config?.buildId || !this.config?.apiToken) {
+      throw new Error(
+        'unable to report tests unless buildId is configured and auth config available'
+      );
+    }
+
+    return await this.req<ReportTestsResponse>({
+      auth: true,
+      path: '/v1/test_group',
+      query: {
+        buildId: this.config?.buildId,
+      },
+      bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`,
+      body: [
+        JSON.stringify({ group }),
+        ...testRuns.map((testRun) => JSON.stringify({ testRun })),
+      ].join('\n'),
     });
   }
 
@@ -241,7 +295,7 @@ export class CiStatsReporter {
     }
   }
 
-  private async req({ auth, body, bodyDesc, path }: ReqOptions) {
+  private async req<T>({ auth, body, bodyDesc, path, query }: ReqOptions) {
     let attempt = 0;
     const maxAttempts = 5;
 
@@ -251,23 +305,24 @@ export class CiStatsReporter {
         Authorization: `token ${this.config.apiToken}`,
       };
     } else if (auth) {
-      throw new Error('this.req() shouldnt be called with auth=true if this.config is defined');
+      throw new Error('this.req() shouldnt be called with auth=true if this.config is not defined');
     }
 
     while (true) {
       attempt += 1;
 
       try {
-        await Axios.request({
+        const resp = await Axios.request<T>({
           method: 'POST',
           url: path,
           baseURL: BASE_URL,
           headers,
           data: body,
+          params: query,
           adapter: httpAdapter,
         });
 
-        return true;
+        return resp.data;
       } catch (error) {
         if (!error?.request) {
           // not an axios error, must be a usage error that we should notify user about

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_test_group_types.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { CiStatsMetadata } from './ci_stats_reporter';
+
+export type CiStatsTestResult = 'fail' | 'pass' | 'skip';
+export type CiStatsTestType =
+  | 'after all hook'
+  | 'after each hook'
+  | 'before all hook'
+  | 'before each hook'
+  | 'test';
+
+export interface CiStatsTestRun {
+  /**
+   * ISO-8601 formatted datetime representing when the tests started running
+   */
+  startTime: string;
+  /**
+   * Duration of the tests in milliseconds
+   */
+  durationMs: number;
+  /**
+   * A sequence number, this is used to order the tests in a specific test run
+   */
+  seq: number;
+  /**
+   * The type of this "test run", usually this is just "test" but when reporting issues in hooks it can be set to the type of hook
+   */
+  type: CiStatsTestType;
+  /**
+   * "fail", "pass" or "skip", the result of the tests
+   */
+  result: CiStatsTestResult;
+  /**
+   * The list of suite names containing this test, the first being the outermost suite
+   */
+  suites: string[];
+  /**
+   * The name of this specific test run
+   */
+  name: string;
+  /**
+   * Relative path from the root of the repo contianing this test
+   */
+  file: string;
+  /**
+   * Error message if the test failed
+   */
+  error?: string;
+  /**
+   * Debug output/stdout produced by the test
+   */
+  stdout?: string;
+  /**
+   * Screenshots captured during the test run
+   */
+  screenshots?: Array<{
+    name: string;
+    base64Png: string;
+  }>;
+}
+
+export interface CiStatsTestGroupInfo {
+  /**
+   * ISO-8601 formatted datetime representing when the group of tests started running
+   */
+  startTime: string;
+  /**
+   * The number of miliseconds that the tests ran for
+   */
+  durationMs: number;
+  /**
+   * The type of tests run in this group, any value is valid but test groups are groupped by type in the UI so use something consistent
+   */
+  type: string;
+  /**
+   * The name of this specific group (within the "type")
+   */
+  name: string;
+  /**
+   * Arbitrary metadata associated with this group. We currently look for a ciGroup metadata property for highlighting that when appropriate
+   */
+  meta: CiStatsMetadata;
+}

packages/kbn-dev-utils/src/ci_stats_reporter/index.ts

@@ -10,3 +10,4 @@ export * from './ci_stats_reporter';
 export type { Config } from './ci_stats_config';
 export * from './ship_ci_stats_cli';
 export { getTimeReporter } from './report_time';
+export * from './ci_stats_test_group_types';

packages/kbn-pm/dist/index.js

@@ -9049,7 +9049,9 @@ var _ci_stats_config = __webpack_require__(218);
  */
 // @ts-expect-error not "public", but necessary to prevent Jest shimming from breaking things
 const BASE_URL = 'https://ci-stats.kibana.dev';
+/** Container for metadata that can be attached to different ci-stats objects */
 
+/** Object that helps report data to the ci-stats service */
 class CiStatsReporter {
   /**
    * Create a CiStatsReporter by inspecting the ENV for the necessary config
@@ -9146,7 +9148,7 @@ class CiStatsReporter {
       totalMem: _os.default.totalmem()
     };
     this.log.debug('CIStatsReporter committerHash: %s', defaultMeta.committerHash);
-    return await this.req({
+    return !!(await this.req({
       auth: !!buildId,
       path: '/v1/timings',
       body: {
@@ -9156,7 +9158,7 @@ class CiStatsReporter {
         timings
       },
       bodyDesc: timings.length === 1 ? `${timings.length} timing` : `${timings.length} timings`
-    });
+    }));
   }
   /**
    * Report metrics data to the ci-stats service. If running outside of CI this method
@@ -9174,10 +9176,10 @@ class CiStatsReporter {
     const buildId = (_this$config4 = this.config) === null || _this$config4 === void 0 ? void 0 : _this$config4.buildId;
 
     if (!buildId) {
-      throw new Error(`CiStatsReporter can't be authorized without a buildId`);
+      throw new Error(`metrics can't be reported without a buildId`);
     }
 
-    return await this.req({
+    return !!(await this.req({
       auth: true,
       path: '/v1/metrics',
       body: {
@@ -9190,6 +9192,35 @@ class CiStatsReporter {
         id,
         value
       }) => `[${group}/${id}=${value}]`).join(' ')}`
+    }));
+  }
+  /**
+   * Send test reports to ci-stats
+   */
+
+
+  async reportTests({
+    group,
+    testRuns
+  }) {
+    var _this$config5, _this$config6, _this$config7;
+
+    if (!((_this$config5 = this.config) !== null && _this$config5 !== void 0 && _this$config5.buildId) || !((_this$config6 = this.config) !== null && _this$config6 !== void 0 && _this$config6.apiToken)) {
+      throw new Error('unable to report tests unless buildId is configured and auth config available');
+    }
+
+    return await this.req({
+      auth: true,
+      path: '/v1/test_group',
+      query: {
+        buildId: (_this$config7 = this.config) === null || _this$config7 === void 0 ? void 0 : _this$config7.buildId
+      },
+      bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`,
+      body: [JSON.stringify({
+        group
+      }), ...testRuns.map(testRun => JSON.stringify({
+        testRun
+      }))].join('\n')
     });
   }
   /**
@@ -9239,7 +9270,8 @@ class CiStatsReporter {
     auth,
     body,
     bodyDesc,
-    path
+    path,
+    query
   }) {
     let attempt = 0;
     const maxAttempts = 5;
@@ -9250,22 +9282,23 @@ class CiStatsReporter {
         Authorization: `token ${this.config.apiToken}`
       };
     } else if (auth) {
-      throw new Error('this.req() shouldnt be called with auth=true if this.config is defined');
+      throw new Error('this.req() shouldnt be called with auth=true if this.config is not defined');
     }
 
     while (true) {
       attempt += 1;
 
       try {
-        await _axios.default.request({
+        const resp = await _axios.default.request({
           method: 'POST',
           url: path,
           baseURL: BASE_URL,
           headers,
           data: body,
+          params: query,
           adapter: _http.default
         });
-        return true;
+        return resp.data;
       } catch (error) {
         var _error$response;
 

packages/kbn-test/BUILD.bazel

@@ -41,8 +41,10 @@ RUNTIME_DEPS = [
     "//packages/kbn-std",
     "//packages/kbn-utils",
     "@npm//@elastic/elasticsearch",
-    "@npm//axios",
     "@npm//@babel/traverse",
+    "@npm//@jest/console",
+    "@npm//@jest/reporters",
+    "@npm//axios",
     "@npm//chance",
     "@npm//dedent",
     "@npm//del",
@@ -58,7 +60,6 @@ RUNTIME_DEPS = [
     "@npm//jest-cli",
     "@npm//jest-snapshot",
     "@npm//jest-styled-components",
-    "@npm//@jest/reporters",
     "@npm//joi",
     "@npm//mustache",
     "@npm//normalize-path",
@@ -81,6 +82,8 @@ TYPES_DEPS = [
   "//packages/kbn-std:npm_module_types",
   "//packages/kbn-utils",
   "@npm//@elastic/elasticsearch",
+  "@npm//@jest/console",
+  "@npm//@jest/reporters",
   "@npm//axios",
   "@npm//elastic-apm-node",
   "@npm//del",

packages/kbn-test/jest-preset.js

@@ -55,6 +55,12 @@ module.exports = {
         rootDirectory: '.',
       },
     ],
+    [
+      '@kbn/test/target_node/jest/ci_stats_jest_reporter',
+      {
+        testGroupType: 'Jest Unit Tests',
+      },
+    ],
   ],
 
   // The paths to modules that run some code to configure or set up the testing environment before each test

packages/kbn-test/jest_integration/jest-preset.js

@@ -21,6 +21,12 @@ module.exports = {
   reporters: [
     'default',
     ['@kbn/test/target_node/jest/junit_reporter', { reportName: 'Jest Integration Tests' }],
+    [
+      '@kbn/test/target_node/jest/ci_stats_jest_reporter',
+      {
+        testGroupType: 'Jest Integration Tests',
+      },
+    ],
   ],
   coverageReporters: !!process.env.CI
     ? [['json', { file: 'jest-integration.json' }]]

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -13,7 +13,7 @@ import { Suite, Test } from './fake_mocha_types';
 import {
   Lifecycle,
   LifecyclePhase,
-  FailureMetadata,
+  TestMetadata,
   readConfigFile,
   ProviderCollection,
   readProviderSpec,
@@ -27,7 +27,7 @@ import {
 
 export class FunctionalTestRunner {
   public readonly lifecycle = new Lifecycle();
-  public readonly failureMetadata = new FailureMetadata(this.lifecycle);
+  public readonly testMetadata = new TestMetadata(this.lifecycle);
   private closed = false;
 
   private readonly esVersion: EsVersion;
@@ -181,7 +181,7 @@ export class FunctionalTestRunner {
       const coreProviders = readProviderSpec('Service', {
         lifecycle: () => this.lifecycle,
         log: () => this.log,
-        failureMetadata: () => this.failureMetadata,
+        testMetadata: () => this.testMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
         esVersion: () => this.esVersion,

packages/kbn-test/src/functional_test_runner/index.ts

@@ -7,7 +7,8 @@
  */
 
 export { FunctionalTestRunner } from './functional_test_runner';
-export { readConfigFile, Config, EsVersion } from './lib';
+export { readConfigFile, Config, EsVersion, Lifecycle, LifecyclePhase } from './lib';
+export type { ScreenshotRecord } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/integration_tests/__fixtures__/failure_hooks/config.js

@@ -35,6 +35,7 @@ export default function () {
     },
     mochaReporter: {
       captureLogOutput: false,
+      sendToCiStats: false,
     },
   };
 }

packages/kbn-test/src/functional_test_runner/integration_tests/__fixtures__/simple_project/config.js

@@ -10,4 +10,7 @@ import { resolve } from 'path';
 
 export default () => ({
   testFiles: [resolve(__dirname, 'tests.js')],
+  mochaReporter: {
+    sendToCiStats: false,
+  },
 });

packages/kbn-test/src/functional_test_runner/lib/config/config.ts

@@ -20,6 +20,7 @@ interface Options {
 }
 
 export class Config {
+  public readonly path: string;
   private [$values]: Record<string, any>;
 
   constructor(options: Options) {
@@ -29,6 +30,7 @@ export class Config {
       throw new TypeError('path is a required option');
     }
 
+    this.path = path;
     const { error, value } = schema.validate(settings, {
       abortEarly: false,
       context: {

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -152,6 +152,7 @@ export const schema = Joi.object()
     mochaReporter: Joi.object()
       .keys({
         captureLogOutput: Joi.boolean().default(!!process.env.CI),
+        sendToCiStats: Joi.boolean().default(!!process.env.CI),
       })
       .default(),
 

packages/kbn-test/src/functional_test_runner/lib/failure_metadata.test.ts

@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { Lifecycle } from './lifecycle';
-import { FailureMetadata } from './failure_metadata';
-import { Test } from '../fake_mocha_types';
-
-it('collects metadata for the current test', async () => {
-  const lifecycle = new Lifecycle();
-  const failureMetadata = new FailureMetadata(lifecycle);
-
-  const test1 = {} as Test;
-  await lifecycle.beforeEachRunnable.trigger(test1);
-  failureMetadata.add({ foo: 'bar' });
-
-  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
-    Object {
-      "foo": "bar",
-    }
-  `);
-
-  const test2 = {} as Test;
-  await lifecycle.beforeEachRunnable.trigger(test2);
-  failureMetadata.add({ test: 2 });
-
-  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
-    Object {
-      "foo": "bar",
-    }
-  `);
-  expect(failureMetadata.get(test2)).toMatchInlineSnapshot(`
-    Object {
-      "test": 2,
-    }
-  `);
-});
-
-it('adds messages to the messages state', () => {
-  const lifecycle = new Lifecycle();
-  const failureMetadata = new FailureMetadata(lifecycle);
-
-  const test1 = {} as Test;
-  lifecycle.beforeEachRunnable.trigger(test1);
-  failureMetadata.addMessages(['foo', 'bar']);
-  failureMetadata.addMessages(['baz']);
-
-  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
-    Object {
-      "messages": Array [
-        "foo",
-        "bar",
-        "baz",
-      ],
-    }
-  `);
-});

packages/kbn-test/src/functional_test_runner/lib/failure_metadata.ts

@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Path from 'path';
-
-import { REPO_ROOT } from '@kbn/utils';
-
-import { Lifecycle } from './lifecycle';
-
-interface Metadata {
-  [key: string]: unknown;
-}
-
-export class FailureMetadata {
-  // mocha's global types mean we can't import Mocha or it will override the global jest types..............
-  private currentRunnable?: any;
-  private readonly allMetadata = new Map<any, Metadata>();
-
-  constructor(lifecycle: Lifecycle) {
-    if (!process.env.GCS_UPLOAD_PREFIX && process.env.CI) {
-      throw new Error(
-        'GCS_UPLOAD_PREFIX environment variable is not set and must always be set on CI'
-      );
-    }
-
-    lifecycle.beforeEachRunnable.add((runnable) => {
-      this.currentRunnable = runnable;
-    });
-  }
-
-  add(metadata: Metadata | ((current: Metadata) => Metadata)) {
-    if (!this.currentRunnable) {
-      throw new Error('no current runnable to associate metadata with');
-    }
-
-    const current = this.allMetadata.get(this.currentRunnable);
-    this.allMetadata.set(this.currentRunnable, {
-      ...current,
-      ...(typeof metadata === 'function' ? metadata(current || {}) : metadata),
-    });
-  }
-
-  addMessages(messages: string[]) {
-    this.add((current) => ({
-      messages: [...(Array.isArray(current.messages) ? current.messages : []), ...messages],
-    }));
-  }
-
-  /**
-   * @param name Name to label the URL with
-   * @param repoPath absolute path, within the repo, that will be uploaded
-   */
-  addScreenshot(name: string, repoPath: string) {
-    const prefix = process.env.GCS_UPLOAD_PREFIX;
-
-    if (!prefix) {
-      return;
-    }
-
-    const slash = prefix.endsWith('/') ? '' : '/';
-    const urlPath = Path.relative(REPO_ROOT, repoPath)
-      .split(Path.sep)
-      .map((c) => encodeURIComponent(c))
-      .join('/');
-
-    if (urlPath.startsWith('..')) {
-      throw new Error(
-        `Only call addUploadLink() with paths that are within the repo root, received ${repoPath} and repo root is ${REPO_ROOT}`
-      );
-    }
-
-    const url = `https://storage.googleapis.com/${prefix}${slash}${urlPath}`;
-    const screenshot = {
-      name,
-      url,
-    };
-
-    this.add((current) => ({
-      screenshots: [...(Array.isArray(current.screenshots) ? current.screenshots : []), screenshot],
-    }));
-
-    return screenshot;
-  }
-
-  get(runnable: any) {
-    return this.allMetadata.get(runnable);
-  }
-}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -12,7 +12,7 @@ export { readConfigFile, Config } from './config';
 export { readProviderSpec, ProviderCollection } from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';
-export { FailureMetadata } from './failure_metadata';
+export * from './test_metadata';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -11,15 +11,23 @@ import { LifecyclePhase } from './lifecycle_phase';
 import { Suite, Test } from '../fake_mocha_types';
 
 export class Lifecycle {
+  /** lifecycle phase that will run handlers once before tests execute */
   public readonly beforeTests = new LifecyclePhase<[Suite]>({
     singular: true,
   });
+  /** lifecycle phase that runs handlers before each runnable (test and hooks) */
   public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
+  /** lifecycle phase that runs handlers before each suite */
   public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
+  /** lifecycle phase that runs handlers before each test */
   public readonly beforeEachTest = new LifecyclePhase<[Test]>();
+  /** lifecycle phase that runs handlers after each suite */
   public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
+  /** lifecycle phase that runs handlers after a test fails */
   public readonly testFailure = new LifecyclePhase<[Error, Test]>();
+  /** lifecycle phase that runs handlers after a hook fails */
   public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
+  /** lifecycle phase that runs handlers at the very end of execution */
   public readonly cleanup = new LifecyclePhase<[]>({
     singular: true,
   });

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/ci_stats_ftr_reporter.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Path from 'path';
+
+import { REPO_ROOT } from '@kbn/utils';
+import { CiStatsReporter, CiStatsReportTestsOptions, CiStatsTestType } from '@kbn/dev-utils';
+
+import { Config } from '../../config';
+import { Runner } from '../../../fake_mocha_types';
+import { TestMetadata, ScreenshotRecord } from '../../test_metadata';
+import { Lifecycle } from '../../lifecycle';
+import { getSnapshotOfRunnableLogs } from '../../../../mocha';
+
+interface Suite {
+  _beforeAll: Runnable[];
+  _beforeEach: Runnable[];
+  _afterEach: Runnable[];
+  _afterAll: Runnable[];
+}
+
+interface Runnable {
+  isFailed(): boolean;
+  isPending(): boolean;
+  duration?: number;
+  titlePath(): string[];
+  file: string;
+  title: string;
+  parent: Suite;
+  _screenshots?: ScreenshotRecord[];
+}
+
+function getHookType(hook: Runnable): CiStatsTestType {
+  if (hook.parent._afterAll.includes(hook)) {
+    return 'after all hook';
+  }
+  if (hook.parent._afterEach.includes(hook)) {
+    return 'after each hook';
+  }
+  if (hook.parent._beforeEach.includes(hook)) {
+    return 'before each hook';
+  }
+  if (hook.parent._beforeAll.includes(hook)) {
+    return 'before all hook';
+  }
+
+  throw new Error(`unable to determine hook type, hook is not owned by it's parent`);
+}
+
+export function setupCiStatsFtrTestGroupReporter({
+  config,
+  lifecycle,
+  runner,
+  testMetadata,
+  reporter,
+}: {
+  config: Config;
+  lifecycle: Lifecycle;
+  runner: Runner;
+  testMetadata: TestMetadata;
+  reporter: CiStatsReporter;
+}) {
+  let startMs: number | undefined;
+  runner.on('start', () => {
+    startMs = Date.now();
+  });
+
+  const start = Date.now();
+  const group: CiStatsReportTestsOptions['group'] = {
+    startTime: new Date(start).toJSON(),
+    durationMs: 0,
+    type: config.path.startsWith('x-pack') ? 'X-Pack Functional Tests' : 'Functional Tests',
+    name: Path.relative(REPO_ROOT, config.path),
+    meta: {
+      ciGroup: config.get('suiteTags.include').find((t: string) => t.startsWith('ciGroup')),
+      tags: [
+        ...config.get('suiteTags.include'),
+        ...config.get('suiteTags.exclude').map((t: string) => `-${t}`),
+      ].filter((t) => !t.startsWith('ciGroup')),
+    },
+  };
+
+  const testRuns: CiStatsReportTestsOptions['testRuns'] = [];
+  function trackRunnable(
+    runnable: Runnable,
+    { error, type }: { error?: Error; type: CiStatsTestType }
+  ) {
+    testRuns.push({
+      startTime: new Date(Date.now() - (runnable.duration ?? 0)).toJSON(),
+      durationMs: runnable.duration ?? 0,
+      seq: testRuns.length + 1,
+      file: Path.relative(REPO_ROOT, runnable.file),
+      name: runnable.title,
+      suites: runnable.titlePath().slice(0, -1),
+      result: runnable.isFailed() ? 'fail' : runnable.isPending() ? 'skip' : 'pass',
+      type,
+      error: error?.stack,
+      stdout: getSnapshotOfRunnableLogs(runnable),
+      screenshots: testMetadata.getScreenshots(runnable).map((s) => ({
+        base64Png: s.base64Png,
+        name: s.name,
+      })),
+    });
+  }
+
+  const errors = new Map<Runnable, Error>();
+  runner.on('fail', (test: Runnable, error: Error) => {
+    errors.set(test, error);
+  });
+
+  runner.on('hook end', (hook: Runnable) => {
+    if (hook.isFailed()) {
+      const error = errors.get(hook);
+      if (!error) {
+        throw new Error(`no error recorded for failed hook`);
+      }
+
+      trackRunnable(hook, {
+        type: getHookType(hook),
+        error,
+      });
+    }
+  });
+
+  runner.on('test end', (test: Runnable) => {
+    const error = errors.get(test);
+    if (test.isFailed() && !error) {
+      throw new Error('no error recorded for failed test');
+    }
+
+    trackRunnable(test, {
+      type: 'test',
+      error,
+    });
+  });
+
+  runner.on('end', () => {
+    if (!startMs) {
+      throw new Error('startMs was not defined');
+    }
+
+    // update the durationMs
+    group.durationMs = Date.now() - startMs;
+  });
+
+  lifecycle.cleanup.add(async () => {
+    await reporter.reportTests({
+      group,
+      testRuns,
+    });
+  });
+}

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -9,7 +9,7 @@
 import { format } from 'util';
 
 import Mocha from 'mocha';
-import { ToolingLogTextWriter } from '@kbn/dev-utils';
+import { ToolingLogTextWriter, CiStatsReporter } from '@kbn/dev-utils';
 import moment from 'moment';
 
 import { recordLog, snapshotLogsForRunnable, setupJUnitReportGeneration } from '../../../../mocha';
@@ -17,11 +17,13 @@ import * as colors from './colors';
 import * as symbols from './symbols';
 import { ms } from './ms';
 import { writeEpilogue } from './write_epilogue';
+import { setupCiStatsFtrTestGroupReporter } from './ci_stats_ftr_reporter';
 
 export function MochaReporterProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
-  const failureMetadata = getService('failureMetadata');
+  const lifecycle = getService('lifecycle');
+  const testMetadata = getService('testMetadata');
   let originalLogWriters;
   let reporterCaptureStartTime;
 
@@ -45,9 +47,23 @@ export function MochaReporterProvider({ getService }) {
       if (config.get('junit.enabled') && config.get('junit.reportName')) {
         setupJUnitReportGeneration(runner, {
           reportName: config.get('junit.reportName'),
-          getTestMetadata: (t) => failureMetadata.get(t),
         });
       }
+
+      if (config.get('mochaReporter.sendToCiStats')) {
+        const reporter = CiStatsReporter.fromEnv(log);
+        if (!reporter.hasBuildConfig()) {
+          log.warning('ci-stats reporter config is not available so test results will not be sent');
+        } else {
+          setupCiStatsFtrTestGroupReporter({
+            reporter,
+            config,
+            lifecycle,
+            runner,
+            testMetadata,
+          });
+        }
+      }
     }
 
     onStart = () => {

packages/kbn-test/src/functional_test_runner/lib/test_metadata.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Lifecycle } from './lifecycle';
+
+export interface ScreenshotRecord {
+  name: string;
+  base64Png: string;
+  baselinePath?: string;
+  failurePath?: string;
+}
+
+export class TestMetadata {
+  // mocha's global types mean we can't import Mocha or it will override the global jest types..............
+  private currentRunnable?: any;
+
+  constructor(lifecycle: Lifecycle) {
+    lifecycle.beforeEachRunnable.add((runnable) => {
+      this.currentRunnable = runnable;
+    });
+  }
+
+  addScreenshot(screenshot: ScreenshotRecord) {
+    this.currentRunnable._screenshots = (this.currentRunnable._screenshots || []).concat(
+      screenshot
+    );
+  }
+
+  getScreenshots(test: any): ScreenshotRecord[] {
+    if (!test || typeof test !== 'object' || !test._screenshots) {
+      return [];
+    }
+
+    return test._screenshots.slice();
+  }
+}

packages/kbn-test/src/functional_test_runner/public_types.ts

@@ -8,10 +8,10 @@
 
 import type { ToolingLog } from '@kbn/dev-utils';
 
-import type { Config, Lifecycle, FailureMetadata, DockerServersService, EsVersion } from './lib';
+import type { Config, Lifecycle, TestMetadata, DockerServersService, EsVersion } from './lib';
 import type { Test, Suite } from './fake_mocha_types';
 
-export { Lifecycle, Config, FailureMetadata };
+export { Lifecycle, Config, TestMetadata };
 
 export interface AsyncInstance<T> {
   /**
@@ -57,7 +57,7 @@ export interface GenericFtrProviderContext<
    * @param serviceName
    */
   hasService(
-    serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata' | 'dockerServers' | 'esVersion'
+    serviceName: 'config' | 'log' | 'lifecycle' | 'testMetadata' | 'dockerServers' | 'esVersion'
   ): true;
   hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
   hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
@@ -71,7 +71,7 @@ export interface GenericFtrProviderContext<
   getService(serviceName: 'log'): ToolingLog;
   getService(serviceName: 'lifecycle'): Lifecycle;
   getService(serviceName: 'dockerServers'): DockerServersService;
-  getService(serviceName: 'failureMetadata'): FailureMetadata;
+  getService(serviceName: 'testMetadata'): TestMetadata;
   getService(serviceName: 'esVersion'): EsVersion;
   getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
 

packages/kbn-test/src/jest/ci_stats_jest_reporter.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Path from 'path';
+
+import getopts from 'getopts';
+import { CiStatsReporter, ToolingLog, CiStatsReportTestsOptions } from '@kbn/dev-utils';
+import type { Config } from '@jest/types';
+import { BaseReporter, Test, TestResult } from '@jest/reporters';
+import { ConsoleBuffer } from '@jest/console';
+
+type LogEntry = ConsoleBuffer[0];
+
+interface ReporterOptions {
+  testGroupType: string;
+}
+
+function formatConsoleLine({ type, message, origin }: LogEntry) {
+  const originLines = origin.split('\n');
+
+  return `console.${type}: ${message}${originLines[0] ? `\n  ${originLines[0]}` : ''}`;
+}
+
+/**
+ * Jest reporter that reports tests to CI Stats
+ * @class JestJUnitReporter
+ */
+
+// eslint-disable-next-line import/no-default-export
+export default class CiStatsJestReporter extends BaseReporter {
+  private reporter: CiStatsReporter | undefined;
+  private readonly testGroupType: string;
+  private readonly reportName: string;
+  private readonly rootDir: string;
+  private startTime: number | undefined;
+
+  private group: CiStatsReportTestsOptions['group'] | undefined;
+  private readonly testRuns: CiStatsReportTestsOptions['testRuns'] = [];
+
+  constructor(config: Config.GlobalConfig, options: ReporterOptions) {
+    super();
+
+    this.rootDir = config.rootDir;
+    this.testGroupType = options?.testGroupType;
+    if (!this.testGroupType) {
+      throw new Error('missing testGroupType reporter option');
+    }
+
+    const configArg = getopts(process.argv).config;
+    if (typeof configArg !== 'string') {
+      throw new Error('expected to find a single --config arg');
+    }
+    this.reportName = configArg;
+  }
+
+  async onRunStart() {
+    const reporter = CiStatsReporter.fromEnv(
+      new ToolingLog({
+        level: 'info',
+        writeTo: process.stdout,
+      })
+    );
+
+    if (!reporter.hasBuildConfig()) {
+      return;
+    }
+
+    this.startTime = Date.now();
+    this.reporter = reporter;
+    this.group = {
+      name: this.reportName,
+      type: this.testGroupType,
+      startTime: new Date(this.startTime).toJSON(),
+      meta: {},
+      durationMs: 0,
+    };
+  }
+
+  async onTestFileResult(_: Test, testResult: TestResult) {
+    if (!this.reporter || !this.group) {
+      return;
+    }
+
+    let elapsedTime = 0;
+    for (const t of testResult.testResults) {
+      const startTime = new Date(testResult.perfStats.start + elapsedTime).toJSON();
+      elapsedTime += t.duration ?? 0;
+      this.testRuns.push({
+        startTime,
+        durationMs: t.duration ?? 0,
+        seq: this.testRuns.length + 1,
+        file: Path.relative(this.rootDir, testResult.testFilePath),
+        name: t.title,
+        result: t.status === 'failed' ? 'fail' : t.status === 'passed' ? 'pass' : 'skip',
+        suites: t.ancestorTitles,
+        type: 'test',
+        error: t.failureMessages.join('\n\n'),
+        stdout: testResult.console?.map(formatConsoleLine).join('\n'),
+      });
+    }
+  }
+
+  async onRunComplete() {
+    if (!this.reporter || !this.group || !this.testRuns.length || !this.startTime) {
+      return;
+    }
+
+    this.group.durationMs = Date.now() - this.startTime;
+
+    await this.reporter.reportTests({
+      group: this.group,
+      testRuns: this.testRuns,
+    });
+  }
+}

packages/kbn-test/src/mocha/index.ts

@@ -11,7 +11,7 @@
 export { setupJUnitReportGeneration } from './junit_report_generation';
 // @ts-ignore not typed yet
 // @internal
-export { recordLog, snapshotLogsForRunnable } from './log_cache';
+export { recordLog, snapshotLogsForRunnable, getSnapshotOfRunnableLogs } from './log_cache';
 // @ts-ignore not typed yet
 // @internal
 export { escapeCdata } from './xml';

test/functional/services/common/screenshots.ts

@@ -22,7 +22,7 @@ const writeFileAsync = promisify(writeFile);
 export class ScreenshotsService extends FtrService {
   private readonly log = this.ctx.getService('log');
   private readonly config = this.ctx.getService('config');
-  private readonly failureMetadata = this.ctx.getService('failureMetadata');
+  private readonly testMetadata = this.ctx.getService('testMetadata');
   private readonly browser = this.ctx.getService('browser');
 
   private readonly SESSION_DIRECTORY = resolve(this.config.get('screenshots.directory'), 'session');
@@ -51,11 +51,17 @@ export class ScreenshotsService extends FtrService {
   async compareAgainstBaseline(name: string, updateBaselines: boolean, el?: WebElementWrapper) {
     this.log.debug('compareAgainstBaseline');
     const sessionPath = resolve(this.SESSION_DIRECTORY, `${name}.png`);
-    await this.capture(sessionPath, el);
-
     const baselinePath = resolve(this.BASELINE_DIRECTORY, `${name}.png`);
     const failurePath = resolve(this.FAILURE_DIRECTORY, `${name}.png`);
 
+    await this.capture({
+      path: sessionPath,
+      name,
+      el,
+      baselinePath,
+      failurePath,
+    });
+
     if (updateBaselines) {
       this.log.debug('Updating baseline snapshot');
       // Make the directory if it doesn't exist
@@ -76,22 +82,42 @@ export class ScreenshotsService extends FtrService {
 
   async take(name: string, el?: WebElementWrapper, subDirectories: string[] = []) {
     const path = resolve(this.SESSION_DIRECTORY, ...subDirectories, `${name}.png`);
-    await this.capture(path, el);
-    this.failureMetadata.addScreenshot(name, path);
+    await this.capture({ path, name, el });
   }
 
   async takeForFailure(name: string, el?: WebElementWrapper) {
     const path = resolve(this.FAILURE_DIRECTORY, `${name}.png`);
-    await this.capture(path, el);
-    this.failureMetadata.addScreenshot(`failure[${name}]`, path);
+    await this.capture({
+      path,
+      name: `failure[${name}]`,
+      el,
+    });
   }
 
-  private async capture(path: string, el?: WebElementWrapper) {
+  private async capture({
+    path,
+    el,
+    name,
+    baselinePath,
+    failurePath,
+  }: {
+    path: string;
+    name: string;
+    el?: WebElementWrapper;
+    baselinePath?: string;
+    failurePath?: string;
+  }) {
     try {
       this.log.info(`Taking screenshot "${path}"`);
       const screenshot = await (el ? el.takeScreenshot() : this.browser.takeScreenshot());
       await mkdirAsync(dirname(path), { recursive: true });
       await writeFileAsync(path, screenshot, 'base64');
+      this.testMetadata.addScreenshot({
+        name,
+        base64Png: Buffer.isBuffer(screenshot) ? screenshot.toString('base64') : screenshot,
+        baselinePath,
+        failurePath,
+      });
     } catch (err) {
       this.log.error('SCREENSHOT FAILED');
       this.log.error(err);