Add failure screenshot links to JUnit failures (#52449)

packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts

@@ -73,12 +73,17 @@ it('rewrites ftr reports with minimal changes', async () => {
     ===================================================================
     --- ftr.xml	[object Object]
     +++ ftr.xml
-    @@ -2,52 +2,56 @@
+    @@ -1,53 +1,56 @@
+     ‹?xml version="1.0" encoding="utf-8"?›
      ‹testsuites›
        ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
          ‹testcase name="maps app  maps loaded from sample data ecommerce "before all" hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
-           ‹system-out›
+    -      ‹system-out›
     -        ‹![CDATA[[00:00:00]       │
+    +      ‹system-out›Failed Tests Reporter:
+    +  - foo bar
+    +
+    +
     +        [00:00:00]       │
      [00:07:04]         └-: maps app
      ...
@@ -94,13 +99,8 @@ it('rewrites ftr reports with minimal changes', async () => {
          at process._tickCallback (internal/process/next_tick.js:68:7)
          at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
     -    at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)]]›
-    -      ‹/failure›
     +    at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
-    +
-    +
-    +Failed Tests Reporter:
-    +  - foo bar
-    +‹/failure›
+           ‹/failure›
          ‹/testcase›
          ‹testcase name="maps app "after all" hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps" time="0.179"›
            ‹system-out›
@@ -181,11 +181,11 @@ it('rewrites jest reports with minimal changes', async () => {
     +      ‹failure›‹![CDATA[
     +        TypeError: Cannot read property '0' of undefined
     +    at Object.‹anonymous›.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
-    +
-    +
-    +Failed Tests Reporter:
+    +      ]]›‹/failure›
+    +      ‹system-out›Failed Tests Reporter:
     +  - foo bar
-    +]]›‹/failure›
+    +
+    +‹/system-out›
          ‹/testcase›
          ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher can start and end a process" time="0.435"/›
          ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher should restart a process if a process died before connected" time="1.502"/›
@@ -216,12 +216,17 @@ it('rewrites mocha reports with minimal changes', async () => {
     ===================================================================
     --- mocha.xml	[object Object]
     +++ mocha.xml
-    @@ -2,12 +2,12 @@
+    @@ -1,13 +1,16 @@
+     ‹?xml version="1.0" encoding="utf-8"?›
      ‹testsuites›
        ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
          ‹testcase name="code in multiple nodes "before all" hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
-           ‹system-out›
+    -      ‹system-out›
     -        ‹![CDATA[]]›
+    +      ‹system-out›Failed Tests Reporter:
+    +  - foo bar
+    +
+    +
     +
            ‹/system-out›
     -      ‹failure›
@@ -232,7 +237,7 @@ it('rewrites mocha reports with minimal changes', async () => {
      ‹head›‹title›503 Service Temporarily Unavailable‹/title›‹/head›
      ‹body bgcolor="white"›
      ‹center›‹h1›503 Service Temporarily Unavailable‹/h1›‹/center›
-    @@ -15,24 +15,28 @@
+    @@ -15,24 +18,24 @@
      ‹/body›
      ‹/html›
 
@@ -240,11 +245,7 @@ it('rewrites mocha reports with minimal changes', async () => {
     -    at process._tickCallback (internal/process/next_tick.js:68:7)]]›
     -      ‹/failure›
     +    at process._tickCallback (internal/process/next_tick.js:68:7)
-    +
-    +
-    +Failed Tests Reporter:
-    +  - foo bar
-    +]]›‹/failure›
+    +      ]]›‹/failure›
          ‹/testcase›
          ‹testcase name="code in multiple nodes "after all" hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.003"›
            ‹system-out›
@@ -324,11 +325,11 @@ it('rewrites karma reports with minimal changes', async () => {
     +    at Generator.prototype.‹computed› [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
     +    at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
     +    at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
-    +
-    +
-    +Failed Tests Reporter:
-    +  - foo bar
     +]]›‹/failure›
+    +    ‹system-out›Failed Tests Reporter:
+    +  - foo bar
+    +
+    +‹/system-out›
        ‹/testcase›
        ‹testcase name="CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should toggle to Heatmap OK" time="0.055" classname="Browser Unit Tests.CoordinateMapsVisualizationTest"/›
        ‹testcase name="VegaParser._parseSchema should warn on vega-lite version too new to be supported" time="0.001" classname="Browser Unit Tests.VegaParser·_parseSchema"/›

packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts

@@ -57,16 +57,14 @@ export async function addMessagesToReport(options: {
     }
 
     log.info(`${classname} - ${name}:${messageList}`);
-    const append = `\n\nFailed Tests Reporter:${messageList}\n`;
+    const output = `Failed Tests Reporter:${messageList}\n\n`;
 
-    if (
-      testCase.failure[0] &&
-      typeof testCase.failure[0] === 'object' &&
-      typeof testCase.failure[0]._ === 'string'
-    ) {
-      testCase.failure[0]._ += append;
+    if (!testCase['system-out']) {
+      testCase['system-out'] = [output];
+    } else if (typeof testCase['system-out'][0] === 'string') {
+      testCase['system-out'][0] = output + String(testCase['system-out'][0]);
     } else {
-      testCase.failure[0] = String(testCase.failure[0]) + append;
+      testCase['system-out'][0]._ = output + testCase['system-out'][0]._;
     }
   }
 

packages/kbn-test/src/failed_tests_reporter/report_metadata.ts

@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { TestReport, makeTestCaseIter } from './test_report';
+import { Message } from './add_messages_to_report';
+
+export function* getMetadataIter(report: TestReport) {
+  for (const testCase of makeTestCaseIter(report)) {
+    if (!testCase.$['metadata-json']) {
+      yield [{}, testCase];
+    } else {
+      yield [{}, JSON.parse(testCase.$['metadata-json'])];
+    }
+  }
+}
+
+export function getReportMessages(report: TestReport) {
+  const messages: Message[] = [];
+  for (const [metadata, testCase] of getMetadataIter(report)) {
+    for (const message of metadata.messages || []) {
+      messages.push({
+        classname: testCase.$.classname,
+        name: testCase.$.name,
+        message,
+      });
+    }
+
+    for (const screenshot of metadata.screenshots || []) {
+      messages.push({
+        classname: testCase.$.classname,
+        name: testCase.$.name,
+        message: `Screenshot: ${screenshot.name} ${screenshot.url}`,
+      });
+    }
+  }
+  return messages;
+}

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -25,7 +25,8 @@ import { GithubApi } from './github_api';
 import { updateFailureIssue, createFailureIssue } from './report_failure';
 import { getIssueMetadata } from './issue_metadata';
 import { readTestReport } from './test_report';
-import { addMessagesToReport, Message } from './add_messages_to_report';
+import { addMessagesToReport } from './add_messages_to_report';
+import { getReportMessages } from './report_metadata';
 
 export function runFailedTestsReporterCli() {
   run(
@@ -74,17 +75,22 @@ export function runFailedTestsReporterCli() {
 
       for (const reportPath of reportPaths) {
         const report = await readTestReport(reportPath);
-        const messages: Message[] = [];
+        const messages = getReportMessages(report);
 
         for (const failure of await getFailures(report)) {
-          if (failure.likelyIrrelevant) {
+          const pushMessage = (msg: string) => {
             messages.push({
               classname: failure.classname,
               name: failure.name,
-              message:
-                'Failure is likely irrelevant' +
-                (updateGithub ? ', so an issue was not created or updated' : ''),
+              message: msg,
             });
+          };
+
+          if (failure.likelyIrrelevant) {
+            pushMessage(
+              'Failure is likely irrelevant' +
+                (updateGithub ? ', so an issue was not created or updated' : '')
+            );
             continue;
           }
 
@@ -97,30 +103,18 @@ export function runFailedTestsReporterCli() {
           if (existingIssue) {
             const newFailureCount = await updateFailureIssue(buildUrl, existingIssue, githubApi);
             const url = existingIssue.html_url;
-            const message =
-              `Test has failed ${newFailureCount - 1} times on tracked branches: ${url}` +
-              (updateGithub
-                ? `. Updated existing issue: ${url} (fail count: ${newFailureCount})`
-                : '');
-
-            messages.push({
-              classname: failure.classname,
-              name: failure.name,
-              message,
-            });
+            pushMessage(`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}`);
+            if (updateGithub) {
+              pushMessage(`Updated existing issue: ${url} (fail count: ${newFailureCount})`);
+            }
             continue;
           }
 
           const newIssueUrl = await createFailureIssue(buildUrl, failure, githubApi);
-          const message =
-            `Test has not failed recently on tracked branches` +
-            (updateGithub ? `Created new issue: ${newIssueUrl}` : '');
-
-          messages.push({
-            classname: failure.classname,
-            name: failure.name,
-            message,
-          });
+          pushMessage('Test has not failed recently on tracked branches');
+          if (updateGithub) {
+            pushMessage(`Created new issue: ${newIssueUrl}`);
+          }
         }
 
         // mutates report to include messages and writes updated report to disk

packages/kbn-test/src/failed_tests_reporter/test_report.ts

@@ -58,13 +58,15 @@ export interface TestCase {
     classname: string;
     /* number of seconds this test took */
     time: string;
+    /* optional JSON encoded metadata */
+    'metadata-json'?: string;
   };
   /* contents of system-out elements */
-  'system-out'?: string[];
+  'system-out'?: Array<string | { _: string }>;
   /* contents of failure elements */
   failure?: Array<string | { _: string }>;
   /* contents of skipped elements */
-  skipped?: string[];
+  skipped?: Array<string | { _: string }>;
 }
 
 export interface FailedTestCase extends TestCase {
@@ -82,19 +84,23 @@ export async function readTestReport(testReportPath: string) {
   return await parseTestReport(await readAsync(testReportPath, 'utf8'));
 }
 
-export function* makeFailedTestCaseIter(report: TestReport) {
-  // Grab the failures. Reporters may report multiple testsuites in a single file.
+export function* makeTestCaseIter(report: TestReport) {
+  // Reporters may report multiple testsuites in a single file.
   const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite];
 
   for (const testSuite of testSuites) {
     for (const testCase of testSuite.testcase) {
-      const { failure } = testCase;
-
-      if (!failure) {
-        continue;
-      }
-
-      yield testCase as FailedTestCase;
+      yield testCase;
     }
   }
 }
+
+export function* makeFailedTestCaseIter(report: TestReport) {
+  for (const testCase of makeTestCaseIter(report)) {
+    if (!testCase.failure) {
+      continue;
+    }
+
+    yield testCase as FailedTestCase;
+  }
+}

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -23,6 +23,7 @@ import { Suite, Test } from './fake_mocha_types';
 import {
   Lifecycle,
   LifecyclePhase,
+  FailureMetadata,
   readConfigFile,
   ProviderCollection,
   readProviderSpec,
@@ -33,6 +34,7 @@ import {
 
 export class FunctionalTestRunner {
   public readonly lifecycle = new Lifecycle();
+  public readonly failureMetadata = new FailureMetadata(this.lifecycle);
   private closed = false;
 
   constructor(
@@ -114,6 +116,7 @@ export class FunctionalTestRunner {
       const coreProviders = readProviderSpec('Service', {
         lifecycle: () => this.lifecycle,
         log: () => this.log,
+        failureMetadata: () => this.failureMetadata,
         config: () => config,
       });
 

packages/kbn-test/src/functional_test_runner/lib/failure_metadata.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Lifecycle } from './lifecycle';
+import { FailureMetadata } from './failure_metadata';
+
+it('collects metadata for the current test', async () => {
+  const lifecycle = new Lifecycle();
+  const failureMetadata = new FailureMetadata(lifecycle);
+
+  const test1 = {};
+  await lifecycle.beforeEachTest.trigger(test1);
+  failureMetadata.add({ foo: 'bar' });
+
+  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
+    Object {
+      "foo": "bar",
+    }
+  `);
+
+  const test2 = {};
+  await lifecycle.beforeEachTest.trigger(test2);
+  failureMetadata.add({ test: 2 });
+
+  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
+    Object {
+      "foo": "bar",
+    }
+  `);
+  expect(failureMetadata.get(test2)).toMatchInlineSnapshot(`
+    Object {
+      "test": 2,
+    }
+  `);
+});
+
+it('adds messages to the messages state', () => {
+  const lifecycle = new Lifecycle();
+  const failureMetadata = new FailureMetadata(lifecycle);
+
+  const test1 = {};
+  lifecycle.beforeEachTest.trigger(test1);
+  failureMetadata.addMessages(['foo', 'bar']);
+  failureMetadata.addMessages(['baz']);
+
+  expect(failureMetadata.get(test1)).toMatchInlineSnapshot(`
+    Object {
+      "messages": Array [
+        "foo",
+        "bar",
+        "baz",
+      ],
+    }
+  `);
+});

packages/kbn-test/src/functional_test_runner/lib/failure_metadata.ts

@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+
+import { REPO_ROOT } from '@kbn/dev-utils';
+
+import { Lifecycle } from './lifecycle';
+
+interface Metadata {
+  [key: string]: unknown;
+}
+
+export class FailureMetadata {
+  // mocha's global types mean we can't import Mocha or it will override the global jest types..............
+  private currentTest?: any;
+  private readonly allMetadata = new Map<any, Metadata>();
+
+  constructor(lifecycle: Lifecycle) {
+    if (!process.env.GCS_UPLOAD_PREFIX && process.env.CI) {
+      throw new Error(
+        'GCS_UPLOAD_PREFIX environment variable is not set and must always be set on CI'
+      );
+    }
+
+    lifecycle.beforeEachTest.add(test => {
+      this.currentTest = test;
+    });
+  }
+
+  add(metadata: Metadata | ((current: Metadata) => Metadata)) {
+    if (!this.currentTest) {
+      throw new Error('no current test to associate metadata with');
+    }
+
+    const current = this.allMetadata.get(this.currentTest);
+    this.allMetadata.set(this.currentTest, {
+      ...current,
+      ...(typeof metadata === 'function' ? metadata(current || {}) : metadata),
+    });
+  }
+
+  addMessages(messages: string[]) {
+    this.add(current => ({
+      messages: [...(Array.isArray(current.messages) ? current.messages : []), ...messages],
+    }));
+  }
+
+  /**
+   * @param name Name to label the URL with
+   * @param repoPath absolute path, within the repo, that will be uploaded
+   */
+  addScreenshot(name: string, repoPath: string) {
+    const prefix = process.env.GCS_UPLOAD_PREFIX;
+
+    if (!prefix) {
+      return;
+    }
+
+    const slash = prefix.endsWith('/') ? '' : '/';
+    const urlPath = Path.relative(REPO_ROOT, repoPath)
+      .split(Path.sep)
+      .map(c => encodeURIComponent(c))
+      .join('/');
+
+    if (urlPath.startsWith('..')) {
+      throw new Error(
+        `Only call addUploadLink() with paths that are within the repo root, received ${repoPath} and repo root is ${REPO_ROOT}`
+      );
+    }
+
+    const url = `https://storage.googleapis.com/${prefix}${slash}${urlPath}`;
+    const screenshot = {
+      name,
+      url,
+    };
+
+    this.add(current => ({
+      screenshots: [...(Array.isArray(current.screenshots) ? current.screenshots : []), screenshot],
+    }));
+
+    return screenshot;
+  }
+
+  get(test: any) {
+    return this.allMetadata.get(test);
+  }
+}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -22,3 +22,4 @@ export { LifecyclePhase } from './lifecycle_phase';
 export { readConfigFile, Config } from './config';
 export { readProviderSpec, ProviderCollection, Provider } from './providers';
 export { runTests, setupMocha } from './mocha';
+export { FailureMetadata } from './failure_metadata';

packages/kbn-test/src/functional_test_runner/lib/lifecycle_event.ts

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import * as Rx from 'rxjs';
+
+export type GetArgsType<T extends LifecycleEvent<any>> = T extends LifecycleEvent<infer X>
+  ? X
+  : never;
+
+export class LifecycleEvent<Args extends readonly any[]> {
+  private readonly handlers: Array<(...args: Args) => Promise<void> | void> = [];
+
+  private readonly beforeSubj = this.options.singular
+    ? new Rx.BehaviorSubject(undefined)
+    : new Rx.Subject<void>();
+  public readonly before$ = this.beforeSubj.asObservable();
+
+  private readonly afterSubj = this.options.singular
+    ? new Rx.BehaviorSubject(undefined)
+    : new Rx.Subject<void>();
+  public readonly after$ = this.afterSubj.asObservable();
+
+  constructor(
+    private readonly options: {
+      singular?: boolean;
+    } = {}
+  ) {}
+
+  public add(fn: (...args: Args) => Promise<void> | void) {
+    this.handlers.push(fn);
+  }
+
+  public async trigger(...args: Args) {
+    if (this.beforeSubj.isStopped) {
+      throw new Error(`singular lifecycle event can only be triggered once`);
+    }
+
+    this.beforeSubj.next(undefined);
+    if (this.options.singular) {
+      this.beforeSubj.complete();
+    }
+
+    try {
+      await Promise.all(this.handlers.map(async fn => await fn(...args)));
+    } finally {
+      this.afterSubj.next(undefined);
+      if (this.options.singular) {
+        this.afterSubj.complete();
+      }
+    }
+  }
+}

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -32,6 +32,7 @@ import { writeEpilogue } from './write_epilogue';
 export function MochaReporterProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
+  const failureMetadata = getService('failureMetadata');
   let originalLogWriters;
   let reporterCaptureStartTime;
 
@@ -53,6 +54,7 @@ export function MochaReporterProvider({ getService }) {
       if (config.get('junit.enabled') && config.get('junit.reportName')) {
         setupJUnitReportGeneration(runner, {
           reportName: config.get('junit.reportName'),
+          getTestMetadata: t => failureMetadata.get(t),
         });
       }
     }

packages/kbn-test/src/mocha/__tests__/junit_report_generation.js

@@ -98,6 +98,7 @@ describe('dev/mocha/junit report generation', () => {
         classname: sharedClassname,
         name: 'SUITE works',
         time: testPass.$.time,
+        'metadata-json': '{}',
       },
       'system-out': testPass['system-out'],
     });
@@ -109,6 +110,7 @@ describe('dev/mocha/junit report generation', () => {
         classname: sharedClassname,
         name: 'SUITE fails',
         time: testFail.$.time,
+        'metadata-json': '{}',
       },
       'system-out': testFail['system-out'],
       failure: [testFail.failure[0]],
@@ -124,6 +126,7 @@ describe('dev/mocha/junit report generation', () => {
         classname: sharedClassname,
         name: 'SUITE SUB_SUITE "before each" hook: fail hook for "never runs"',
         time: beforeEachFail.$.time,
+        'metadata-json': '{}',
       },
       'system-out': testFail['system-out'],
       failure: [beforeEachFail.failure[0]],
@@ -133,6 +136,7 @@ describe('dev/mocha/junit report generation', () => {
       $: {
         classname: sharedClassname,
         name: 'SUITE SUB_SUITE never runs',
+        'metadata-json': '{}',
       },
       'system-out': testFail['system-out'],
       skipped: [''],

packages/kbn-test/src/mocha/junit_report_generation.js

@@ -32,6 +32,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
   const {
     reportName = 'Unnamed Mocha Tests',
     rootDirectory = dirname(require.resolve('../../../../package.json')),
+    getTestMetadata = () => ({}),
   } = options;
 
   const stats = {};
@@ -118,6 +119,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
         name: getFullTitle(node),
         classname: `${reportName}.${getPath(node).replace(/\./g, '·')}`,
         time: getDuration(node),
+        'metadata-json': JSON.stringify(getTestMetadata(node) || {}),
       });
     }
 

packages/kbn-test/types/ftr.d.ts

@@ -18,9 +18,9 @@
  */
 
 import { ToolingLog } from '@kbn/dev-utils';
-import { Config, Lifecycle } from '../src/functional_test_runner/lib';
+import { Config, Lifecycle, FailureMetadata } from '../src/functional_test_runner/lib';
 
-export { Lifecycle, Config };
+export { Lifecycle, Config, FailureMetadata };
 
 interface AsyncInstance<T> {
   /**
@@ -61,7 +61,7 @@ export interface GenericFtrProviderContext<
    * Determine if a service is avaliable
    * @param serviceName
    */
-  hasService(serviceName: 'config' | 'log' | 'lifecycle'): true;
+  hasService(serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata'): true;
   hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
   hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
 
@@ -73,6 +73,7 @@ export interface GenericFtrProviderContext<
   getService(serviceName: 'config'): Config;
   getService(serviceName: 'log'): ToolingLog;
   getService(serviceName: 'lifecycle'): Lifecycle;
+  getService(serviceName: 'failureMetadata'): FailureMetadata;
   getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
 
   /**

test/functional/services/screenshots.ts

@@ -22,6 +22,7 @@ import { writeFile, readFileSync, mkdir } from 'fs';
 import { promisify } from 'util';
 
 import del from 'del';
+
 import { comparePngs } from './lib/compare_pngs';
 import { FtrProviderContext } from '../ftr_provider_context';
 import { WebElementWrapper } from './lib/web_element_wrapper';
@@ -32,6 +33,7 @@ const writeFileAsync = promisify(writeFile);
 export async function ScreenshotsProvider({ getService }: FtrProviderContext) {
   const log = getService('log');
   const config = getService('config');
+  const failureMetadata = getService('failureMetadata');
   const browser = getService('browser');
 
   const SESSION_DIRECTORY = resolve(config.get('screenshots.directory'), 'session');
@@ -68,11 +70,15 @@ export async function ScreenshotsProvider({ getService }: FtrProviderContext) {
     }
 
     async take(name: string, el?: WebElementWrapper) {
-      return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`), el);
+      const path = resolve(SESSION_DIRECTORY, `${name}.png`);
+      await this._take(path, el);
+      failureMetadata.addScreenshot(name, path);
     }
 
     async takeForFailure(name: string, el?: WebElementWrapper) {
-      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`), el);
+      const path = resolve(FAILURE_DIRECTORY, `${name}.png`);
+      await this._take(path, el);
+      failureMetadata.addScreenshot(`failure[${name}]`, path);
     }
 
     async _take(path: string, el?: WebElementWrapper) {

vars/kibanaPipeline.groovy

@@ -1,47 +1,45 @@
-def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
+def withWorkers(machineName, preWorkerClosure = {}, workerClosures = [:]) {
   return {
     jobRunner('tests-xl', true) {
-      try {
-        doSetup()
-        preWorkerClosure()
+      withGcsArtifactUpload(machineName, {
+        try {
+          doSetup()
+          preWorkerClosure()
 
-        def nextWorker = 1
-        def worker = { workerClosure ->
-          def workerNumber = nextWorker
-          nextWorker++
+          def nextWorker = 1
+          def worker = { workerClosure ->
+            def workerNumber = nextWorker
+            nextWorker++
 
-          return {
-            // This delay helps smooth out CPU load caused by ES/Kibana instances starting up at the same time
-            def delay = (workerNumber-1)*20
-            sleep(delay)
+            return {
+              // This delay helps smooth out CPU load caused by ES/Kibana instances starting up at the same time
+              def delay = (workerNumber-1)*20
+              sleep(delay)
 
-            workerClosure(workerNumber)
+              workerClosure(workerNumber)
+            }
+          }
+
+          def workers = [:]
+          workerClosures.each { workerName, workerClosure ->
+            workers[workerName] = worker(workerClosure)
+          }
+
+          parallel(workers)
+        } finally {
+          catchError {
+            runErrorReporter()
+          }
+
+          catchError {
+            runbld.junit()
+          }
+
+          catchError {
+            publishJunit()
           }
         }
-
-        def workers = [:]
-        workerClosures.each { workerName, workerClosure ->
-          workers[workerName] = worker(workerClosure)
-        }
-
-        parallel(workers)
-      } finally {
-        catchError {
-          uploadAllGcsArtifacts(name)
-        }
-
-        catchError {
-          runErrorReporter()
-        }
-
-        catchError {
-          runbld.junit()
-        }
-
-        catchError {
-          publishJunit()
-        }
-      }
+      })
     }
   }
 }
@@ -96,19 +94,19 @@ def legacyJobRunner(name) {
           "JOB=${name}",
         ]) {
           jobRunner('linux && immutable', false) {
-            try {
-              runbld('.ci/run.sh', "Execute ${name}", true)
-            } finally {
-              catchError {
-                uploadAllGcsArtifacts(name)
+            withGcsArtifactUpload(name, {
+              try {
+                runbld('.ci/run.sh', "Execute ${name}", true)
+              } finally {
+                catchError {
+                  runErrorReporter()
+                }
+
+                catchError {
+                  publishJunit()
+                }
               }
-              catchError {
-                runErrorReporter()
-              }
-              catchError {
-                publishJunit()
-              }
-            }
+            })
           }
         }
       }
@@ -171,19 +169,18 @@ def jobRunner(label, useRamDisk, closure) {
 
 // TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build?
 
-def uploadGcsArtifact(workerName, pattern) {
-  def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" // TODO
-
+def uploadGcsArtifact(uploadPrefix, pattern) {
   googleStorageUpload(
     credentialsId: 'kibana-ci-gcs-plugin',
-    bucket: storageLocation,
+    bucket: "gs://${uploadPrefix}",
     pattern: pattern,
     sharedPublicly: true,
     showInline: true,
   )
 }
 
-def uploadAllGcsArtifacts(workerName) {
+def withGcsArtifactUpload(workerName, closure) {
+  def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}"
   def ARTIFACT_PATTERNS = [
     'target/kibana-*',
     'target/junit/**/*',
@@ -194,9 +191,19 @@ def uploadAllGcsArtifacts(workerName) {
     'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
   ]
 
-  ARTIFACT_PATTERNS.each { pattern ->
-    uploadGcsArtifact(workerName, pattern)
-  }
+  withEnv([
+    "GCS_UPLOAD_PREFIX=${uploadPrefix}"
+  ], {
+    try {
+      closure()
+    } finally {
+      catchError {
+        ARTIFACT_PATTERNS.each { pattern ->
+          uploadGcsArtifact(uploadPrefix, pattern)
+        }
+      }
+    }
+  })
 }
 
 def publishJunit() {