[CI] Display command on failure page (#186999)

## Summary This PR adds the executed command line to the failures page. We tweak the reporters to export the executed command to the junit xmls, then we read those attributes after parsing the results. The tests needed some adjustment, because they're very brittle, and don't seem to be very accurate anymore. Closes: https://github.com/elastic/kibana-operations/issues/127 Check out the `[logs]` for the failed tests here (ftr/jest/jest_integration): https://buildkite.com/elastic/kibana-pull-request/builds/218457
2025-04-23 17:28:26 -04:00 · 2024-07-02 11:45:46 +02:00 · 2024-07-02 11:45:46 +02:00 · afec9eb0e2
commit afec9eb0e2
parent 3699c3a449
12 changed files with 100 additions and 24 deletions
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/ftr_report.xml
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/ftr_report.xml
@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites>
-  <testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71">
+<testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
+  <testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
    <testcase name="maps app  maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378">
      <system-out>
        <![CDATA[[00:00:00]       │
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/jest_report.xml
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/jest_report.xml
@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9">
-  <testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts">
+<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9" command-line="node scripts/jest --config some/jest/config.ts">
+  <testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts">
    <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/>
    <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/>
    <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060">
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/mocha_report.xml
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/fixtures/mocha_report.xml
@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites>
-  <testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3">
+<testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js">
+  <testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js">
    <testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121">
      <system-out>
        <![CDATA[]]>
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts
@ -60,8 +60,8 @@ it('rewrites ftr reports with minimal changes', async () => {
    +++ ftr.xml
    @@ -1,53 +1,56 @@
     ‹?xml version="1.0" encoding="utf-8"?›
-     ‹testsuites›
-       ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
+     ‹testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
+       ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
         ‹testcase name="maps app  maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
    -      ‹system-out›
    -        ‹![CDATA[[00:00:00]       │
@ -155,7 +155,7 @@ it('rewrites jest reports with minimal changes', async () => {
    --- jest.xml
    +++ jest.xml
    @@ -3,13 +3,17 @@
-       ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"›
+       ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts"›
         ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/›
         ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/›
         ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"›
@ -203,8 +203,8 @@ it('rewrites mocha reports with minimal changes', async () => {
    +++ mocha.xml
    @@ -1,13 +1,16 @@
     ‹?xml version="1.0" encoding="utf-8"?›
-     ‹testsuites›
-       ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
+     ‹testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
+       ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
         ‹testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
    -      ‹system-out›
    -        ‹![CDATA[]]›
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts
@ -16,6 +16,7 @@ it('discovers failures in ftr report', async () => {
    Array [
      Object {
        "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
        "failure": "
            Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
    Wait timed out after 10055ms
@ -37,6 +38,7 @@ it('discovers failures in ftr report', async () => {
      },
      Object {
        "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
        "failure": "
            { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
        at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
@ -56,6 +58,7 @@ it('discovers failures in ftr report', async () => {
      },
      Object {
        "classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
        "failure": "{ NoSuchSessionError: Tried to run command without establishing a connection
        at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
        at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
@ -76,6 +79,7 @@ it('discovers failures in jest report', async () => {
    Array [
      Object {
        "classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
+        "commandLine": "node scripts/jest --config some/jest/config.ts",
        "failure": "
            TypeError: Cannot read property '0' of undefined
        at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
@ -95,6 +99,7 @@ it('discovers failures in mocha report', async () => {
    Array [
      Object {
        "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
+        "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
        "failure": "
            Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
      <html>
@ -117,6 +122,7 @@ it('discovers failures in mocha report', async () => {
      },
      Object {
        "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
+        "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
        "failure": "
            TypeError: Cannot read property 'shutdown' of undefined
        at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts
@ -16,6 +16,7 @@ export type TestFailure = FailedTestCase['$'] & {
  'system-out'?: string;
  githubIssue?: string;
  failureCount?: number;
+  commandLine?: string;
 };

 const getText = (node?: Array<string | { _: string }>) => {
@ -71,19 +72,35 @@ const isLikelyIrrelevant = (name: string, failure: string) => {
 export function getFailures(report: TestReport) {
  const failures: TestFailure[] = [];

+  const commandLine = getCommandLineFromReport(report);
+
  for (const testCase of makeFailedTestCaseIter(report)) {
    const failure = getText(testCase.failure);
    const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure);

-    failures.push({
+    const failureObj = {
      // unwrap xml weirdness
      ...testCase.$,
      // Strip ANSI color characters
      failure,
      likelyIrrelevant,
      'system-out': getText(testCase['system-out']),
-    });
+      commandLine,
+    };
+
+    // cleaning up duplicates
+    delete failureObj['command-line'];
+
+    failures.push(failureObj);
  }

  return failures;
 }
+
+function getCommandLineFromReport(report: TestReport) {
+  if ('testsuites' in report) {
+    return report.testsuites?.testsuite?.[0]?.$['command-line'] || '';
+  } else {
+    return report.testsuite?.$['command-line'] || '';
+  }
+}
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts
@ -170,14 +170,23 @@ export async function reportFailuresToFile(
        <p><strong>${escape(failure.name)}</strong></p>
        <p>
          <small>
-            <strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
-              failure.failureCount || 0
-            }</span>
+            ${
+              failure.commandLine
+                ? `<div>
+                     <strong>Command Line</strong>:
+                     <pre>${escape(failure.commandLine)}</pre>
+                   </div>`
+                : ''
+            }
+            <div>
+                <strong>Failures in tracked branches</strong>:
+                    <span class="badge rounded-pill bg-danger">${failure.failureCount || 0}</span>
+            </div>
            ${
              failure.githubIssue
-                ? `<br /><a href="${escape(failure.githubIssue)}">${escape(
-                    failure.githubIssue
-                  )}</a>`
+                ? `<div>
+                     <a href="${escape(failure.githubIssue)}">${escape(failure.githubIssue)}</a>
+                   </div>`
                : ''
            }
          </small>
--- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts
@ -37,6 +37,8 @@ export interface TestSuite {
    skipped: string;
    /* optional JSON encoded metadata */
    'metadata-json'?: string;
+    /* the command that ran this suite */
+    'command-line'?: string;
  };
  testcase?: TestCase[];
 }
@ -51,6 +53,8 @@ export interface TestCase {
    time: string;
    /* optional JSON encoded metadata */
    'metadata-json'?: string;
+    /* the command that ran this suite */
+    'command-line'?: string;
  };
  /* contents of system-out elements */
  'system-out'?: Array<string | { _: string }>;
--- a/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts
+++ b/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts
@ -17,6 +17,7 @@ import { AggregatedResult, Test, BaseReporter } from '@jest/reporters';

 import { escapeCdata } from '../../mocha/xml';
 import { getUniqueJunitReportPath } from '../../report_path';
+import { prettifyCommandLine } from '../../prettify_command_line';

 interface ReporterOptions {
  reportName?: string;
@ -71,6 +72,7 @@ export default class JestJUnitReporter extends BaseReporter {
      tests: results.numTotalTests,
      failures: results.numFailedTests,
      skipped: results.numPendingTests,
+      'command-line': prettifyCommandLine(process.argv),
    });

    // top level test results are the files/suites
@ -83,6 +85,7 @@ export default class JestJUnitReporter extends BaseReporter {
        failures: suite.numFailingTests,
        skipped: suite.numPendingTests,
        file: suite.testFilePath,
+        'command-line': prettifyCommandLine(process.argv),
      });

      // nested in there are the tests in that file
--- a/packages/kbn-test/src/mocha/junit_report_generation.js
+++ b/packages/kbn-test/src/mocha/junit_report_generation.js
@ -17,6 +17,7 @@ import { getUniqueJunitReportPath } from '../report_path';

 import { getSnapshotOfRunnableLogs } from './log_cache';
 import { escapeCdata } from '../..';
+import { prettifyCommandLine } from '../prettify_command_line';

 const dateNow = Date.now.bind(Date);

@ -95,14 +96,25 @@ export function setupJUnitReportGeneration(runner, options = {}) {
    // cache codeowners for quicker lookup
    const reversedCodeowners = getPathsWithOwnersReversed();

-    const builder = xmlBuilder.create(
+    const commandLine = prettifyCommandLine(process.argv);
+
+    const root = xmlBuilder.create(
      'testsuites',
      { encoding: 'utf-8' },
      {},
      { skipNullAttributes: true }
    );

-    const testsuitesEl = builder.ele('testsuite', {
+    root.att({
+      name: 'ftr',
+      time: getDuration(stats),
+      tests: allTests.length + failedHooks.length,
+      failures: failures.length,
+      skipped: skippedResults.length,
+      'command-line': commandLine,
+    });
+
+    const testsuitesEl = root.ele('testsuite', {
      name: reportName,
      timestamp: new Date(stats.startTime).toISOString().slice(0, -5),
      time: getDuration(stats),
@ -110,6 +122,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
      failures: failures.length,
      skipped: skippedResults.length,
      'metadata-json': JSON.stringify(metadata ?? {}),
+      'command-line': commandLine,
    });

    function addTestcaseEl(node, failed) {
@ -147,7 +160,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
    });

    const reportPath = getUniqueJunitReportPath(rootDirectory, reportName);
-    const reportXML = builder.end();
+    const reportXML = root.end();
    mkdirSync(dirname(reportPath), { recursive: true });
    writeFileSync(reportPath, reportXML, 'utf8');
  });
--- a/packages/kbn-test/src/mocha/junit_report_generation.test.js
+++ b/packages/kbn-test/src/mocha/junit_report_generation.test.js
@ -45,9 +45,9 @@ describe('dev/mocha/junit report generation', () => {

    // test case results are wrapped in <testsuites></testsuites>
    expect(report).toEqual({
-      testsuites: {
+      testsuites: expect.objectContaining({
        testsuite: [report.testsuites.testsuite[0]],
-      },
+      }),
    });

    // the single <testsuite> element at the root contains summary data for all tests results
@ -55,6 +55,8 @@ describe('dev/mocha/junit report generation', () => {
    expect(testsuite.$.time).toMatch(DURATION_REGEX);
    expect(testsuite.$.timestamp).toMatch(ISO_DATE_SEC_REGEX);
    expect(testsuite.$).toEqual({
+      'command-line':
+        'node scripts/jest --config=packages/kbn-test/jest.config.js --runInBand --coverage=false --passWithNoTests',
      failures: '2',
      name: 'test',
      skipped: '1',
--- a/packages/kbn-test/src/prettify_command_line.ts
+++ b/packages/kbn-test/src/prettify_command_line.ts
@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+import * as path from 'path';
+
+const kibanaRoot = execSync('git rev-parse --show-toplevel').toString().trim() || process.cwd();
+
+export function prettifyCommandLine(args: string[]) {
+  let [executable, ...rest] = args;
+  if (executable.endsWith('node')) {
+    executable = 'node';
+  }
+  rest = rest.map((arg) => path.relative(kibanaRoot, arg));
+
+  return [executable, ...rest].join(' ');
+}