github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.14] [CI] Display command on failure page (#186999) (#187351)

Browse source 
# Backport

This will backport the following commits from `main` to `8.14`:
- [[CI] Display command on failure page
(#186999)](https://github.com/elastic/kibana/pull/186999)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Alex
Szabo","email":"alex.szabo@elastic.co"},"sourceCommit":{"committedDate":"2024-07-02T09:45:46Z","message":"[CI]
Display command on failure page (#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466","branchLabelMapping":{"^v8.15.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","backport:prev-minor","v8.15.0"],"number":186999,"url":"https://github.com/elastic/kibana/pull/186999","mergeCommit":{"message":"[CI]
Display command on failure page (#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.15.0","labelRegex":"^v8.15.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/186999","number":186999,"mergeCommit":{"message":"[CI]
Display command on failure page (#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466"}}]}]
BACKPORT-->
This commit is contained in:
Alex Szabo 2024-07-02 18:43:16 +02:00 • committed by GitHub
parent 34a01009cc
commit c47097b998
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 108 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

4
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71">
<testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
<testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
<testcase name="maps app maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378">
<system-out>
<![CDATA[[00:00:00] │

4
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/jest_report.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9">
<testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts">
<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9" command-line="node scripts/jest --config some/jest/config.ts">
<testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts">
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/>
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/>
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060">

4
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3">
<testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js">
<testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js">
<testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121">
<system-out>
<![CDATA[]]>

10
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts
View file

@ -60,8 +60,8 @@ it('rewrites ftr reports with minimal changes', async () => {
+++ ftr.xml
@@ -1,53 +1,56 @@
‹?xml version="1.0" encoding="utf-8"?›
‹testsuites›
‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
‹testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
‹testcase name="maps app maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
- ‹system-out›
- ‹![CDATA[[00:00:00] │
@ -155,7 +155,7 @@ it('rewrites jest reports with minimal changes', async () => {
--- jest.xml
+++ jest.xml
@@ -3,13 +3,17 @@
‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"›
‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts"›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"›
@ -203,8 +203,8 @@ it('rewrites mocha reports with minimal changes', async () => {
+++ mocha.xml
@@ -1,13 +1,16 @@
‹?xml version="1.0" encoding="utf-8"?›
‹testsuites›
‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
‹testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
‹testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
- ‹system-out›
- ‹![CDATA[]]›

6
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts
View file

@ -16,6 +16,7 @@ it('discovers failures in ftr report', async () => {
Array [
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "
Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
Wait timed out after 10055ms
@ -37,6 +38,7 @@ it('discovers failures in ftr report', async () => {
},
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "
{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
@ -56,6 +58,7 @@ it('discovers failures in ftr report', async () => {
},
Object {
"classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "{ NoSuchSessionError: Tried to run command without establishing a connection
at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
@ -76,6 +79,7 @@ it('discovers failures in jest report', async () => {
Array [
Object {
"classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
"commandLine": "node scripts/jest --config some/jest/config.ts",
"failure": "
TypeError: Cannot read property '0' of undefined
at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
@ -95,6 +99,7 @@ it('discovers failures in mocha report', async () => {
Array [
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
"failure": "
Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
<html>
@ -117,6 +122,7 @@ it('discovers failures in mocha report', async () => {
},
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
"failure": "
TypeError: Cannot read property 'shutdown' of undefined
at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)

21
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts
View file

@ -16,6 +16,7 @@ export type TestFailure = FailedTestCase['$'] & {
'system-out'?: string;
githubIssue?: string;
failureCount?: number;
commandLine?: string;
};
const getText = (node?: Array<string | { _: string }>) => {
@ -71,19 +72,35 @@ const isLikelyIrrelevant = (name: string, failure: string) => {
export function getFailures(report: TestReport) {
const failures: TestFailure[] = [];
const commandLine = getCommandLineFromReport(report);
for (const testCase of makeFailedTestCaseIter(report)) {
const failure = getText(testCase.failure);
const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure);
failures.push({
const failureObj = {
// unwrap xml weirdness
...testCase.$,
// Strip ANSI color characters
failure,
likelyIrrelevant,
'system-out': getText(testCase['system-out']),
});
commandLine,
};
// cleaning up duplicates
delete failureObj['command-line'];
failures.push(failureObj);
}
return failures;
}
function getCommandLineFromReport(report: TestReport) {
if ('testsuites' in report) {
return report.testsuites?.testsuite?.[0]?.$['command-line'] || '';
} else {
return report.testsuite?.$['command-line'] || '';
}
}

21
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts
View file

@ -170,14 +170,23 @@ export async function reportFailuresToFile(
<p><strong>${escape(failure.name)}</strong></p>
<p>
<small>
<strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
failure.failureCount || 0
}</span>
${
failure.commandLine
? `<div>
<strong>Command Line</strong>:
<pre>${escape(failure.commandLine)}</pre>
</div>`
: ''
}
<div>
<strong>Failures in tracked branches</strong>:
<span class="badge rounded-pill bg-danger">${failure.failureCount || 0}</span>
</div>
${
failure.githubIssue
? `<br /><a href="${escape(failure.githubIssue)}">${escape(
failure.githubIssue
)}</a>`
? `<div>
<a href="${escape(failure.githubIssue)}">${escape(failure.githubIssue)}</a>
</div>`
: ''
}
</small>

4
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts
View file

@ -37,6 +37,8 @@ export interface TestSuite {
skipped: string;
/* optional JSON encoded metadata */
'metadata-json'?: string;
/* the command that ran this suite */
'command-line'?: string;
};
testcase?: TestCase[];
}
@ -51,6 +53,8 @@ export interface TestCase {
time: string;
/* optional JSON encoded metadata */
'metadata-json'?: string;
/* the command that ran this suite */
'command-line'?: string;
};
/* contents of system-out elements */
'system-out'?: Array<string | { _: string }>;

3
packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts
View file

@ -17,6 +17,7 @@ import { AggregatedResult, Test, BaseReporter } from '@jest/reporters';
import { escapeCdata } from '../../mocha/xml';
import { getUniqueJunitReportPath } from '../../report_path';
import { prettifyCommandLine } from '../../prettify_command_line';
interface ReporterOptions {
reportName?: string;
@ -71,6 +72,7 @@ export default class JestJUnitReporter extends BaseReporter {
tests: results.numTotalTests,
failures: results.numFailedTests,
skipped: results.numPendingTests,
'command-line': prettifyCommandLine(process.argv),
});
// top level test results are the files/suites
@ -83,6 +85,7 @@ export default class JestJUnitReporter extends BaseReporter {
failures: suite.numFailingTests,
skipped: suite.numPendingTests,
file: suite.testFilePath,
'command-line': prettifyCommandLine(process.argv),
});
// nested in there are the tests in that file

19
packages/kbn-test/src/mocha/junit_report_generation.js
View file

@ -16,6 +16,7 @@ import { getUniqueJunitReportPath } from '../report_path';
import { getSnapshotOfRunnableLogs } from './log_cache';
import { escapeCdata } from '../..';
import { prettifyCommandLine } from '../prettify_command_line';
const dateNow = Date.now.bind(Date);
@ -91,14 +92,25 @@ export function setupJUnitReportGeneration(runner, options = {}) {
.filter((node) => node.pending || !results.find((result) => result.node === node))
.map((node) => ({ skipped: true, node }));
const builder = xmlBuilder.create(
const commandLine = prettifyCommandLine(process.argv);
const root = xmlBuilder.create(
'testsuites',
{ encoding: 'utf-8' },
{},
{ skipNullAttributes: true }
);
const testsuitesEl = builder.ele('testsuite', {
root.att({
name: 'ftr',
time: getDuration(stats),
tests: allTests.length + failedHooks.length,
failures: failures.length,
skipped: skippedResults.length,
'command-line': commandLine,
});
const testsuitesEl = root.ele('testsuite', {
name: reportName,
timestamp: new Date(stats.startTime).toISOString().slice(0, -5),
time: getDuration(stats),
@ -106,6 +118,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
failures: failures.length,
skipped: skippedResults.length,
'metadata-json': JSON.stringify(metadata ?? {}),
'command-line': commandLine,
});
function addTestcaseEl(node) {
@ -134,7 +147,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
});
const reportPath = getUniqueJunitReportPath(rootDirectory, reportName);
const reportXML = builder.end();
const reportXML = root.end();
mkdirSync(dirname(reportPath), { recursive: true });
writeFileSync(reportPath, reportXML, 'utf8');
});

25
packages/kbn-test/src/mocha/junit_report_generation.test.js
View file

@ -45,26 +45,25 @@ describe('dev/mocha/junit report generation', () => {
// test case results are wrapped in <testsuites></testsuites>
expect(report).toEqual({
testsuites: {
testsuites: expect.objectContaining({
testsuite: [report.testsuites.testsuite[0]],
},
}),
});
// the single <testsuite> element at the root contains summary data for all tests results
const [testsuite] = report.testsuites.testsuite;
expect(testsuite.$.time).toMatch(DURATION_REGEX);
expect(testsuite.$.timestamp).toMatch(ISO_DATE_SEC_REGEX);
expect(testsuite).toEqual({
$: {
failures: '2',
name: 'test',
skipped: '1',
tests: '4',
'metadata-json': '{}',
time: testsuite.$.time,
timestamp: testsuite.$.timestamp,
},
testcase: testsuite.testcase,
expect(testsuite.$).toEqual({
'command-line':
'node scripts/jest --config=packages/kbn-test/jest.config.js --runInBand --coverage=false --passWithNoTests',
failures: '2',
name: 'test',
skipped: '1',
tests: '4',
'metadata-json': '{}',
time: testsuite.$.time,
timestamp: testsuite.$.timestamp,
});
// there are actually only three tests, but since the hook failed

22
packages/kbn-test/src/prettify_command_line.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { execSync } from 'child_process';
import * as path from 'path';
const kibanaRoot = execSync('git rev-parse --show-toplevel').toString().trim() || process.cwd();
export function prettifyCommandLine(args: string[]) {
let [executable, ...rest] = args;
if (executable.endsWith('node')) {
executable = 'node';
}
rest = rest.map((arg) => path.relative(kibanaRoot, arg));
return [executable, ...rest].join(' ');
}