mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[failed_tests_cli] update reports with links to github issues
This commit is contained in:
spalger
parent
be088a16ed
commit
207cff0966
10 changed files with 595 additions and 127 deletions
|
|
@ -15,7 +15,8 @@
|
|||
"@kbn/dev-utils": "1.0.0",
|
||||
"@types/parse-link-header": "^1.0.0",
|
||||
"@types/strip-ansi": "^5.2.1",
|
||||
"@types/xml2js": "^0.4.5"
|
||||
"@types/xml2js": "^0.4.5",
|
||||
"diff": "^4.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"chalk": "^2.4.2",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
const Fs = jest.requireActual('fs');
|
||||
|
||||
export const FTR_REPORT = Fs.readFileSync(require.resolve('./ftr_report.xml'), 'utf8');
|
||||
export const JEST_REPORT = Fs.readFileSync(require.resolve('./jest_report.xml'), 'utf8');
|
||||
export const KARMA_REPORT = Fs.readFileSync(require.resolve('./karma_report.xml'), 'utf8');
|
||||
export const MOCHA_REPORT = Fs.readFileSync(require.resolve('./mocha_report.xml'), 'utf8');
|
||||
|
|
@ -17,14 +17,12 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
|
||||
import { getFailures } from './get_failures';
|
||||
|
||||
const log = new ToolingLog();
|
||||
import { parseTestReport } from './test_report';
|
||||
import { FTR_REPORT, JEST_REPORT, KARMA_REPORT, MOCHA_REPORT } from './__fixtures__';
|
||||
|
||||
it('discovers failures in ftr report', async () => {
|
||||
const failures = await getFailures(log, require.resolve('./__fixtures__/ftr_report.xml'));
|
||||
const failures = getFailures(await parseTestReport(FTR_REPORT));
|
||||
expect(failures).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
|
|
@ -45,7 +43,7 @@ it('discovers failures in ftr report', async () => {
|
|||
});
|
||||
|
||||
it('discovers failures in jest report', async () => {
|
||||
const failures = await getFailures(log, require.resolve('./__fixtures__/jest_report.xml'));
|
||||
const failures = getFailures(await parseTestReport(JEST_REPORT));
|
||||
expect(failures).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
|
|
@ -62,7 +60,7 @@ it('discovers failures in jest report', async () => {
|
|||
});
|
||||
|
||||
it('discovers failures in karma report', async () => {
|
||||
const failures = await getFailures(log, require.resolve('./__fixtures__/karma_report.xml'));
|
||||
const failures = getFailures(await parseTestReport(KARMA_REPORT));
|
||||
expect(failures).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
|
|
@ -86,6 +84,6 @@ it('discovers failures in karma report', async () => {
|
|||
});
|
||||
|
||||
it('discovers failures in mocha report', async () => {
|
||||
const failures = await getFailures(log, require.resolve('./__fixtures__/mocha_report.xml'));
|
||||
const failures = getFailures(await parseTestReport(MOCHA_REPORT));
|
||||
expect(failures).toMatchInlineSnapshot(`Array []`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -17,69 +17,15 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import { promisify } from 'util';
|
||||
import Fs from 'fs';
|
||||
|
||||
import xml2js from 'xml2js';
|
||||
import stripAnsi from 'strip-ansi';
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
|
||||
type TestReport =
|
||||
| {
|
||||
testsuites: {
|
||||
testsuite: TestSuite[];
|
||||
};
|
||||
}
|
||||
| {
|
||||
testsuite: TestSuite;
|
||||
};
|
||||
import { FailedTestCase, TestReport, makeFailedTestCaseIter } from './test_report';
|
||||
|
||||
interface TestSuite {
|
||||
$: {
|
||||
/* ISO8601 timetamp when test suite ran */
|
||||
timestamp: string;
|
||||
/* number of second this tests suite took */
|
||||
time: string;
|
||||
/* number of tests as a string */
|
||||
tests: string;
|
||||
/* number of failed tests as a string */
|
||||
failures: string;
|
||||
/* number of skipped tests as a string */
|
||||
skipped: string;
|
||||
};
|
||||
testcase: TestCase[];
|
||||
}
|
||||
|
||||
interface TestCase {
|
||||
$: {
|
||||
/* unique test name */
|
||||
name: string;
|
||||
/* somewhat human readable combination of test name and file */
|
||||
classname: string;
|
||||
/* number of seconds this test took */
|
||||
time: string;
|
||||
};
|
||||
/* contents of system-out elements */
|
||||
'system-out'?: string[];
|
||||
/* contents of failure elements */
|
||||
failure?: Array<string | { _: string }>;
|
||||
/* contents of skipped elements */
|
||||
skipped?: string[];
|
||||
}
|
||||
|
||||
export type TestFailure = TestCase['$'] & {
|
||||
export type TestFailure = FailedTestCase['$'] & {
|
||||
failure: string;
|
||||
};
|
||||
|
||||
const readAsync = promisify(Fs.readFile);
|
||||
|
||||
const indent = (text: string) =>
|
||||
` ${text
|
||||
.split('\n')
|
||||
.map(l => ` ${l}`)
|
||||
.join('\n')}`;
|
||||
|
||||
const getFailureText = (failure: NonNullable<TestCase['failure']>) => {
|
||||
const getFailureText = (failure: FailedTestCase['failure']) => {
|
||||
const [failureNode] = failure;
|
||||
|
||||
if (failureNode && typeof failureNode === 'object' && typeof failureNode._ === 'string') {
|
||||
|
|
@ -119,45 +65,22 @@ const isLikelyIrrelevant = ({ name, failure }: TestFailure) => {
|
|||
}
|
||||
};
|
||||
|
||||
export async function getFailures(log: ToolingLog, testReportPath: string) {
|
||||
const xml = await readAsync(testReportPath, 'utf8');
|
||||
export function getFailures(report: TestReport) {
|
||||
const failures: Array<{ type: 'ignore' | 'report'; failure: TestFailure }> = [];
|
||||
|
||||
// Parses junit XML files
|
||||
const report: TestReport = await xml2js.parseStringPromise(xml);
|
||||
for (const testCase of makeFailedTestCaseIter(report)) {
|
||||
// unwrap xml weirdness
|
||||
const failure: TestFailure = {
|
||||
...testCase.$,
|
||||
// Strip ANSI color characters
|
||||
failure: getFailureText(testCase.failure),
|
||||
};
|
||||
|
||||
// Grab the failures. Reporters may report multiple testsuites in a single file.
|
||||
const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite];
|
||||
|
||||
const failures: TestFailure[] = [];
|
||||
for (const testSuite of testSuites) {
|
||||
for (const testCase of testSuite.testcase) {
|
||||
const { failure } = testCase;
|
||||
|
||||
if (!failure) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// unwrap xml weirdness
|
||||
const failureCase: TestFailure = {
|
||||
...testCase.$,
|
||||
// Strip ANSI color characters
|
||||
failure: getFailureText(failure),
|
||||
};
|
||||
|
||||
if (isLikelyIrrelevant(failureCase)) {
|
||||
log.warning(
|
||||
`Ignoring likely irrelevant failure: ${failureCase.classname} - ${
|
||||
failureCase.name
|
||||
}\n${indent(failureCase.failure)}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
failures.push(failureCase);
|
||||
}
|
||||
failures.push({
|
||||
type: isLikelyIrrelevant(failure) ? 'ignore' : 'report',
|
||||
failure,
|
||||
});
|
||||
}
|
||||
|
||||
log.info(`Found ${failures.length} test failures`);
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,329 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
// @ts-ignore
|
||||
import { createPatch } from 'diff';
|
||||
|
||||
jest.mock('fs', () => {
|
||||
const realFs = jest.requireActual('fs');
|
||||
return {
|
||||
readFile: realFs.read,
|
||||
writeFile: (...args: any[]) => {
|
||||
setTimeout(args[args.length - 1], 0);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { FTR_REPORT, JEST_REPORT, MOCHA_REPORT, KARMA_REPORT } from './__fixtures__';
|
||||
import { parseTestReport } from './test_report';
|
||||
import { mentionGithubIssuesInReport } from './mention_github_issues_in_report';
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
const log = new ToolingLog();
|
||||
|
||||
it('rewrites ftr reports with minimal changes', async () => {
|
||||
const xml = await mentionGithubIssuesInReport(
|
||||
await parseTestReport(FTR_REPORT),
|
||||
[
|
||||
{
|
||||
name: 'maps app maps loaded from sample data ecommerce "before all" hook',
|
||||
classname:
|
||||
'Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js',
|
||||
message: 'foo bar',
|
||||
},
|
||||
],
|
||||
log,
|
||||
Path.resolve(__dirname, './__fixtures__/ftr_report.xml')
|
||||
);
|
||||
|
||||
expect(createPatch('ftr.xml', FTR_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
|
||||
"Index: ftr.xml
|
||||
===================================================================
|
||||
--- ftr.xml [object Object]
|
||||
+++ ftr.xml
|
||||
@@ -2,44 +2,48 @@
|
||||
<testsuites>
|
||||
<testsuite timestamp=\\"2019-06-05T23:37:10\\" time=\\"903.670\\" tests=\\"129\\" failures=\\"5\\" skipped=\\"71\\">
|
||||
<testcase name=\\"maps app maps loaded from sample data ecommerce "before all" hook\\" classname=\\"Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js\\" time=\\"154.378\\">
|
||||
<system-out>
|
||||
- <![CDATA[[00:00:00] │
|
||||
+ [00:00:00] │
|
||||
[00:07:04] └-: maps app
|
||||
...
|
||||
[00:15:02] │
|
||||
-]]>
|
||||
+
|
||||
</system-out>
|
||||
<failure>
|
||||
- <![CDATA[Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
|
||||
+ Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
|
||||
Wait timed out after 10055ms
|
||||
at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
|
||||
at process._tickCallback (internal/process/next_tick.js:68:7)
|
||||
at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
|
||||
- at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)]]>
|
||||
- </failure>
|
||||
+ at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
|
||||
+
|
||||
+
|
||||
+Failed Tests Reporter:
|
||||
+ - foo bar
|
||||
+</failure>
|
||||
</testcase>
|
||||
<testcase name=\\"maps app "after all" hook\\" classname=\\"Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps\\" time=\\"0.179\\">
|
||||
<system-out>
|
||||
- <![CDATA[[00:00:00] │
|
||||
+ [00:00:00] │
|
||||
[00:07:04] └-: maps app
|
||||
...
|
||||
-]]>
|
||||
+
|
||||
</system-out>
|
||||
<failure>
|
||||
- <![CDATA[{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
|
||||
+ { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
|
||||
at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
|
||||
at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12)
|
||||
- at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]>
|
||||
+ at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name=\\"InfraOps app feature controls infrastructure security global infrastructure all privileges shows infrastructure navlink\\" classname=\\"Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/infra/feature_controls/infrastructure_security·ts\\">
|
||||
<system-out>
|
||||
- <![CDATA[[00:00:00] │
|
||||
+ [00:00:00] │
|
||||
[00:05:13] └-: InfraOps app
|
||||
...
|
||||
-]]>
|
||||
+
|
||||
</system-out>
|
||||
<skipped/>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
-</testsuites>
|
||||
+</testsuites>
|
||||
\\\\ No newline at end of file
|
||||
"
|
||||
`);
|
||||
});
|
||||
|
||||
it('rewrites jest reports with minimal changes', async () => {
|
||||
const xml = await mentionGithubIssuesInReport(
|
||||
await parseTestReport(JEST_REPORT),
|
||||
[
|
||||
{
|
||||
classname: 'X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp',
|
||||
name: 'launcher can reconnect if process died',
|
||||
message: 'foo bar',
|
||||
},
|
||||
],
|
||||
log,
|
||||
Path.resolve(__dirname, './__fixtures__/jest_report.xml')
|
||||
);
|
||||
|
||||
expect(createPatch('jest.xml', JEST_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
|
||||
"Index: jest.xml
|
||||
===================================================================
|
||||
--- jest.xml [object Object]
|
||||
+++ jest.xml
|
||||
@@ -3,13 +3,17 @@
|
||||
<testsuite name=\\"x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts\\" timestamp=\\"2019-06-07T03:42:21\\" time=\\"14.504\\" tests=\\"5\\" failures=\\"1\\" skipped=\\"0\\" file=\\"/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts\\">
|
||||
<testcase classname=\\"X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp\\" name=\\"launcher can start and end a process\\" time=\\"1.316\\"/>
|
||||
<testcase classname=\\"X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp\\" name=\\"launcher can force kill the process if langServer can not exit\\" time=\\"3.182\\"/>
|
||||
<testcase classname=\\"X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp\\" name=\\"launcher can reconnect if process died\\" time=\\"7.060\\">
|
||||
- <failure>
|
||||
- <![CDATA[TypeError: Cannot read property '0' of undefined
|
||||
- at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)]]>
|
||||
- </failure>
|
||||
+ <failure><![CDATA[
|
||||
+ TypeError: Cannot read property '0' of undefined
|
||||
+ at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
|
||||
+
|
||||
+
|
||||
+Failed Tests Reporter:
|
||||
+ - foo bar
|
||||
+]]></failure>
|
||||
</testcase>
|
||||
<testcase classname=\\"X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp\\" name=\\"passive launcher can start and end a process\\" time=\\"0.435\\"/>
|
||||
<testcase classname=\\"X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp\\" name=\\"passive launcher should restart a process if a process died before connected\\" time=\\"1.502\\"/>
|
||||
</testsuite>
|
||||
-</testsuites>
|
||||
+</testsuites>
|
||||
\\\\ No newline at end of file
|
||||
"
|
||||
`);
|
||||
});
|
||||
|
||||
it('rewrites mocha reports with minimal changes', async () => {
|
||||
const xml = await mentionGithubIssuesInReport(
|
||||
await parseTestReport(MOCHA_REPORT),
|
||||
[
|
||||
{
|
||||
name: 'code in multiple nodes "before all" hook',
|
||||
classname: 'X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts',
|
||||
message: 'foo bar',
|
||||
},
|
||||
],
|
||||
log,
|
||||
Path.resolve(__dirname, './__fixtures__/mocha_report.xml')
|
||||
);
|
||||
|
||||
expect(createPatch('mocha.xml', MOCHA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
|
||||
"Index: mocha.xml
|
||||
===================================================================
|
||||
--- mocha.xml [object Object]
|
||||
+++ mocha.xml
|
||||
@@ -2,12 +2,12 @@
|
||||
<testsuites>
|
||||
<testsuite timestamp=\\"2019-06-13T23:29:36\\" time=\\"30.739\\" tests=\\"1444\\" failures=\\"2\\" skipped=\\"3\\">
|
||||
<testcase name=\\"code in multiple nodes "before all" hook\\" classname=\\"X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts\\" time=\\"0.121\\">
|
||||
<system-out>
|
||||
- <![CDATA[]]>
|
||||
+
|
||||
</system-out>
|
||||
- <failure>
|
||||
- <![CDATA[Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
|
||||
+ <failure><![CDATA[
|
||||
+ Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
|
||||
<html>
|
||||
<head><title>503 Service Temporarily Unavailable</title></head>
|
||||
<body bgcolor=\\"white\\">
|
||||
<center><h1>503 Service Temporarily Unavailable</h1></center>
|
||||
@@ -15,24 +15,28 @@
|
||||
</body>
|
||||
</html>
|
||||
|
||||
at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13)
|
||||
- at process._tickCallback (internal/process/next_tick.js:68:7)]]>
|
||||
- </failure>
|
||||
+ at process._tickCallback (internal/process/next_tick.js:68:7)
|
||||
+
|
||||
+
|
||||
+Failed Tests Reporter:
|
||||
+ - foo bar
|
||||
+]]></failure>
|
||||
</testcase>
|
||||
<testcase name=\\"code in multiple nodes "after all" hook\\" classname=\\"X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts\\" time=\\"0.003\\">
|
||||
<system-out>
|
||||
- <![CDATA[]]>
|
||||
+
|
||||
</system-out>
|
||||
<failure>
|
||||
- <![CDATA[TypeError: Cannot read property 'shutdown' of undefined
|
||||
+ TypeError: Cannot read property 'shutdown' of undefined
|
||||
at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
|
||||
- at process.topLevelDomainCallback (domain.js:120:23)]]>
|
||||
+ at process.topLevelDomainCallback (domain.js:120:23)
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name=\\"repository service test can not clone a repo by ssh without a key\\" classname=\\"X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/repository_service·ts\\" time=\\"0.005\\">
|
||||
<system-out>
|
||||
- <![CDATA[]]>
|
||||
+
|
||||
</system-out>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
-</testsuites>
|
||||
+</testsuites>
|
||||
\\\\ No newline at end of file
|
||||
"
|
||||
`);
|
||||
});
|
||||
|
||||
it('rewrites karma reports with minimal changes', async () => {
|
||||
const xml = await mentionGithubIssuesInReport(
|
||||
await parseTestReport(KARMA_REPORT),
|
||||
[
|
||||
{
|
||||
name:
|
||||
'CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK',
|
||||
classname: 'Browser Unit Tests.CoordinateMapsVisualizationTest',
|
||||
message: 'foo bar',
|
||||
},
|
||||
],
|
||||
log,
|
||||
Path.resolve(__dirname, './__fixtures__/karma_report.xml')
|
||||
);
|
||||
|
||||
expect(createPatch('karma.xml', KARMA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
|
||||
"Index: karma.xml
|
||||
===================================================================
|
||||
--- karma.xml [object Object]
|
||||
+++ karma.xml
|
||||
@@ -1,5 +1,5 @@
|
||||
-<?xml version=\\"1.0\\"?>
|
||||
+<?xml version=\\"1.0\\" encoding=\\"utf-8\\"?>
|
||||
<testsuite name=\\"Chrome 75.0.3770 (Mac OS X 10.14.5)\\" package=\\"\\" timestamp=\\"2019-07-02T19:53:21\\" id=\\"0\\" hostname=\\"spalger.lan\\" tests=\\"648\\" errors=\\"0\\" failures=\\"4\\" time=\\"1.759\\">
|
||||
<properties>
|
||||
<property name=\\"browser.fullName\\" value=\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36\\"/>
|
||||
</properties>
|
||||
@@ -7,27 +7,31 @@
|
||||
<testcase name=\\"Vis-Editor-Agg-Params plugin directive should hide custom label parameter\\" time=\\"0\\" classname=\\"Browser Unit Tests.Vis-Editor-Agg-Params plugin directive\\">
|
||||
<skipped/>
|
||||
</testcase>
|
||||
<testcase name=\\"CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK\\" time=\\"0.265\\" classname=\\"Browser Unit Tests.CoordinateMapsVisualizationTest\\">
|
||||
- <failure type=\\"\\">Error: expected 7069 to be below 64
|
||||
- at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11)
|
||||
- at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8)
|
||||
- at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15)
|
||||
- at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60)
|
||||
+ <failure type=\\"\\"><![CDATA[Error: expected 7069 to be below 64
|
||||
+ at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11)
|
||||
+ at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8)
|
||||
+ at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15)
|
||||
+ at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60)
|
||||
at tryCatch (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:62:40)
|
||||
at Generator.invoke [as _invoke] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:288:22)
|
||||
- at Generator.prototype.<computed> [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
|
||||
- at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
|
||||
- at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
|
||||
-</failure>
|
||||
+ at Generator.prototype.<computed> [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
|
||||
+ at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
|
||||
+ at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
|
||||
+
|
||||
+
|
||||
+Failed Tests Reporter:
|
||||
+ - foo bar
|
||||
+]]></failure>
|
||||
</testcase>
|
||||
<testcase name=\\"CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should toggle to Heatmap OK\\" time=\\"0.055\\" classname=\\"Browser Unit Tests.CoordinateMapsVisualizationTest\\"/>
|
||||
<testcase name=\\"VegaParser._parseSchema should warn on vega-lite version too new to be supported\\" time=\\"0.001\\" classname=\\"Browser Unit Tests.VegaParser·_parseSchema\\"/>
|
||||
<system-out>
|
||||
- <![CDATA[Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4'
|
||||
+ Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4'
|
||||
,Chrome 75.0.3770 (Mac OS X 10.14.5) WARN: 'Unmatched GET to http://localhost:9876/api/interpreter/fns'
|
||||
...
|
||||
|
||||
-]]>
|
||||
+
|
||||
</system-out>
|
||||
<system-err/>
|
||||
-</testsuite>
|
||||
+</testsuite>
|
||||
\\\\ No newline at end of file
|
||||
"
|
||||
`);
|
||||
});
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import Fs from 'fs';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
import xml2js from 'xml2js';
|
||||
|
||||
import { TestReport, makeFailedTestCaseIter } from './test_report';
|
||||
|
||||
const writeAsync = promisify(Fs.writeFile);
|
||||
|
||||
export interface Update {
|
||||
classname: string;
|
||||
name: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutate the report to include mentions of Github issues related to test failures,
|
||||
* then write the updated report to disk
|
||||
*/
|
||||
export async function mentionGithubIssuesInReport(
|
||||
report: TestReport,
|
||||
updates: Update[],
|
||||
log: ToolingLog,
|
||||
reportPath: string
|
||||
) {
|
||||
for (const testCase of makeFailedTestCaseIter(report)) {
|
||||
const { classname, name } = testCase.$;
|
||||
const messageList = updates
|
||||
.filter((u: Update) => u.classname === classname && u.name === name)
|
||||
.reduce((acc, u) => `${acc}\n - ${u.message}`, '');
|
||||
|
||||
if (!messageList) {
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(`${classname} - ${name}:${messageList}`);
|
||||
const append = `\n\nFailed Tests Reporter:${messageList}\n`;
|
||||
|
||||
if (
|
||||
testCase.failure[0] &&
|
||||
typeof testCase.failure[0] === 'object' &&
|
||||
typeof testCase.failure[0]._ === 'string'
|
||||
) {
|
||||
testCase.failure[0]._ += append;
|
||||
} else {
|
||||
testCase.failure[0] = String(testCase.failure[0]) + append;
|
||||
}
|
||||
}
|
||||
|
||||
const builder = new xml2js.Builder({
|
||||
cdata: true,
|
||||
xmldec: { version: '1.0', encoding: 'utf-8' },
|
||||
});
|
||||
const xml = builder.buildObject(report);
|
||||
await writeAsync(reportPath, xml, 'utf8');
|
||||
return xml;
|
||||
}
|
||||
|
|
@ -20,7 +20,7 @@
|
|||
import dedent from 'dedent';
|
||||
import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/dev-utils';
|
||||
|
||||
import { createFailureIssue, updatedFailureIssue } from './report_failure';
|
||||
import { createFailureIssue, updateFailureIssue } from './report_failure';
|
||||
|
||||
jest.mock('./github_api');
|
||||
const { GithubApi } = jest.requireMock('./github_api');
|
||||
|
|
@ -80,7 +80,7 @@ describe('createFailureIssue()', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('updatedFailureIssue()', () => {
|
||||
describe('updateFailureIssue()', () => {
|
||||
it('increments failure count and adds new comment to issue', async () => {
|
||||
const log = new ToolingLog();
|
||||
const writer = new ToolingLogCollectingWriter();
|
||||
|
|
@ -88,7 +88,7 @@ describe('updatedFailureIssue()', () => {
|
|||
|
||||
const api = new GithubApi();
|
||||
|
||||
await updatedFailureIssue(
|
||||
await updateFailureIssue(
|
||||
'https://build-url',
|
||||
{
|
||||
html_url: 'https://github.com/issues/1234',
|
||||
|
|
|
|||
|
|
@ -17,18 +17,11 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
|
||||
import { TestFailure } from './get_failures';
|
||||
import { GithubIssue, GithubApi } from './github_api';
|
||||
import { getIssueMetadata, updateIssueMetadata } from './issue_metadata';
|
||||
|
||||
export async function createFailureIssue(
|
||||
buildUrl: string,
|
||||
failure: TestFailure,
|
||||
log: ToolingLog,
|
||||
api: GithubApi
|
||||
) {
|
||||
export async function createFailureIssue(buildUrl: string, failure: TestFailure, api: GithubApi) {
|
||||
const title = `Failing test: ${failure.classname} - ${failure.name}`;
|
||||
|
||||
const body = updateIssueMetadata(
|
||||
|
|
@ -48,16 +41,10 @@ export async function createFailureIssue(
|
|||
}
|
||||
);
|
||||
|
||||
const newIssueUrl = await api.createIssue(title, body, ['failed-test']);
|
||||
log.info(`Created issue ${newIssueUrl}`);
|
||||
return await api.createIssue(title, body, ['failed-test']);
|
||||
}
|
||||
|
||||
export async function updatedFailureIssue(
|
||||
buildUrl: string,
|
||||
issue: GithubIssue,
|
||||
log: ToolingLog,
|
||||
api: GithubApi
|
||||
) {
|
||||
export async function updateFailureIssue(buildUrl: string, issue: GithubIssue, api: GithubApi) {
|
||||
// Increment failCount
|
||||
const newCount = getIssueMetadata(issue.body, 'test.failCount', 0) + 1;
|
||||
const newBody = updateIssueMetadata(issue.body, {
|
||||
|
|
@ -67,5 +54,5 @@ export async function updatedFailureIssue(
|
|||
await api.editIssueBodyAndEnsureOpen(issue.number, newBody);
|
||||
await api.addIssueComment(issue.number, `New failure: [Jenkins Build](${buildUrl})`);
|
||||
|
||||
log.info(`Updated issue ${issue.html_url}, failCount: ${newCount}`);
|
||||
return newCount;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,8 +22,10 @@ import globby from 'globby';
|
|||
|
||||
import { getFailures } from './get_failures';
|
||||
import { GithubApi } from './github_api';
|
||||
import { updatedFailureIssue, createFailureIssue } from './report_failure';
|
||||
import { updateFailureIssue, createFailureIssue } from './report_failure';
|
||||
import { getIssueMetadata } from './issue_metadata';
|
||||
import { readTestReport } from './test_report';
|
||||
import { mentionGithubIssuesInReport, Update } from './mention_github_issues_in_report';
|
||||
|
||||
export function runFailedTestsReporterCli() {
|
||||
run(
|
||||
|
|
@ -67,7 +69,19 @@ export function runFailedTestsReporterCli() {
|
|||
});
|
||||
|
||||
for (const reportPath of reportPaths) {
|
||||
for (const failure of await getFailures(log, reportPath)) {
|
||||
const report = await readTestReport(reportPath);
|
||||
const updates: Update[] = [];
|
||||
|
||||
for (const { failure, type } of await getFailures(report)) {
|
||||
if (type === 'ignore') {
|
||||
updates.push({
|
||||
classname: failure.classname,
|
||||
name: failure.name,
|
||||
message: 'Failure is likely irrelevant, so an issue was not created or updated',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const existingIssue = await githubApi.findFailedTestIssue(
|
||||
i =>
|
||||
getIssueMetadata(i.body, 'test.class') === failure.classname &&
|
||||
|
|
@ -75,11 +89,25 @@ export function runFailedTestsReporterCli() {
|
|||
);
|
||||
|
||||
if (existingIssue) {
|
||||
await updatedFailureIssue(buildUrl, existingIssue, log, githubApi);
|
||||
} else {
|
||||
await createFailureIssue(buildUrl, failure, log, githubApi);
|
||||
const newFailureCount = await updateFailureIssue(buildUrl, existingIssue, githubApi);
|
||||
updates.push({
|
||||
classname: failure.classname,
|
||||
name: failure.name,
|
||||
message: `Updated existing issue: ${existingIssue.html_url}, (fail count: ${newFailureCount})`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const newIssueUrl = await createFailureIssue(buildUrl, failure, githubApi);
|
||||
updates.push({
|
||||
classname: failure.classname,
|
||||
name: failure.name,
|
||||
message: `Created new issue: ${newIssueUrl}`,
|
||||
});
|
||||
}
|
||||
|
||||
// mutates report to have mentions of updates made and writes updated report to disk
|
||||
await mentionGithubIssuesInReport(report, updates, log, reportPath);
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
|
|||
100
packages/kbn-test/src/failed_tests_reporter/test_report.ts
Normal file
100
packages/kbn-test/src/failed_tests_reporter/test_report.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import Fs from 'fs';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import xml2js from 'xml2js';
|
||||
|
||||
const readAsync = promisify(Fs.readFile);
|
||||
|
||||
export type TestReport =
|
||||
| {
|
||||
testsuites: {
|
||||
testsuite: TestSuite[];
|
||||
};
|
||||
}
|
||||
| {
|
||||
testsuite: TestSuite;
|
||||
};
|
||||
|
||||
export interface TestSuite {
|
||||
$: {
|
||||
/* ISO8601 timetamp when test suite ran */
|
||||
timestamp: string;
|
||||
/* number of second this tests suite took */
|
||||
time: string;
|
||||
/* number of tests as a string */
|
||||
tests: string;
|
||||
/* number of failed tests as a string */
|
||||
failures: string;
|
||||
/* number of skipped tests as a string */
|
||||
skipped: string;
|
||||
};
|
||||
testcase: TestCase[];
|
||||
}
|
||||
|
||||
export interface TestCase {
|
||||
$: {
|
||||
/* unique test name */
|
||||
name: string;
|
||||
/* somewhat human readable combination of test name and file */
|
||||
classname: string;
|
||||
/* number of seconds this test took */
|
||||
time: string;
|
||||
};
|
||||
/* contents of system-out elements */
|
||||
'system-out'?: string[];
|
||||
/* contents of failure elements */
|
||||
failure?: Array<string | { _: string }>;
|
||||
/* contents of skipped elements */
|
||||
skipped?: string[];
|
||||
}
|
||||
|
||||
export interface FailedTestCase extends TestCase {
|
||||
failure: Array<string | { _: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JUnit XML Files
|
||||
*/
|
||||
export async function parseTestReport(xml: string): Promise<TestReport> {
|
||||
return await xml2js.parseStringPromise(xml);
|
||||
}
|
||||
|
||||
export async function readTestReport(testReportPath: string) {
|
||||
return await parseTestReport(await readAsync(testReportPath, 'utf8'));
|
||||
}
|
||||
|
||||
export function* makeFailedTestCaseIter(report: TestReport) {
|
||||
// Grab the failures. Reporters may report multiple testsuites in a single file.
|
||||
const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite];
|
||||
|
||||
for (const testSuite of testSuites) {
|
||||
for (const testCase of testSuite.testcase) {
|
||||
const { failure } = testCase;
|
||||
|
||||
if (!failure) {
|
||||
continue;
|
||||
}
|
||||
|
||||
yield testCase as FailedTestCase;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue