github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[failedTestReporter] send github api request counts to ci-stats (#120684) (#120719)

Browse source 
Co-authored-by: Spencer <email@spalger.com>
This commit is contained in:
Kibana Machine 2021-12-08 04:24:26 -05:00 • committed by GitHub
parent ad352b0fd4
commit 0bf3ea83f2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 209 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

38
packages/kbn-test/src/failed_tests_reporter/buildkite_metadata.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export interface BuildkiteMetadata {
buildId?: string;
jobId?: string;
url?: string;
jobName?: string;
jobUrl?: string;
}
export function getBuildkiteMetadata(): BuildkiteMetadata {
// Buildkite steps that use `parallelism` need a numerical suffix added to identify them
// We should also increment the number by one, since it's 0-based
const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB
? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}`
: '';
const buildUrl = process.env.BUILDKITE_BUILD_URL;
const jobUrl = process.env.BUILDKITE_JOB_ID
? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}`
: undefined;
return {
buildId: process.env.BUJILDKITE_BUILD_ID,
jobId: process.env.BUILDKITE_JOB_ID,
url: buildUrl,
jobUrl,
jobName: process.env.BUILDKITE_LABEL
? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}`
: undefined,
};
}

78
packages/kbn-test/src/failed_tests_reporter/github_api.ts
View file

@ -42,6 +42,7 @@ export class GithubApi {
private readonly token: string | undefined;
private readonly dryRun: boolean;
private readonly x: AxiosInstance;
private requestCount: number = 0;
/**
* Create a GithubApi helper object, if token is undefined requests won't be
@ -68,6 +69,10 @@ export class GithubApi {
});
}
getRequestCount() {
return this.requestCount;
}
private failedTestIssuesPageCache: {
pages: GithubIssue[][];
nextRequest: RequestOptions | undefined;
@ -191,53 +196,50 @@ export class GithubApi {
}> {
const executeRequest = !this.dryRun || options.safeForDryRun;
const maxAttempts = options.maxAttempts || 5;
const attempt = options.attempt || 1;
this.log.verbose('Github API', executeRequest ? 'Request' : 'Dry Run', options);
let attempt = 0;
while (true) {
attempt += 1;
this.log.verbose('Github API', executeRequest ? 'Request' : 'Dry Run', options);
if (!executeRequest) {
return {
status: 200,
statusText: 'OK',
headers: {},
data: dryRunResponse,
};
}
if (!executeRequest) {
return {
status: 200,
statusText: 'OK',
headers: {},
data: dryRunResponse,
};
}
try {
return await this.x.request<T>(options);
} catch (error) {
const unableToReachGithub = isAxiosRequestError(error);
const githubApiFailed = isAxiosResponseError(error) && error.response.status >= 500;
const errorResponseLog =
isAxiosResponseError(error) &&
`[${error.config.method} ${error.config.url}] ${error.response.status} ${error.response.statusText} Error`;
try {
this.requestCount += 1;
return await this.x.request<T>(options);
} catch (error) {
const unableToReachGithub = isAxiosRequestError(error);
const githubApiFailed = isAxiosResponseError(error) && error.response.status >= 500;
const errorResponseLog =
isAxiosResponseError(error) &&
`[${error.config.method} ${error.config.url}] ${error.response.status} ${error.response.statusText} Error`;
if ((unableToReachGithub || githubApiFailed) && attempt < maxAttempts) {
const waitMs = 1000 * attempt;
if ((unableToReachGithub || githubApiFailed) && attempt < maxAttempts) {
const waitMs = 1000 * attempt;
if (errorResponseLog) {
this.log.error(`${errorResponseLog}: waiting ${waitMs}ms to retry`);
} else {
this.log.error(`Unable to reach github, waiting ${waitMs}ms to retry`);
if (errorResponseLog) {
this.log.error(`${errorResponseLog}: waiting ${waitMs}ms to retry`);
} else {
this.log.error(`Unable to reach github, waiting ${waitMs}ms to retry`);
}
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
await new Promise((resolve) => setTimeout(resolve, waitMs));
return await this.request<T>(
{
...options,
maxAttempts,
attempt: attempt + 1,
},
dryRunResponse
);
}
if (errorResponseLog) {
throw new Error(`${errorResponseLog}: ${JSON.stringify(error.response.data)}`);
}
if (errorResponseLog) {
throw new Error(`${errorResponseLog}: ${JSON.stringify(error.response.data)}`);
throw error;
}
throw error;
}
}
}

34
packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts
View file

@ -14,6 +14,7 @@ import { ToolingLog } from '@kbn/dev-utils';
import { REPO_ROOT } from '@kbn/utils';
import { escape } from 'he';
import { BuildkiteMetadata } from './buildkite_metadata';
import { TestFailure } from './get_failures';
const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
@ -37,7 +38,11 @@ const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
return allScreenshots;
};
export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
export function reportFailuresToFile(
log: ToolingLog,
failures: TestFailure[],
bkMeta: BuildkiteMetadata
) {
if (!failures?.length) {
return;
}
@ -76,28 +81,15 @@ export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
.flat()
.join('\n');
// Buildkite steps that use `parallelism` need a numerical suffix added to identify them
// We should also increment the number by one, since it's 0-based
const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB
? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}`
: '';
const buildUrl = process.env.BUILDKITE_BUILD_URL || '';
const jobUrl = process.env.BUILDKITE_JOB_ID
? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}`
: '';
const failureJSON = JSON.stringify(
{
...failure,
hash,
buildId: process.env.BUJILDKITE_BUILD_ID || '',
jobId: process.env.BUILDKITE_JOB_ID || '',
url: buildUrl,
jobUrl,
jobName: process.env.BUILDKITE_LABEL
? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}`
: '',
buildId: bkMeta.buildId,
jobId: bkMeta.jobId,
url: bkMeta.url,
jobUrl: bkMeta.jobUrl,
jobName: bkMeta.jobName,
},
null,
2
@ -149,11 +141,11 @@ export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
</small>
</p>
${
jobUrl
bkMeta.jobUrl
? `<p>
<small>
<strong>Buildkite Job</strong><br />
<a href="${escape(jobUrl)}">${escape(jobUrl)}</a>
<a href="${escape(bkMeta.jobUrl)}">${escape(bkMeta.jobUrl)}</a>
</small>
</p>`
: ''

216
packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts
View file

@ -9,7 +9,7 @@
import Path from 'path';
import { REPO_ROOT } from '@kbn/utils';
import { run, createFailError, createFlagError } from '@kbn/dev-utils';
import { run, createFailError, createFlagError, CiStatsReporter } from '@kbn/dev-utils';
import globby from 'globby';
import normalize from 'normalize-path';
@ -22,6 +22,7 @@ import { addMessagesToReport } from './add_messages_to_report';
import { getReportMessageIter } from './report_metadata';
import { reportFailuresToEs } from './report_failures_to_es';
import { reportFailuresToFile } from './report_failures_to_file';
import { getBuildkiteMetadata } from './buildkite_metadata';
const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
@ -71,108 +72,127 @@ export function runFailedTestsReporterCli() {
dryRun: !updateGithub,
});
const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
if (typeof buildUrl !== 'string' || !buildUrl) {
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
}
const bkMeta = getBuildkiteMetadata();
const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
normalize(Path.resolve(p))
);
log.info('Searching for reports at', patterns);
const reportPaths = await globby(patterns, {
absolute: true,
});
if (!reportPaths.length) {
throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
}
log.info('found', reportPaths.length, 'junit reports', reportPaths);
const newlyCreatedIssues: Array<{
failure: TestFailure;
newIssue: GithubIssueMini;
}> = [];
for (const reportPath of reportPaths) {
const report = await readTestReport(reportPath);
const messages = Array.from(getReportMessageIter(report));
const failures = await getFailures(report);
if (indexInEs) {
await reportFailuresToEs(log, failures);
try {
const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
if (typeof buildUrl !== 'string' || !buildUrl) {
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
}
for (const failure of failures) {
const pushMessage = (msg: string) => {
messages.push({
classname: failure.classname,
name: failure.name,
message: msg,
});
};
if (failure.likelyIrrelevant) {
pushMessage(
'Failure is likely irrelevant' +
(updateGithub ? ', so an issue was not created or updated' : '')
);
continue;
}
let existingIssue: GithubIssueMini | undefined = await githubApi.findFailedTestIssue(
(i) =>
getIssueMetadata(i.body, 'test.class') === failure.classname &&
getIssueMetadata(i.body, 'test.name') === failure.name
);
if (!existingIssue) {
const newlyCreated = newlyCreatedIssues.find(
({ failure: f }) => f.classname === failure.classname && f.name === failure.name
);
if (newlyCreated) {
existingIssue = newlyCreated.newIssue;
}
}
if (existingIssue) {
const newFailureCount = await updateFailureIssue(
buildUrl,
existingIssue,
githubApi,
branch
);
const url = existingIssue.html_url;
failure.githubIssue = url;
failure.failureCount = updateGithub ? newFailureCount : newFailureCount - 1;
pushMessage(`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}`);
if (updateGithub) {
pushMessage(`Updated existing issue: ${url} (fail count: ${newFailureCount})`);
}
continue;
}
const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
pushMessage('Test has not failed recently on tracked branches');
if (updateGithub) {
pushMessage(`Created new issue: ${newIssue.html_url}`);
failure.githubIssue = newIssue.html_url;
}
newlyCreatedIssues.push({ failure, newIssue });
failure.failureCount = updateGithub ? 1 : 0;
}
// mutates report to include messages and writes updated report to disk
await addMessagesToReport({
report,
messages,
log,
reportPath,
dryRun: !flags['report-update'],
const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
normalize(Path.resolve(p))
);
log.info('Searching for reports at', patterns);
const reportPaths = await globby(patterns, {
absolute: true,
});
reportFailuresToFile(log, failures);
if (!reportPaths.length) {
throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
}
log.info('found', reportPaths.length, 'junit reports', reportPaths);
const newlyCreatedIssues: Array<{
failure: TestFailure;
newIssue: GithubIssueMini;
}> = [];
for (const reportPath of reportPaths) {
const report = await readTestReport(reportPath);
const messages = Array.from(getReportMessageIter(report));
const failures = await getFailures(report);
if (indexInEs) {
await reportFailuresToEs(log, failures);
}
for (const failure of failures) {
const pushMessage = (msg: string) => {
messages.push({
classname: failure.classname,
name: failure.name,
message: msg,
});
};
if (failure.likelyIrrelevant) {
pushMessage(
'Failure is likely irrelevant' +
(updateGithub ? ', so an issue was not created or updated' : '')
);
continue;
}
let existingIssue: GithubIssueMini | undefined = await githubApi.findFailedTestIssue(
(i) =>
getIssueMetadata(i.body, 'test.class') === failure.classname &&
getIssueMetadata(i.body, 'test.name') === failure.name
);
if (!existingIssue) {
const newlyCreated = newlyCreatedIssues.find(
({ failure: f }) => f.classname === failure.classname && f.name === failure.name
);
if (newlyCreated) {
existingIssue = newlyCreated.newIssue;
}
}
if (existingIssue) {
const newFailureCount = await updateFailureIssue(
buildUrl,
existingIssue,
githubApi,
branch
);
const url = existingIssue.html_url;
failure.githubIssue = url;
failure.failureCount = updateGithub ? newFailureCount : newFailureCount - 1;
pushMessage(
`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}`
);
if (updateGithub) {
pushMessage(`Updated existing issue: ${url} (fail count: ${newFailureCount})`);
}
continue;
}
const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
pushMessage('Test has not failed recently on tracked branches');
if (updateGithub) {
pushMessage(`Created new issue: ${newIssue.html_url}`);
failure.githubIssue = newIssue.html_url;
}
newlyCreatedIssues.push({ failure, newIssue });
failure.failureCount = updateGithub ? 1 : 0;
}
// mutates report to include messages and writes updated report to disk
await addMessagesToReport({
report,
messages,
log,
reportPath,
dryRun: !flags['report-update'],
});
reportFailuresToFile(log, failures, bkMeta);
}
} finally {
await CiStatsReporter.fromEnv(log).metrics([
{
group: 'github api request count',
id: `failed test reporter`,
value: githubApi.getRequestCount(),
meta: Object.fromEntries(
Object.entries(bkMeta).map(
([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const
)
),
},
]);
}
},
{