mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[failedTestReporter] send github api request counts to ci-stats (#120684)
This commit is contained in:
Spencer
GitHub
parent
82979a345f
commit
78372d14b8
4 changed files with 209 additions and 157 deletions
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export interface BuildkiteMetadata {
|
||||
buildId?: string;
|
||||
jobId?: string;
|
||||
url?: string;
|
||||
jobName?: string;
|
||||
jobUrl?: string;
|
||||
}
|
||||
|
||||
export function getBuildkiteMetadata(): BuildkiteMetadata {
|
||||
// Buildkite steps that use `parallelism` need a numerical suffix added to identify them
|
||||
// We should also increment the number by one, since it's 0-based
|
||||
const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB
|
||||
? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}`
|
||||
: '';
|
||||
|
||||
const buildUrl = process.env.BUILDKITE_BUILD_URL;
|
||||
const jobUrl = process.env.BUILDKITE_JOB_ID
|
||||
? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}`
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
buildId: process.env.BUJILDKITE_BUILD_ID,
|
||||
jobId: process.env.BUILDKITE_JOB_ID,
|
||||
url: buildUrl,
|
||||
jobUrl,
|
||||
jobName: process.env.BUILDKITE_LABEL
|
||||
? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}`
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
|
@ -42,6 +42,7 @@ export class GithubApi {
|
|||
private readonly token: string | undefined;
|
||||
private readonly dryRun: boolean;
|
||||
private readonly x: AxiosInstance;
|
||||
private requestCount: number = 0;
|
||||
|
||||
/**
|
||||
* Create a GithubApi helper object, if token is undefined requests won't be
|
||||
|
|
@ -68,6 +69,10 @@ export class GithubApi {
|
|||
});
|
||||
}
|
||||
|
||||
getRequestCount() {
|
||||
return this.requestCount;
|
||||
}
|
||||
|
||||
private failedTestIssuesPageCache: {
|
||||
pages: GithubIssue[][];
|
||||
nextRequest: RequestOptions | undefined;
|
||||
|
|
@ -191,53 +196,50 @@ export class GithubApi {
|
|||
}> {
|
||||
const executeRequest = !this.dryRun || options.safeForDryRun;
|
||||
const maxAttempts = options.maxAttempts || 5;
|
||||
const attempt = options.attempt || 1;
|
||||
|
||||
this.log.verbose('Github API', executeRequest ? 'Request' : 'Dry Run', options);
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
attempt += 1;
|
||||
this.log.verbose('Github API', executeRequest ? 'Request' : 'Dry Run', options);
|
||||
|
||||
if (!executeRequest) {
|
||||
return {
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
headers: {},
|
||||
data: dryRunResponse,
|
||||
};
|
||||
}
|
||||
if (!executeRequest) {
|
||||
return {
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
headers: {},
|
||||
data: dryRunResponse,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.x.request<T>(options);
|
||||
} catch (error) {
|
||||
const unableToReachGithub = isAxiosRequestError(error);
|
||||
const githubApiFailed = isAxiosResponseError(error) && error.response.status >= 500;
|
||||
const errorResponseLog =
|
||||
isAxiosResponseError(error) &&
|
||||
`[${error.config.method} ${error.config.url}] ${error.response.status} ${error.response.statusText} Error`;
|
||||
try {
|
||||
this.requestCount += 1;
|
||||
return await this.x.request<T>(options);
|
||||
} catch (error) {
|
||||
const unableToReachGithub = isAxiosRequestError(error);
|
||||
const githubApiFailed = isAxiosResponseError(error) && error.response.status >= 500;
|
||||
const errorResponseLog =
|
||||
isAxiosResponseError(error) &&
|
||||
`[${error.config.method} ${error.config.url}] ${error.response.status} ${error.response.statusText} Error`;
|
||||
|
||||
if ((unableToReachGithub || githubApiFailed) && attempt < maxAttempts) {
|
||||
const waitMs = 1000 * attempt;
|
||||
if ((unableToReachGithub || githubApiFailed) && attempt < maxAttempts) {
|
||||
const waitMs = 1000 * attempt;
|
||||
|
||||
if (errorResponseLog) {
|
||||
this.log.error(`${errorResponseLog}: waiting ${waitMs}ms to retry`);
|
||||
} else {
|
||||
this.log.error(`Unable to reach github, waiting ${waitMs}ms to retry`);
|
||||
if (errorResponseLog) {
|
||||
this.log.error(`${errorResponseLog}: waiting ${waitMs}ms to retry`);
|
||||
} else {
|
||||
this.log.error(`Unable to reach github, waiting ${waitMs}ms to retry`);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
return await this.request<T>(
|
||||
{
|
||||
...options,
|
||||
maxAttempts,
|
||||
attempt: attempt + 1,
|
||||
},
|
||||
dryRunResponse
|
||||
);
|
||||
}
|
||||
if (errorResponseLog) {
|
||||
throw new Error(`${errorResponseLog}: ${JSON.stringify(error.response.data)}`);
|
||||
}
|
||||
|
||||
if (errorResponseLog) {
|
||||
throw new Error(`${errorResponseLog}: ${JSON.stringify(error.response.data)}`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { ToolingLog } from '@kbn/dev-utils';
|
|||
import { REPO_ROOT } from '@kbn/utils';
|
||||
import { escape } from 'he';
|
||||
|
||||
import { BuildkiteMetadata } from './buildkite_metadata';
|
||||
import { TestFailure } from './get_failures';
|
||||
|
||||
const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
|
||||
|
|
@ -37,7 +38,11 @@ const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
|
|||
return allScreenshots;
|
||||
};
|
||||
|
||||
export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
|
||||
export function reportFailuresToFile(
|
||||
log: ToolingLog,
|
||||
failures: TestFailure[],
|
||||
bkMeta: BuildkiteMetadata
|
||||
) {
|
||||
if (!failures?.length) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -76,28 +81,15 @@ export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
|
|||
.flat()
|
||||
.join('\n');
|
||||
|
||||
// Buildkite steps that use `parallelism` need a numerical suffix added to identify them
|
||||
// We should also increment the number by one, since it's 0-based
|
||||
const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB
|
||||
? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}`
|
||||
: '';
|
||||
|
||||
const buildUrl = process.env.BUILDKITE_BUILD_URL || '';
|
||||
const jobUrl = process.env.BUILDKITE_JOB_ID
|
||||
? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}`
|
||||
: '';
|
||||
|
||||
const failureJSON = JSON.stringify(
|
||||
{
|
||||
...failure,
|
||||
hash,
|
||||
buildId: process.env.BUJILDKITE_BUILD_ID || '',
|
||||
jobId: process.env.BUILDKITE_JOB_ID || '',
|
||||
url: buildUrl,
|
||||
jobUrl,
|
||||
jobName: process.env.BUILDKITE_LABEL
|
||||
? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}`
|
||||
: '',
|
||||
buildId: bkMeta.buildId,
|
||||
jobId: bkMeta.jobId,
|
||||
url: bkMeta.url,
|
||||
jobUrl: bkMeta.jobUrl,
|
||||
jobName: bkMeta.jobName,
|
||||
},
|
||||
null,
|
||||
2
|
||||
|
|
@ -149,11 +141,11 @@ export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
|
|||
</small>
|
||||
</p>
|
||||
${
|
||||
jobUrl
|
||||
bkMeta.jobUrl
|
||||
? `<p>
|
||||
<small>
|
||||
<strong>Buildkite Job</strong><br />
|
||||
<a href="${escape(jobUrl)}">${escape(jobUrl)}</a>
|
||||
<a href="${escape(bkMeta.jobUrl)}">${escape(bkMeta.jobUrl)}</a>
|
||||
</small>
|
||||
</p>`
|
||||
: ''
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
import Path from 'path';
|
||||
|
||||
import { REPO_ROOT } from '@kbn/utils';
|
||||
import { run, createFailError, createFlagError } from '@kbn/dev-utils';
|
||||
import { run, createFailError, createFlagError, CiStatsReporter } from '@kbn/dev-utils';
|
||||
import globby from 'globby';
|
||||
import normalize from 'normalize-path';
|
||||
|
||||
|
|
@ -22,6 +22,7 @@ import { addMessagesToReport } from './add_messages_to_report';
|
|||
import { getReportMessageIter } from './report_metadata';
|
||||
import { reportFailuresToEs } from './report_failures_to_es';
|
||||
import { reportFailuresToFile } from './report_failures_to_file';
|
||||
import { getBuildkiteMetadata } from './buildkite_metadata';
|
||||
|
||||
const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
|
||||
|
||||
|
|
@ -71,108 +72,127 @@ export function runFailedTestsReporterCli() {
|
|||
dryRun: !updateGithub,
|
||||
});
|
||||
|
||||
const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
|
||||
if (typeof buildUrl !== 'string' || !buildUrl) {
|
||||
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
|
||||
}
|
||||
const bkMeta = getBuildkiteMetadata();
|
||||
|
||||
const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
|
||||
normalize(Path.resolve(p))
|
||||
);
|
||||
log.info('Searching for reports at', patterns);
|
||||
const reportPaths = await globby(patterns, {
|
||||
absolute: true,
|
||||
});
|
||||
|
||||
if (!reportPaths.length) {
|
||||
throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
|
||||
}
|
||||
|
||||
log.info('found', reportPaths.length, 'junit reports', reportPaths);
|
||||
const newlyCreatedIssues: Array<{
|
||||
failure: TestFailure;
|
||||
newIssue: GithubIssueMini;
|
||||
}> = [];
|
||||
|
||||
for (const reportPath of reportPaths) {
|
||||
const report = await readTestReport(reportPath);
|
||||
const messages = Array.from(getReportMessageIter(report));
|
||||
const failures = await getFailures(report);
|
||||
|
||||
if (indexInEs) {
|
||||
await reportFailuresToEs(log, failures);
|
||||
try {
|
||||
const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
|
||||
if (typeof buildUrl !== 'string' || !buildUrl) {
|
||||
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
|
||||
}
|
||||
|
||||
for (const failure of failures) {
|
||||
const pushMessage = (msg: string) => {
|
||||
messages.push({
|
||||
classname: failure.classname,
|
||||
name: failure.name,
|
||||
message: msg,
|
||||
});
|
||||
};
|
||||
|
||||
if (failure.likelyIrrelevant) {
|
||||
pushMessage(
|
||||
'Failure is likely irrelevant' +
|
||||
(updateGithub ? ', so an issue was not created or updated' : '')
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let existingIssue: GithubIssueMini | undefined = await githubApi.findFailedTestIssue(
|
||||
(i) =>
|
||||
getIssueMetadata(i.body, 'test.class') === failure.classname &&
|
||||
getIssueMetadata(i.body, 'test.name') === failure.name
|
||||
);
|
||||
|
||||
if (!existingIssue) {
|
||||
const newlyCreated = newlyCreatedIssues.find(
|
||||
({ failure: f }) => f.classname === failure.classname && f.name === failure.name
|
||||
);
|
||||
|
||||
if (newlyCreated) {
|
||||
existingIssue = newlyCreated.newIssue;
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIssue) {
|
||||
const newFailureCount = await updateFailureIssue(
|
||||
buildUrl,
|
||||
existingIssue,
|
||||
githubApi,
|
||||
branch
|
||||
);
|
||||
const url = existingIssue.html_url;
|
||||
failure.githubIssue = url;
|
||||
failure.failureCount = updateGithub ? newFailureCount : newFailureCount - 1;
|
||||
pushMessage(`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}`);
|
||||
if (updateGithub) {
|
||||
pushMessage(`Updated existing issue: ${url} (fail count: ${newFailureCount})`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
|
||||
pushMessage('Test has not failed recently on tracked branches');
|
||||
if (updateGithub) {
|
||||
pushMessage(`Created new issue: ${newIssue.html_url}`);
|
||||
failure.githubIssue = newIssue.html_url;
|
||||
}
|
||||
newlyCreatedIssues.push({ failure, newIssue });
|
||||
failure.failureCount = updateGithub ? 1 : 0;
|
||||
}
|
||||
|
||||
// mutates report to include messages and writes updated report to disk
|
||||
await addMessagesToReport({
|
||||
report,
|
||||
messages,
|
||||
log,
|
||||
reportPath,
|
||||
dryRun: !flags['report-update'],
|
||||
const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) =>
|
||||
normalize(Path.resolve(p))
|
||||
);
|
||||
log.info('Searching for reports at', patterns);
|
||||
const reportPaths = await globby(patterns, {
|
||||
absolute: true,
|
||||
});
|
||||
|
||||
reportFailuresToFile(log, failures);
|
||||
if (!reportPaths.length) {
|
||||
throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
|
||||
}
|
||||
|
||||
log.info('found', reportPaths.length, 'junit reports', reportPaths);
|
||||
const newlyCreatedIssues: Array<{
|
||||
failure: TestFailure;
|
||||
newIssue: GithubIssueMini;
|
||||
}> = [];
|
||||
|
||||
for (const reportPath of reportPaths) {
|
||||
const report = await readTestReport(reportPath);
|
||||
const messages = Array.from(getReportMessageIter(report));
|
||||
const failures = await getFailures(report);
|
||||
|
||||
if (indexInEs) {
|
||||
await reportFailuresToEs(log, failures);
|
||||
}
|
||||
|
||||
for (const failure of failures) {
|
||||
const pushMessage = (msg: string) => {
|
||||
messages.push({
|
||||
classname: failure.classname,
|
||||
name: failure.name,
|
||||
message: msg,
|
||||
});
|
||||
};
|
||||
|
||||
if (failure.likelyIrrelevant) {
|
||||
pushMessage(
|
||||
'Failure is likely irrelevant' +
|
||||
(updateGithub ? ', so an issue was not created or updated' : '')
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let existingIssue: GithubIssueMini | undefined = await githubApi.findFailedTestIssue(
|
||||
(i) =>
|
||||
getIssueMetadata(i.body, 'test.class') === failure.classname &&
|
||||
getIssueMetadata(i.body, 'test.name') === failure.name
|
||||
);
|
||||
|
||||
if (!existingIssue) {
|
||||
const newlyCreated = newlyCreatedIssues.find(
|
||||
({ failure: f }) => f.classname === failure.classname && f.name === failure.name
|
||||
);
|
||||
|
||||
if (newlyCreated) {
|
||||
existingIssue = newlyCreated.newIssue;
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIssue) {
|
||||
const newFailureCount = await updateFailureIssue(
|
||||
buildUrl,
|
||||
existingIssue,
|
||||
githubApi,
|
||||
branch
|
||||
);
|
||||
const url = existingIssue.html_url;
|
||||
failure.githubIssue = url;
|
||||
failure.failureCount = updateGithub ? newFailureCount : newFailureCount - 1;
|
||||
pushMessage(
|
||||
`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}`
|
||||
);
|
||||
if (updateGithub) {
|
||||
pushMessage(`Updated existing issue: ${url} (fail count: ${newFailureCount})`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch);
|
||||
pushMessage('Test has not failed recently on tracked branches');
|
||||
if (updateGithub) {
|
||||
pushMessage(`Created new issue: ${newIssue.html_url}`);
|
||||
failure.githubIssue = newIssue.html_url;
|
||||
}
|
||||
newlyCreatedIssues.push({ failure, newIssue });
|
||||
failure.failureCount = updateGithub ? 1 : 0;
|
||||
}
|
||||
|
||||
// mutates report to include messages and writes updated report to disk
|
||||
await addMessagesToReport({
|
||||
report,
|
||||
messages,
|
||||
log,
|
||||
reportPath,
|
||||
dryRun: !flags['report-update'],
|
||||
});
|
||||
|
||||
reportFailuresToFile(log, failures, bkMeta);
|
||||
}
|
||||
} finally {
|
||||
await CiStatsReporter.fromEnv(log).metrics([
|
||||
{
|
||||
group: 'github api request count',
|
||||
id: `failed test reporter`,
|
||||
value: githubApi.getRequestCount(),
|
||||
meta: Object.fromEntries(
|
||||
Object.entries(bkMeta).map(
|
||||
([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const
|
||||
)
|
||||
),
|
||||
},
|
||||
]);
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue