github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[kbn-scout-reporting] add failed test reporter (#205096)

Browse source 
## Summary

Extending scout-reporter with `failed-test-reporter`, that saves
failures in json summary file. For each test failure html report file is
generated and linked in summary report:
```
[
  {
    "name": "stateful - Discover app - saved searches - should customize time range on dashboards",
    "htmlReportFilename": "c51fcf067a95b48e2bbf6098a90ab14.html"
  },
  {
    "name": "stateful - Discover app - value suggestions: useTimeRange enabled - dont show up if outside of range",
    "htmlReportFilename": "9622dcc1ac732f30e82ad6d20d7eeaa.html"
  }
]
```

This PR updates `failed_tests_reporter_cli` to look for potential Scout
test failures and re-generate test failure artifacts in the same format
we already use for FTR ones.

These new artifacts are used to list failures in BK annotation: 

<img width="1092" alt="image"
src="https://github.com/user-attachments/assets/09464c55-cdaa-45a4-ab47-c5f0375b701c"
/>


test failure html report example: 
<img width="1072" alt="image"
src="https://github.com/user-attachments/assets/81f6e475-1435-445d-82eb-ecf5253c42d3"
/>

Note for reviewer: 3 Scout + 1 FTR tests were "broken" to show/test
reporter, those changes must be reverted before merge. See failed
pipeline
[here](https://buildkite.com/elastic/kibana-pull-request/builds/266822)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dzmitry Lemechko 2025-01-15 19:40:34 +01:00 • committed by GitHub
parent 31cbf2980f
commit 1023402f8d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 1426 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/scripts/lifecycle/post_command.sh
View file

@ -10,6 +10,7 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_
if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
echo "--- Upload Artifacts"
buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*/**/*'
buildkite-agent artifact upload 'target/junit/**/*'
buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*'
buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'

137
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts
View file

@ -15,7 +15,6 @@ import { createFailError, createFlagError } from '@kbn/dev-cli-errors';
import { CiStatsReporter } from '@kbn/ci-stats-reporter';
import globby from 'globby';
import normalize from 'normalize-path';
import { getFailures } from './get_failures';
import { GithubApi } from './github_api';
import { updateFailureIssue, createFailureIssue } from './report_failure';
@ -26,6 +25,7 @@ import { reportFailuresToEs } from './report_failures_to_es';
import { reportFailuresToFile } from './report_failures_to_file';
import { getBuildkiteMetadata } from './buildkite_metadata';
import { ExistingFailedTestIssues } from './existing_failed_test_issues';
import { generateScoutTestFailureArtifacts } from './generate_scout_test_failure_artifacts';
const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
const DISABLE_MISSING_TEST_REPORT_ERRORS =
@ -101,88 +101,89 @@ run(
return;
}
if (!reportPaths.length) {
throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
}
// Scout test failures reporting
await generateScoutTestFailureArtifacts({ log, bkMeta });
log.info('found', reportPaths.length, 'junit reports', reportPaths);
if (reportPaths.length) {
log.info('found', reportPaths.length, 'junit reports', reportPaths);
const existingIssues = new ExistingFailedTestIssues(log);
for (const reportPath of reportPaths) {
const report = await readTestReport(reportPath);
const messages = Array.from(getReportMessageIter(report));
const failures = getFailures(report);
const existingIssues = new ExistingFailedTestIssues(log);
for (const reportPath of reportPaths) {
const report = await readTestReport(reportPath);
const messages = Array.from(getReportMessageIter(report));
const failures = getFailures(report);
await existingIssues.loadForFailures(failures);
await existingIssues.loadForFailures(failures);
if (indexInEs) {
await reportFailuresToEs(log, failures);
}
for (const failure of failures) {
const pushMessage = (msg: string) => {
messages.push({
classname: failure.classname,
name: failure.name,
message: msg,
});
};
if (failure.likelyIrrelevant) {
pushMessage(
'Failure is likely irrelevant' +
(updateGithub ? ', so an issue was not created or updated' : '')
);
continue;
if (indexInEs) {
await reportFailuresToEs(log, failures);
}
const existingIssue = existingIssues.getForFailure(failure);
if (existingIssue) {
const { newBody, newCount } = await updateFailureIssue(
for (const failure of failures) {
const pushMessage = (msg: string) => {
messages.push({
classname: failure.classname,
name: failure.name,
message: msg,
});
};
if (failure.likelyIrrelevant) {
pushMessage(
'Failure is likely irrelevant' +
(updateGithub ? ', so an issue was not created or updated' : '')
);
continue;
}
const existingIssue = existingIssues.getForFailure(failure);
if (existingIssue) {
const { newBody, newCount } = await updateFailureIssue(
buildUrl,
existingIssue,
githubApi,
branch,
pipeline
);
const url = existingIssue.github.htmlUrl;
existingIssue.github.body = newBody;
failure.githubIssue = url;
failure.failureCount = updateGithub ? newCount : newCount - 1;
pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`);
if (updateGithub) {
pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`);
}
continue;
}
const newIssue = await createFailureIssue(
buildUrl,
existingIssue,
failure,
githubApi,
branch,
pipeline
pipeline,
prependTitle
);
const url = existingIssue.github.htmlUrl;
existingIssue.github.body = newBody;
failure.githubIssue = url;
failure.failureCount = updateGithub ? newCount : newCount - 1;
pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`);
existingIssues.addNewlyCreated(failure, newIssue);
pushMessage('Test has not failed recently on tracked branches');
if (updateGithub) {
pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`);
pushMessage(`Created new issue: ${newIssue.html_url}`);
failure.githubIssue = newIssue.html_url;
}
continue;
failure.failureCount = updateGithub ? 1 : 0;
}
const newIssue = await createFailureIssue(
buildUrl,
failure,
githubApi,
branch,
pipeline,
prependTitle
);
existingIssues.addNewlyCreated(failure, newIssue);
pushMessage('Test has not failed recently on tracked branches');
if (updateGithub) {
pushMessage(`Created new issue: ${newIssue.html_url}`);
failure.githubIssue = newIssue.html_url;
}
failure.failureCount = updateGithub ? 1 : 0;
// mutates report to include messages and writes updated report to disk
await addMessagesToReport({
report,
messages,
log,
reportPath,
dryRun: !flags['report-update'],
});
await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report));
}
// mutates report to include messages and writes updated report to disk
await addMessagesToReport({
report,
messages,
log,
reportPath,
dryRun: !flags['report-update'],
});
await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report));
}
} finally {
await CiStatsReporter.fromEnv(log).metrics([

79
packages/kbn-failed-test-reporter-cli/failed_tests_reporter/generate_scout_test_failure_artifacts.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import Path from 'path';
import globby from 'globby';
import fs from 'fs';
import { createHash } from 'crypto';
import { ToolingLog } from '@kbn/tooling-log';
import { BuildkiteMetadata } from './buildkite_metadata';
const SCOUT_TEST_FAILURE_DIR_PATTERN = '.scout/reports/scout-playwright-test-failures-*';
const SUMMARY_REPORT_FILENAME = 'test-failures-summary.json';
export async function generateScoutTestFailureArtifacts({
log,
bkMeta,
}: {
log: ToolingLog;
bkMeta: BuildkiteMetadata;
}) {
log.info('Searching for Scout test failure reports');
const dirs = await globby(SCOUT_TEST_FAILURE_DIR_PATTERN, {
onlyDirectories: true,
});
if (dirs.length === 0) {
log.info(`No directories found matching pattern: ${SCOUT_TEST_FAILURE_DIR_PATTERN}`);
return;
}
log.info(`Found ${dirs.length} directories matching pattern: ${SCOUT_TEST_FAILURE_DIR_PATTERN}`);
for (const dirPath of dirs) {
const summaryFilePath = Path.join(dirPath, SUMMARY_REPORT_FILENAME);
// Check if summary JSON exists
if (!fs.existsSync(summaryFilePath)) {
throw new Error(`Summary file not found in: ${dirPath}`);
}
const summaryData: Array<{ name: string; htmlReportFilename: string }> = JSON.parse(
fs.readFileSync(summaryFilePath, 'utf-8')
);
log.info(`Creating failure artifacts for report in ${dirPath}`);
for (const { name, htmlReportFilename } of summaryData) {
const htmlFilePath = Path.join(dirPath, htmlReportFilename);
const failureHTML = fs.readFileSync(htmlFilePath, 'utf-8');
const hash = createHash('md5').update(name).digest('hex'); // eslint-disable-line @kbn/eslint/no_unsafe_hash
const filenameBase = `${
process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : ''
}${hash}`;
const dir = Path.join('target', 'test_failures');
const failureJSON = JSON.stringify(
{
name,
hash,
buildId: bkMeta.buildId,
jobId: bkMeta.jobId,
url: bkMeta.url,
jobUrl: bkMeta.jobUrl,
jobName: bkMeta.jobName,
},
null,
2
);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8');
fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8');
}
}
}

2
packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts
View file

@ -8,7 +8,7 @@
*/
import { Command } from '@kbn/dev-cli-runner';
import { ScoutReportDataStream } from '../reporting/report';
import { ScoutReportDataStream } from '../reporting/report/events';
import { getValidatedESClient } from './common';
export const initializeReportDatastream: Command<void> = {

2
packages/kbn-scout-reporting/src/cli/upload_events.ts
View file

@ -10,7 +10,7 @@
import fs from 'node:fs';
import { Command } from '@kbn/dev-cli-runner';
import { createFlagError } from '@kbn/dev-cli-errors';
import { ScoutReportDataStream } from '../reporting/report';
import { ScoutReportDataStream } from '../reporting/report/events';
import { getValidatedESClient } from './common';
export const uploadEvents: Command<void> = {

114
packages/kbn-scout-reporting/src/helpers/cli_processing.test.ts Normal file
View file

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { getRunTarget, stripRunCommand } from './cli_processing';
describe('cli_processing', () => {
describe('stripRunCommand', () => {
it(`should return the correct run command when started with 'npx'`, () => {
const argv = ['npx', 'playwright', 'test', '--config', 'path/to/config', '--grep=@svlSearch'];
expect(stripRunCommand(argv)).toBe(
'npx playwright test --config path/to/config --grep=@svlSearch'
);
});
it(`should return the correct run command when started with 'node'`, () => {
const argv = [
'/Users/user/.nvm/versions/node/v20.15.1/bin/node',
'node_modules/.bin/playwright',
'test',
'--config',
'path/to/config',
'--grep=@svlSearch',
];
expect(stripRunCommand(argv)).toBe(
'npx playwright test --config path/to/config --grep=@svlSearch'
);
});
it(`should throw error if command has less than 3 arguments`, () => {
const argv = [
'/Users/user/.nvm/versions/node/v20.15.1/bin/node',
'node_modules/.bin/playwright',
];
expect(() => stripRunCommand(argv)).toThrow(
/Invalid command arguments: must include at least 'npx playwright test'/
);
});
it(`should throw error if command does not start with 'node' or 'npx'`, () => {
const argv = [
'node_modules/.bin/playwright',
'test',
'--config',
'path/to/config',
'--grep=@svlSearch',
];
expect(() => stripRunCommand(argv)).toThrow(
/Invalid command structure: Expected "node <playwright_path> test" or "npx playwright test"/
);
});
});
describe('getRunTarget', () => {
it(`should return the correct mode for '--grep=@svlSearch'`, () => {
const argv = [
'node',
'scripts/scout.js',
'run-tests',
'--config',
'path/to/config',
'--grep=@svlSearch',
];
expect(getRunTarget(argv)).toBe('serverless-search');
});
it(`should return the correct mode for '--grep @svlSearch'`, () => {
const argv = [
'node',
'scripts/scout.js',
'run-tests',
'--config',
'path/to/config',
'--grep',
'@svlSearch',
];
expect(getRunTarget(argv)).toBe('serverless-search');
});
it(`should return 'undefined' for an invalid --grep tag`, () => {
const argv = [
'node',
'scripts/scout.js',
'run-tests',
'--config',
'path/to/config',
'--grep=@invalidTag',
];
expect(getRunTarget(argv)).toBe('undefined');
});
it(`should return 'undefined' if --grep argument is not provided`, () => {
const argv = ['node', 'scripts/scout.js'];
expect(getRunTarget(argv)).toBe('undefined');
});
it(`should return 'undefined' for '--grep='`, () => {
const argv = ['node', 'scripts/scout.js', '--grep='];
expect(getRunTarget(argv)).toBe('undefined');
});
it(`should return 'undefined' if '--grep' argument is without value`, () => {
const argv = ['node', 'scripts/scout.js', '--grep'];
expect(getRunTarget(argv)).toBe('undefined');
});
});
});

47
packages/kbn-scout-reporting/src/helpers/cli_processing.ts Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export const stripRunCommand = (commandArgs: string[]): string => {
if (!Array.isArray(commandArgs) || commandArgs.length < 3) {
throw new Error(`Invalid command arguments: must include at least 'npx playwright test'`);
}
const isNodeCommand = commandArgs[0].endsWith('node');
const isNpxCommand = commandArgs[0] === 'npx' && commandArgs[1] === 'playwright';
if (!isNodeCommand && !isNpxCommand) {
throw new Error(
'Invalid command structure: Expected "node <playwright_path> test" or "npx playwright test".'
);
}
const restArgs = commandArgs.slice(2);
// Rebuild the command with only valid arguments
return `npx playwright ${restArgs.join(' ')}`;
};
export function getRunTarget(argv: string[] = process.argv): string {
const tagsToMode: Record<string, string> = {
'@ess': 'stateful',
'@svlSearch': 'serverless-search',
'@svlOblt': 'serverless-oblt',
'@svlSecurity': 'serverless-security',
};
const grepIndex = argv.findIndex((arg) => arg === '--grep' || arg.startsWith('--grep='));
if (grepIndex !== -1) {
const tag = argv[grepIndex].startsWith('--grep=')
? argv[grepIndex].split('=')[1]
: argv[grepIndex + 1] || ''; // Look at the next argument if '--grep' is used without `=`
return tagsToMode[tag] || 'undefined';
}
return 'undefined';
}

13
packages/kbn-scout-reporting/src/helpers/index.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { getPluginManifestData, type PluginManifest } from './plugin_manifest';
export { stripFilePath, parseStdout } from './text_processing';
export { getRunTarget, stripRunCommand } from './cli_processing';
export { getTestIDForTitle, generateTestRunId } from './test_id_generator';

122
packages/kbn-scout-reporting/src/helpers/plugin_manifest.test.ts Normal file
View file

@ -0,0 +1,122 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import path from 'path';
import fs from 'fs';
import { getManifestPath, readPluginManifest, getPluginManifestData } from './plugin_manifest';
jest.mock('fs');
describe('plugin_manifest', () => {
describe('getManifestPath', () => {
it('should resolve the manifest path correctly for a valid config path', () => {
const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts';
const expectedPath = path.resolve('/plugins/my_plugin/kibana.jsonc');
expect(getManifestPath(configPath)).toBe(expectedPath);
});
it(`should throw an error if 'ui_tests' is not in the path`, () => {
const configPath = '/plugins/my_plugin/tests/playwright.config.ts';
expect(() => getManifestPath(configPath)).toThrow(
/Invalid path: "ui_tests" directory not found/
);
});
});
describe('readPluginManifest', () => {
const filePath = '/plugins/my_plugin/kibana.jsonc';
beforeEach(() => {
jest.clearAllMocks();
});
it('should read and parse the manifest file correctly', () => {
const fileContent = `
{
"id": "my_plugin",
"group": "platform",
"visibility": "private",
"owner": ["team"],
"plugin": { "id": "my_plugin" }
}
`;
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
jest.spyOn(fs, 'readFileSync').mockReturnValue(fileContent);
const result = readPluginManifest(filePath);
expect(result).toEqual({
id: 'my_plugin',
group: 'platform',
visibility: 'private',
owner: ['team'],
plugin: { id: 'my_plugin' },
});
});
it('should throw an error if the file does not exist', () => {
jest.spyOn(fs, 'existsSync').mockReturnValue(false);
expect(() => readPluginManifest(filePath)).toThrow(/Manifest file not found/);
});
it('should throw an error if the file cannot be read', () => {
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
jest.spyOn(fs, 'readFileSync').mockImplementation(() => {
throw new Error('File read error');
});
expect(() => readPluginManifest(filePath)).toThrow(/Failed to read manifest file/);
});
it('should throw an error for invalid JSON content', () => {
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
jest.spyOn(fs, 'readFileSync').mockReturnValue('{ invalid json }');
expect(() => readPluginManifest(filePath)).toThrow(/Invalid JSON format in manifest file/);
});
it('should throw an error for missing required fields', () => {
const fileContent = `{
"group": "platform",
"visibility": "public"
}`;
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
jest.spyOn(fs, 'readFileSync').mockReturnValue(fileContent);
expect(() => readPluginManifest(filePath)).toThrow(/Invalid manifest structure/);
});
});
describe('getPluginManifestData', () => {
const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts';
const manifestContent = `
{
"id": "my_plugin",
"group": "platform",
"visibility": "public",
"owner": ["team"],
"plugin": { "id": "my_plugin" }
}
`;
beforeEach(() => {
jest.clearAllMocks();
});
it('should resolve and parse the manifest data correctly', () => {
jest.spyOn(fs, 'existsSync').mockReturnValue(true);
jest.spyOn(fs, 'readFileSync').mockReturnValue(manifestContent);
const result = getPluginManifestData(configPath);
expect(result).toEqual({
id: 'my_plugin',
group: 'platform',
visibility: 'public',
owner: ['team'],
plugin: { id: 'my_plugin' },
});
});
});
});

86
packages/kbn-scout-reporting/src/helpers/plugin_manifest.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import path from 'path';
import fs from 'fs';
import { Jsonc } from '@kbn/repo-packages';
export interface PluginManifest {
id: string;
group: string;
owner: string[];
visibility: string;
plugin: { id: string };
}
/**
* Resolves the path to the `kibana.jsonc` manifest based on the Playwright configuration file path.
* @param configPath - Absolute path to the Playwright configuration file.
* @returns Absolute path to the `kibana.jsonc` file.
* @throws Error if `ui_tests` is not found in the path.
*/
export const getManifestPath = (configPath: string): string => {
const pathSegments = configPath.split(path.sep);
const testDirIndex = pathSegments.indexOf('ui_tests');
if (testDirIndex === -1) {
throw new Error(
`Invalid path: "ui_tests" directory not found in ${configPath}.
Ensure playwright configuration file is in the plugin directory: '/plugins/<plugin-name>/ui_tests/<config-file>'`
);
}
const manifestSegments = pathSegments.slice(0, testDirIndex).concat('kibana.jsonc');
return path.resolve('/', ...manifestSegments); // Ensure absolute path
};
/**
* Reads and parses the `kibana.jsonc` manifest file.
* @param filePath - Absolute path to the `kibana.jsonc` file.
* @returns Parsed `PluginManifest` object.
* @throws Error if the file does not exist, cannot be read, or is invalid.
*/
export const readPluginManifest = (filePath: string): PluginManifest => {
if (!fs.existsSync(filePath)) {
throw new Error(`Manifest file not found: ${filePath}`);
}
let fileContent: string;
try {
fileContent = fs.readFileSync(filePath, 'utf8');
} catch (error) {
throw new Error(`Failed to read manifest file at ${filePath}: ${error.message}`);
}
let manifest: Partial<PluginManifest>;
try {
manifest = Jsonc.parse(fileContent) as Partial<PluginManifest>;
} catch (error) {
throw new Error(`Invalid JSON format in manifest file at ${filePath}: ${error.message}`);
}
const { id, group, visibility, owner, plugin } = manifest;
if (!id || !group || !visibility || !plugin?.id) {
throw new Error(
`Invalid manifest structure at ${filePath}. Expected required fields: id, group, visibility, plugin.id`
);
}
return { id, group, visibility, owner: owner || [], plugin };
};
/**
* Resolves the plugin manifest file path and reads its content.
* @param configPath - Absolute path to the Playwright configuration file in the plugin directory.
* @returns Parsed `PluginManifest` object.
*/
export const getPluginManifestData = (configPath: string): PluginManifest => {
const manifestPath = getManifestPath(configPath);
return readPluginManifest(manifestPath);
};

18
packages/kbn-scout-reporting/src/helpers/test_id_generator.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { createHash, randomBytes } from 'node:crypto';
export function generateTestRunId() {
return randomBytes(8).toString('hex');
}
export function getTestIDForTitle(title: string) {
return createHash('sha256').update(title).digest('hex').slice(0, 31);
}

64
packages/kbn-scout-reporting/src/helpers/text_processing.test.ts Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import stripANSI from 'strip-ansi';
import { parseStdout } from './text_processing';
jest.mock('strip-ansi', () => jest.fn((input) => input.replace(/\x1b\[[0-9;]*m/g, '')));
describe('text_processing', () => {
describe('parseStdout', () => {
it('should concatenate multiple strings and strip ANSI codes', () => {
const stdout = ['Line 1 with ANSI \x1b[31mred\x1b[0m text', '\nLine 2 plain text'];
const result = parseStdout(stdout);
expect(stripANSI).toHaveBeenCalledWith(
'Line 1 with ANSI \x1b[31mred\x1b[0m text\nLine 2 plain text'
);
expect(result).toBe('Line 1 with ANSI red text\nLine 2 plain text');
});
it('should concatenate multiple buffers and strip ANSI codes', () => {
const stdout = [
Buffer.from('Buffer line 1 with ANSI \x1b[32mgreen\x1b[0m text'),
Buffer.from('\nBuffer line 2 plain text'),
];
const result = parseStdout(stdout);
expect(stripANSI).toHaveBeenCalledWith(
'Buffer line 1 with ANSI \x1b[32mgreen\x1b[0m text\nBuffer line 2 plain text'
);
expect(result).toBe('Buffer line 1 with ANSI green text\nBuffer line 2 plain text');
});
it('should handle an empty array and return an empty string', () => {
const stdout: Array<string | Buffer> = [];
const result = parseStdout(stdout);
expect(stripANSI).toHaveBeenCalledWith('');
expect(result).toBe('');
});
it('should handle an array with only an empty string', () => {
const stdout = [''];
const result = parseStdout(stdout);
expect(stripANSI).toHaveBeenCalledWith('');
expect(result).toBe('');
});
it('should handle an array with only an empty buffer', () => {
const stdout = [Buffer.from('')];
const result = parseStdout(stdout);
expect(stripANSI).toHaveBeenCalledWith('');
expect(result).toBe('');
});
});
});

23
packages/kbn-scout-reporting/src/helpers/text_processing.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import stripANSI from 'strip-ansi';
import { REPO_ROOT } from '@kbn/repo-info';
export const stripFilePath = (filePath: string): string =>
stripANSI(filePath.replaceAll(`${REPO_ROOT}/`, ''));
export function parseStdout(stdout: Array<string | Buffer>): string {
const stdoutContent = stdout
.map((chunk) => (Buffer.isBuffer(chunk) ? chunk.toString() : chunk))
.join('');
// Escape special HTML characters
return stripANSI(stdoutContent);
}

25
packages/kbn-scout-reporting/src/reporting/index.ts
View file

@ -7,24 +7,23 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { createHash, randomBytes } from 'node:crypto';
import type { ReporterDescription } from 'playwright/test';
import type { ScoutPlaywrightReporterOptions } from './playwright';
import { ScoutPlaywrightReporterOptions } from './playwright/scout_playwright_reporter';
export * from './report';
// ID helpers
export function generateTestRunId() {
return randomBytes(8).toString('hex');
}
export function getTestIDForTitle(title: string) {
return createHash('sha256').update(title).digest('hex').slice(0, 31);
}
// Playwright reporting
// Playwright event-based reporting
export const scoutPlaywrightReporter = (
options?: ScoutPlaywrightReporterOptions
): ReporterDescription => {
return ['@kbn/scout-reporting/src/reporting/playwright.ts', options];
return ['@kbn/scout-reporting/src/reporting/playwright/events', options];
};
// Playwright failed test reporting
export const scoutFailedTestsReporter = (
options?: ScoutPlaywrightReporterOptions
): ReporterDescription => {
return ['@kbn/scout-reporting/src/reporting/playwright/failed_test', options];
};
export { generateTestRunId, getTestIDForTitle } from '../helpers';

12
packages/kbn-scout-reporting/src/reporting/playwright/events/index.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { ScoutPlaywrightReporter } from './playwright_reporter';
// eslint-disable-next-line import/no-default-export
export default ScoutPlaywrightReporter;

23
packages/kbn-scout-reporting/src/reporting/playwright.ts → packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts
View file

@ -28,16 +28,10 @@ import {
getCodeOwnersEntries,
getOwningTeamsForPath,
} from '@kbn/code-owners';
import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '.';
import { environmentMetadata } from '../datasources';
/**
* Configuration options for the Scout Playwright reporter
*/
export interface ScoutPlaywrightReporterOptions {
name?: string;
outputPath?: string;
}
import { ScoutEventsReport, ScoutReportEventAction } from '../../report';
import { environmentMetadata } from '../../../datasources';
import type { ScoutPlaywrightReporterOptions } from '../scout_playwright_reporter';
import { generateTestRunId, getTestIDForTitle } from '../../../helpers';
/**
* Scout Playwright reporter
@ -46,7 +40,7 @@ export class ScoutPlaywrightReporter implements Reporter {
readonly log: ToolingLog;
readonly name: string;
readonly runId: string;
private report: ScoutReport;
private report: ScoutEventsReport;
private readonly codeOwnersEntries: CodeOwnersEntry[];
constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) {
@ -56,10 +50,10 @@ export class ScoutPlaywrightReporter implements Reporter {
});
this.name = this.reporterOptions.name || 'unknown';
this.runId = generateTestRunId();
this.runId = this.reporterOptions.runId || generateTestRunId();
this.log.info(`Scout test run ID: ${this.runId}`);
this.report = new ScoutReport(this.log);
this.report = new ScoutEventsReport(this.log);
this.codeOwnersEntries = getCodeOwnersEntries();
}
@ -290,6 +284,3 @@ export class ScoutPlaywrightReporter implements Reporter {
});
}
}
// eslint-disable-next-line import/no-default-export
export default ScoutPlaywrightReporter;

128
packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts Normal file
View file

@ -0,0 +1,128 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type {
FullConfig,
FullResult,
Reporter,
Suite,
TestCase,
TestResult,
} from '@playwright/test/reporter';
import path from 'node:path';
import { ToolingLog } from '@kbn/tooling-log';
import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info';
import { REPO_ROOT } from '@kbn/repo-info';
import {
type CodeOwnersEntry,
getCodeOwnersEntries,
getOwningTeamsForPath,
} from '@kbn/code-owners';
import type { TestFailure } from '../../report';
import { ScoutFailureReport } from '../../report';
import type { ScoutPlaywrightReporterOptions } from '../scout_playwright_reporter';
import {
getRunTarget,
getPluginManifestData,
parseStdout,
generateTestRunId,
getTestIDForTitle,
stripRunCommand,
stripFilePath,
} from '../../../helpers';
/**
* Scout Failed Test reporter
*/
export class ScoutFailedTestReporter implements Reporter {
private readonly log: ToolingLog;
private readonly runId: string;
private readonly codeOwnersEntries: CodeOwnersEntry[];
private readonly report: ScoutFailureReport;
private target: string;
private plugin: TestFailure['plugin'];
private command: string;
constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) {
this.log = new ToolingLog({
level: 'info',
writeTo: process.stdout,
});
this.report = new ScoutFailureReport(this.log);
this.codeOwnersEntries = getCodeOwnersEntries();
this.runId = this.reporterOptions.runId || generateTestRunId();
this.target = 'undefined'; // when '--grep' is not provided in the command line
this.command = stripRunCommand(process.argv);
}
private getFileOwners(filePath: string): string[] {
return getOwningTeamsForPath(filePath, this.codeOwnersEntries);
}
public get reportRootPath(): string {
const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT;
return path.join(outputPath, `scout-playwright-test-failures-${this.runId}`);
}
printsToStdio(): boolean {
return false; // Avoid taking over console output
}
onBegin(config: FullConfig, suite: Suite) {
// Get plugin metadata from kibana.jsonc
if (config.configFile) {
const metadata = getPluginManifestData(config.configFile);
this.plugin = {
id: metadata.plugin.id,
visibility: metadata.visibility,
group: metadata.group,
};
}
this.target = getRunTarget();
}
onTestEnd(test: TestCase, result: TestResult) {
if (result.status === 'failed') {
this.report.logEvent({
id: getTestIDForTitle(test.titlePath().join(' ')),
suite: test.parent.title,
title: test.title,
target: this.target,
command: this.command,
location: stripFilePath(test.location.file),
owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)),
plugin: this.plugin,
duration: result.duration,
error: {
message: result.error?.message ? stripFilePath(result.error.message) : undefined,
stack_trace: result.error?.stack ? stripFilePath(result.error.stack) : undefined,
},
stdout: result.stdout ? parseStdout(result.stdout) : undefined,
attachments: result.attachments.map((attachment) => ({
name: attachment.name,
path: attachment.path,
contentType: attachment.contentType,
})),
});
}
}
onEnd(result: FullResult) {
// Save & conclude the report
try {
this.report.save(this.reportRootPath);
} finally {
this.report.conclude();
}
}
}

12
packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { ScoutFailedTestReporter } from './failed_test_reporter';
// eslint-disable-next-line import/no-default-export
export default ScoutFailedTestReporter;

17
packages/kbn-scout-reporting/src/reporting/playwright/scout_playwright_reporter.ts Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
/**
* Configuration options for the Scout Playwright reporter
*/
export interface ScoutPlaywrightReporterOptions {
name?: string;
runId?: string;
outputPath?: string;
}

65
packages/kbn-scout-reporting/src/reporting/report/base.ts Normal file
View file

@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
// eslint-disable-next-line max-classes-per-file
import os from 'node:os';
import path from 'node:path';
import fs from 'node:fs';
import { ToolingLog } from '@kbn/tooling-log';
/**
* Generic error raised by a Scout report
*/
export class ScoutReportError extends Error {}
export abstract class ScoutReport {
log: ToolingLog;
workDir: string;
concluded = false;
reportName: string;
protected constructor(reportName: string, log?: ToolingLog) {
this.log = log || new ToolingLog();
this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-'));
this.reportName = reportName;
}
/**
* Defensive utility function used to guard against modifying the report after it has concluded
*
* @param additionalInfo Description of the report action that was prevented
* @protected
*/
protected raiseIfConcluded(additionalInfo?: string) {
if (this.concluded) {
let message = `${this.reportName} at ${this.workDir} was concluded`;
if (additionalInfo) {
message += `: ${additionalInfo}`;
}
throw new ScoutReportError(message);
}
}
/**
* Call this when you're done adding information to this report.
*
* ⚠️**This will delete all the contents of the report's working directory**
*/
conclude() {
// Remove the working directory
this.log.info(`Removing ${this.reportName} working directory ${this.workDir}`);
fs.rmSync(this.workDir, { recursive: true, force: true });
// Mark this report as concluded
this.concluded = true;
this.log.success(`${this.reportName} has concluded.`);
}
}

2
packages/kbn-scout-reporting/src/reporting/report/event.ts → packages/kbn-scout-reporting/src/reporting/report/events/event.ts
View file

@ -7,7 +7,7 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { BuildkiteMetadata, HostMetadata } from '../../datasources';
import { BuildkiteMetadata, HostMetadata } from '../../../datasources';
/**
* Scout reporter event type

12
packages/kbn-scout-reporting/src/reporting/report/events/index.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { ScoutEventsReport } from './report';
export * from './event';
export * from './persistence';

0
packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts → packages/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts
View file

0
packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts → packages/kbn-scout-reporting/src/reporting/report/events/persistence/index.ts
View file

0
packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts → packages/kbn-scout-reporting/src/reporting/report/events/persistence/index_templates.ts
View file

0
packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts → packages/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts
View file

65
packages/kbn-scout-reporting/src/reporting/report/events/report.ts Normal file
View file

@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import os from 'node:os';
import path from 'node:path';
import fs from 'node:fs';
import { ToolingLog } from '@kbn/tooling-log';
import { ScoutReportEvent } from './event';
import { ScoutReport, ScoutReportError } from '../base';
/**
*
*/
export class ScoutEventsReport extends ScoutReport {
constructor(log?: ToolingLog) {
super('Scout Events report', log);
this.log = log || new ToolingLog();
this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-'));
}
public get eventLogPath(): string {
return path.join(this.workDir, `event-log.ndjson`);
}
/**
* Logs an event to be processed by this reporter
*
* @param event {ScoutReportEvent} - Event to record
*/
logEvent(event: ScoutReportEvent) {
this.raiseIfConcluded('logging new events is no longer allowed');
if (event['@timestamp'] === undefined) {
event['@timestamp'] = new Date();
}
fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n');
}
/**
* Save the report to a non-ephemeral location
*
* @param destination - Full path to the save location. Must not exist.
*/
save(destination: string) {
this.raiseIfConcluded('nothing to save because workdir has been cleared');
if (fs.existsSync(destination)) {
throw new ScoutReportError(`Save destination path '${destination}' already exists`);
}
// Create the destination directory
this.log.info(`Saving ${this.reportName} to ${destination}`);
fs.mkdirSync(destination, { recursive: true });
// Copy the workdir data to the destination
fs.cpSync(this.workDir, destination, { recursive: true });
}
}

250
packages/kbn-scout-reporting/src/reporting/report/failed_test/html.ts Normal file
View file

@ -0,0 +1,250 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import fs from 'fs';
import { TestFailure } from './test_failure';
export const buildFailureHtml = (testFailure: TestFailure): string => {
const {
suite,
title,
target,
command,
location,
owner,
plugin,
duration,
error,
stdout,
attachments,
} = testFailure;
const testDuration = duration < 1000 ? `${duration}ms` : `${(duration / 1000).toFixed(2)}s`;
const screenshots = attachments
.filter((a) => a.contentType.startsWith('image/'))
.map((s) => {
const base64 = fs.readFileSync(s.path!).toString('base64');
return `
<div class="screenshotContainer">
<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${base64}" alt="${s.name}"/>
</div>
`;
});
return `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.1/dist/css/bootstrap.min.css"
rel="stylesheet"
integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU"
crossorigin="anonymous"
/>
<style>
body {
font-family: Arial, sans-serif;
font-size: 14px;
line-height: 1.6;
margin: 0;
padding: 0;
background-color: #f8f9fa;
}
h4, h5 {
margin-top: 20px;
margin-bottom: 10px;
}
hr {
margin: 20px 0;
}
.table-details {
width: 100%;
margin-top: 15px;
border: 1px solid #dee2e6;
}
.table-details td {
padding: 8px;
border: 1px solid #dee2e6;
}
.plugin-info {
margin-top: 10px;
}
pre {
font-size: 0.9em;
background-color: #f8f9fa;
padding: 10px;
border-radius: 5px;
border: 1px solid #dee2e6;
overflow-x: auto;
}
img.screenshot {
cursor: pointer;
margin: 5px 0;
}
.screenshotContainer:not(.expanded) img.screenshot {
height: 200px;
}
.screenshotContainer:not(.fs) img.screenshot.fs,
.screenshotContainer:not(.fs) button.toggleFs.off,
.screenshotContainer.fs img.screenshot:not(.fs),
.screenshotContainer.fs button.toggleFs.on {
display: none;
}
.screenshotContainer .toggleFs {
background: none;
border: none;
margin: 0 0 0 5px;
vertical-align: top;
}
.section {
margin-bottom: 20px;
}
</style>
<title>Scout Test Failure Report</title>
</head>
<body>
<div class="container my-5">
<main>
<h4>Scout Test Failure Report</h4>
<h5>Location: ${location}</h5>
<hr />
<div class="section">
<h5>Test Details</h5>
<table class="table-details">
<tr>
<td><strong>Suite Title</strong></td>
<td>${suite}</td>
</tr>
<tr>
<td><strong>Test Title</strong></td>
<td>${title}</td>
</tr>
<tr>
<td><strong>Execution Details</strong></td>
<td>
Target: <em>${target}</em>,
Duration: <em>${testDuration}</em>
</td>
</tr>
<tr>
<td><strong>Plugin</strong></td>
<td>
ID: <em>${plugin?.id}</em>,
Visibility: <em>${plugin?.visibility}</em>,
Group: <em>${plugin?.group}</em>
</td>
</tr>
</table>
</div>
<div id="ci-links-placeholder">
<!-- Placeholder for GitHub and Buildkite links -->
</div>
<div class="section">
<h5>Command Line</h5>
<pre>${command}</pre>
</div>
<div class="section">
<h5>Owners</h5>
<pre>${owner.join(', ')}</pre>
</div>
<div class="section">
<h5>Error Details</h5>
<pre>${error?.stack_trace || 'No stack trace available'}</pre>
</div>
<div>
<details>
<summary>
<strong>Failures in tracked branches</strong>:
<span class="badge rounded-pill bg-danger" id="failure-count">0</span>
</summary>
<div id="github-issue-section" style="display: none;">
<a id="github-issue-link" href="" target="_blank"></a>
</div>
</details>
</div>
<div class="section">
<h5>Output Logs</h5>
<details>
<pre>${stdout || 'No output available'}</pre>
</details>
</div>
<div class="section">
<h5>Attachments</h5>
${screenshots.join('/n')}
</div>
</main>
</div>
<script type="text/javascript">
/**
* @param {HTMLElement} el
* @param {(el: HTMLElement) => boolean} className
*/
function findParent(el, test) {
while (el) {
if (test(el)) {
return el
}
// stop if we iterate all the way up to the document body
if (el.parentElement === document.body) {
break
}
el = el.parentElement
}
return null
}
function isContainer(el) {
return el.classList.contains('screenshotContainer')
}
function isButtonOrImg(el) {
return el instanceof HTMLImageElement || el instanceof HTMLButtonElement
}
document.addEventListener('click', (event) => {
const el = findParent(event.target, isButtonOrImg)
if (el instanceof HTMLImageElement && el.classList.contains('screenshot')) {
findParent(el, isContainer)?.classList.toggle('expanded')
}
if (el instanceof HTMLButtonElement && el.classList.contains('toggleFs')) {
findParent(el, isContainer)?.classList.toggle('fs')
}
})
</script>
</body>
</html>
`;
};

11
packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { ScoutFailureReport } from './report';
export type { TestFailure } from './test_failure';

119
packages/kbn-scout-reporting/src/reporting/report/failed_test/report.ts Normal file
View file

@ -0,0 +1,119 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import path from 'node:path';
import fs from 'node:fs';
import { ToolingLog } from '@kbn/tooling-log';
import { buildFailureHtml } from './html';
import { TestFailure } from './test_failure';
import { ScoutReport, ScoutReportError } from '../base';
const saveTestFailuresReport = (
reportPath: string,
testFailureHtml: string,
log: ToolingLog,
message: string
): void => {
try {
fs.writeFileSync(reportPath, testFailureHtml, 'utf-8');
log.info(message);
} catch (error) {
log.error(`Failed to save report at ${reportPath}: ${error.message}`);
}
};
export class ScoutFailureReport extends ScoutReport {
constructor(log?: ToolingLog) {
super('Scout Failure report', log);
}
public get testFailuresPath(): string {
return path.join(this.workDir, `test-failures.ndjson`);
}
/**
* Logs a failure to be processed by this reporter
*
* @param failure {TestFailure} - test failure to record
*/
logEvent(failure: TestFailure) {
this.raiseIfConcluded('logging new failures is no longer allowed');
fs.appendFileSync(this.testFailuresPath, JSON.stringify(failure) + '\n');
}
/**
* Save the report to a non-ephemeral location
*
* @param destination - Full path to the save location. Must not exist.
*/
save(destination: string) {
this.raiseIfConcluded('nothing to save because workdir has been cleared');
if (fs.existsSync(destination)) {
throw new ScoutReportError(`Save destination path '${destination}' already exists`);
}
const testFailures: TestFailure[] = this.readFailuresFromNDJSON();
if (testFailures.length === 0) {
this.log.info('No test failures to report');
return;
}
// Create the destination directory
this.log.info(
`Saving ${this.reportName} to ${destination}: ${testFailures.length} failures reported`
);
fs.mkdirSync(destination, { recursive: true });
// Generate HTML report for each failed test with embedded screenshots
for (const failure of testFailures) {
const htmlContent = buildFailureHtml(failure);
const htmlReportPath = path.join(destination, `${failure.id}.html`);
saveTestFailuresReport(
htmlReportPath,
htmlContent,
this.log,
`Html report for ${failure.id} is saved at ${htmlReportPath}`
);
}
const summaryContent = testFailures.map((failure) => {
return {
name: `${failure.target} - ${failure.suite} - ${failure.title}`,
htmlReportFilename: `${failure.id}.html`,
};
});
// Short summary report linking to the detailed HTML reports
const testFailuresSummaryReportPath = path.join(destination, 'test-failures-summary.json');
saveTestFailuresReport(
testFailuresSummaryReportPath,
JSON.stringify(summaryContent, null, 2),
this.log,
`Test Failures Summary is saved at ${testFailuresSummaryReportPath}`
);
}
/**
* Reads all failures from the NDJSON file and parses them as TestFailure[].
*/
private readFailuresFromNDJSON(): TestFailure[] {
if (!fs.existsSync(this.testFailuresPath)) {
return [];
}
const fileContent = fs.readFileSync(this.testFailuresPath, 'utf-8');
return fileContent
.split('\n') // Split lines
.filter((line) => line.trim() !== '') // Remove empty lines
.map((line) => JSON.parse(line) as TestFailure); // Parse each line into an object
}
}

34
packages/kbn-scout-reporting/src/reporting/report/failed_test/test_failure.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export interface TestFailure {
id: string;
suite: string;
title: string;
target: string;
command: string;
location: string;
owner: string[];
plugin?: {
id: string;
visibility: string;
group: string;
};
duration: number;
error: {
message?: string;
stack_trace?: string;
};
stdout?: string;
attachments: Array<{
name: string;
path?: string;
contentType: string;
}>;
}

96
packages/kbn-scout-reporting/src/reporting/report/index.ts
View file

@ -7,97 +7,5 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
// eslint-disable-next-line max-classes-per-file
import os from 'node:os';
import path from 'node:path';
import fs from 'node:fs';
import { ToolingLog } from '@kbn/tooling-log';
import { ScoutReportEvent } from './event';
/**
* Generic error raised by a Scout report
*/
export class ScoutReportError extends Error {}
/**
*
*/
export class ScoutReport {
log: ToolingLog;
workDir: string;
concluded = false;
constructor(log?: ToolingLog) {
this.log = log || new ToolingLog();
this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-'));
}
public get eventLogPath(): string {
return path.join(this.workDir, `event-log.ndjson`);
}
private raiseIfConcluded(additionalInfo?: string) {
if (this.concluded) {
let message = `Report at ${this.workDir} was concluded`;
if (additionalInfo) {
message += `: ${additionalInfo}`;
}
throw new ScoutReportError(message);
}
}
/**
* Logs an event to be processed by this reporter
*
* @param event {ScoutReportEvent} - Event to record
*/
logEvent(event: ScoutReportEvent) {
this.raiseIfConcluded('logging new events is no longer allowed');
if (event['@timestamp'] === undefined) {
event['@timestamp'] = new Date();
}
fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n');
}
/**
* Save the report to a non-ephemeral location
*
* @param destination - Full path to the save location. Must not exist.
*/
save(destination: string) {
this.raiseIfConcluded('nothing to save because workdir has been cleared');
if (fs.existsSync(destination)) {
throw new ScoutReportError(`Save destination path '${destination}' already exists`);
}
// Create the destination directory
this.log.info(`Saving Scout report to ${destination}`);
fs.mkdirSync(destination, { recursive: true });
// Copy the workdir data to the destination
fs.cpSync(this.workDir, destination, { recursive: true });
}
/**
* Call this when you're done adding information to this report.
*
* ⚠️**This will delete all the contents of the report's working directory**
*/
conclude() {
// Remove the working directory
this.log.info(`Removing Scout report working directory ${this.workDir}`);
fs.rmSync(this.workDir, { recursive: true, force: true });
// Mark this report as concluded
this.concluded = true;
this.log.success('Scout report has concluded.');
}
}
export * from './event';
export * from './persistence';
export { ScoutEventsReport, ScoutReportEventAction } from './events';
export { ScoutFailureReport, type TestFailure } from './failed_test';

1
packages/kbn-scout-reporting/tsconfig.json
View file

@ -20,5 +20,6 @@
"@kbn/scout-info",
"@kbn/repo-info",
"@kbn/code-owners",
"@kbn/repo-packages",
]
}

27
packages/kbn-scout/src/playwright/config/create_config.test.ts
View file

@ -10,13 +10,31 @@
import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
import { createPlaywrightConfig } from './create_config';
import { VALID_CONFIG_MARKER } from '../types';
import { generateTestRunId } from '@kbn/scout-reporting';
jest.mock('@kbn/scout-reporting', () => ({
...jest.requireActual('@kbn/scout-reporting'),
generateTestRunId: jest.fn(),
}));
describe('createPlaywrightConfig', () => {
const mockGenerateTestRunId = generateTestRunId as jest.Mock;
beforeEach(() => {
jest.clearAllMocks();
});
it('should return a valid default Playwright configuration', () => {
const testRunId = 'test-run-id';
mockGenerateTestRunId.mockImplementationOnce(() => testRunId);
const testDir = './my_tests';
const config = createPlaywrightConfig({ testDir });
expect(mockGenerateTestRunId).toHaveBeenCalledTimes(1);
expect(config.testDir).toBe(testDir);
expect(config.retries).toBe(0);
expect(config.workers).toBe(1);
expect(config.fullyParallel).toBe(false);
expect(config.use).toEqual({
@ -30,7 +48,14 @@ describe('createPlaywrightConfig', () => {
expect(config.reporter).toEqual([
['html', { open: 'never', outputFolder: './output/reports' }],
['json', { outputFile: './output/reports/test-results.json' }],
['@kbn/scout-reporting/src/reporting/playwright.ts', { name: 'scout-playwright' }],
[
'@kbn/scout-reporting/src/reporting/playwright/events',
{ name: 'scout-playwright', runId: testRunId },
],
[
'@kbn/scout-reporting/src/reporting/playwright/failed_test',
{ name: 'scout-playwright-failed-tests', runId: testRunId },
],
]);
expect(config.timeout).toBe(60000);
expect(config.expect?.timeout).toBe(10000);

13
packages/kbn-scout/src/playwright/config/create_config.ts
View file

@ -8,11 +8,17 @@
*/
import { defineConfig, PlaywrightTestConfig, devices } from '@playwright/test';
import { scoutPlaywrightReporter } from '@kbn/scout-reporting';
import {
scoutFailedTestsReporter,
scoutPlaywrightReporter,
generateTestRunId,
} from '@kbn/scout-reporting';
import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
import { ScoutPlaywrightOptions, ScoutTestOptions, VALID_CONFIG_MARKER } from '../types';
export function createPlaywrightConfig(options: ScoutPlaywrightOptions): PlaywrightTestConfig {
const runId = generateTestRunId();
return defineConfig<ScoutTestOptions>({
testDir: options.testDir,
/* Run tests in files in parallel */
@ -20,14 +26,15 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: process.env.CI ? 2 : 0,
retries: 0, // disable retry for Playwright runner
/* Opt out of parallel tests on CI. */
workers: options.workers ?? 1,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: [
['html', { outputFolder: './output/reports', open: 'never' }], // HTML report configuration
['json', { outputFile: './output/reports/test-results.json' }], // JSON report
scoutPlaywrightReporter({ name: 'scout-playwright' }), // Scout report
scoutPlaywrightReporter({ name: 'scout-playwright', runId }), // Scout events report
scoutFailedTestsReporter({ name: 'scout-playwright-failed-tests', runId }), // Scout failed test report
],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {

8
packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts
View file

@ -14,9 +14,9 @@ import { REPO_ROOT } from '@kbn/repo-info';
import {
generateTestRunId,
getTestIDForTitle,
ScoutReport,
ScoutReportEventAction,
datasources,
ScoutEventsReport,
ScoutReportEventAction,
} from '@kbn/scout-reporting';
import {
getOwningTeamsForPath,
@ -40,7 +40,7 @@ export class ScoutFTRReporter {
readonly log: ToolingLog;
readonly name: string;
readonly runId: string;
private report: ScoutReport;
private report: ScoutEventsReport;
private readonly codeOwnersEntries: CodeOwnersEntry[];
constructor(private runner: Runner, private reporterOptions: ScoutFTRReporterOptions = {}) {
@ -53,7 +53,7 @@ export class ScoutFTRReporter {
this.runId = generateTestRunId();
this.log.info(`Scout test run ID: ${this.runId}`);
this.report = new ScoutReport(this.log);
this.report = new ScoutEventsReport(this.log);
this.codeOwnersEntries = getCodeOwnersEntries();
// Register event listeners