[Synthetics] Clean up e2e test helpers !! (#203812)

## Summary

Clean up e2e test helpers
This commit is contained in:
Shahzad 2024-12-12 10:52:19 +01:00 committed by GitHub
parent f4795cdcd7
commit 0203bba44f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
64 changed files with 129 additions and 1541 deletions

View file

@ -6,3 +6,4 @@
*/
export { makeUpSummary, makeDownSummary } from './src/make_summaries';
export * from './src/e2e';

View file

@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { expect, Page } from '@elastic/synthetics';
export async function waitForLoadingToFinish({ page }: { page: Page }) {
while (true) {
if (!(await page.isVisible(byTestId('kbnLoadingMessage'), { timeout: 5000 }))) break;
await page.waitForTimeout(1000);
}
}
export async function loginToKibana({
page,
user,
}: {
page: Page;
user?: { username: string; password: string };
}) {
await page.fill('[data-test-subj=loginUsername]', user?.username ?? 'elastic', {
timeout: 60 * 1000,
});
await page.fill('[data-test-subj=loginPassword]', user?.password ?? 'changeme');
await page.click('[data-test-subj=loginSubmit]');
await waitForLoadingToFinish({ page });
}
export const byTestId = (testId: string) => {
return `[data-test-subj=${testId}]`;
};
export const assertText = async ({ page, text }: { page: Page; text: string }) => {
const element = await page.waitForSelector(`text=${text}`);
expect(await element.isVisible()).toBeTruthy();
};
export const assertNotText = async ({ page, text }: { page: Page; text: string }) => {
expect(await page.$(`text=${text}`)).toBeFalsy();
};
export const getQuerystring = (params: object) => {
return Object.entries(params)
.map(([key, value]) => encodeURIComponent(key) + '=' + encodeURIComponent(value))
.join('&');
};
export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
export const TIMEOUT_60_SEC = {
timeout: 60 * 1000,
};

View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { recordVideo } from './helpers/record_video';
export { SyntheticsRunner } from './helpers/synthetics_runner';
export { argv } from './helpers/parse_args_params';
export { readKibanaConfig } from './tasks/read_kibana_config';

View file

@ -7,6 +7,7 @@
import Path from 'path';
import { execSync } from 'child_process';
import { REPO_ROOT } from '@kbn/repo-info';
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
@ -16,7 +17,7 @@ const NODE_TLS_REJECT_UNAUTHORIZED = '1';
export const esArchiverLoad = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../scripts/es_archiver load "${path}" --config ../../../test/functional/config.base.js`,
`node ${REPO_ROOT}/scripts/es_archiver load "${path}" --config ${REPO_ROOT}/test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
@ -24,14 +25,7 @@ export const esArchiverLoad = (folder: string) => {
export const esArchiverUnload = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../scripts/es_archiver unload "${path}" --config ../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverResetKibana = () => {
execSync(
`node ../../../../scripts/es_archiver empty-kibana-index --config ../../../test/functional/config.base.js`,
`node ${REPO_ROOT}/scripts/es_archiver unload "${path}" --config ${REPO_ROOT}/test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};

View file

@ -16,5 +16,8 @@
"target/**/*"
],
"kbn_references": [
"@kbn/apm-plugin",
"@kbn/es-archiver",
"@kbn/repo-info",
]
}

View file

@ -6,7 +6,7 @@
*/
import { journey, step, before } from '@elastic/synthetics';
import { recordVideo } from '../record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, before } from '@elastic/synthetics';
import { recordVideo } from '../record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';

View file

@ -6,10 +6,10 @@
*/
import { journey, step } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import moment from 'moment';
import { recordVideo } from '../record_video';
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
import { byTestId, loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
journey('Step Duration series', async ({ page, params }) => {
recordVideo(page);
@ -56,7 +56,8 @@ journey('Step Duration series', async ({ page, params }) => {
await page.click('[aria-label="Remove report metric"]');
await page.click('button:has-text("Select report metric")');
await page.click('button:has-text("Step duration")');
await page.click(byTestId('seriesBreakdown'));
await page.waitForSelector('[data-test-subj=seriesBreakdown]');
await page.getByTestId('seriesBreakdown').click();
await page.click('button[role="option"]:has-text("Step name")');
await page.click('.euiComboBox__inputWrap');
await page.click('[role="combobox"][placeholder="Search Monitor name"]');

View file

@ -1,33 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import yargs from 'yargs';
const { argv } = yargs(process.argv.slice(2))
.option('headless', {
default: true,
type: 'boolean',
description: 'Start in headless mode',
})
.option('bail', {
default: false,
type: 'boolean',
description: 'Pause on error',
})
.option('watch', {
default: false,
type: 'boolean',
description: 'Runs the server in watch mode, restarting on changes',
})
.option('grep', {
default: undefined,
type: 'string',
description: 'run only journeys with a name or tags that matches the glob',
})
.help();
export { argv };

View file

@ -1,32 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fs from 'fs';
import Runner from '@elastic/synthetics/dist/core/runner';
import { after, Page } from '@elastic/synthetics';
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
// @ts-ignore
export const runner: Runner = global[SYNTHETICS_RUNNER];
export const recordVideo = (page: Page, postfix = '') => {
after(async () => {
try {
const videoFilePath = await page.video()?.path();
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
const newVideoPath = videoFilePath?.replace(
pathToVideo!,
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
);
fs.renameSync(videoFilePath!, newVideoPath!);
} catch (e) {
// eslint-disable-next-line no-console
console.log('Error while renaming video file', e);
}
});
};

View file

@ -6,8 +6,8 @@
*/
import { FtrConfigProviderContext } from '@kbn/test';
import path from 'path';
import { SyntheticsRunner } from './synthetics_runner';
import { argv } from './parse_args_params';
import { REPO_ROOT } from '@kbn/repo-info';
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
const { headless, grep, bail: pauseOnError } = argv;
@ -24,13 +24,12 @@ async function runE2ETests({ readConfigFile }: FtrConfigProviderContext) {
});
await syntheticsRunner.setup();
await syntheticsRunner.loadTestData(path.join(__dirname, '../../ux/e2e/fixtures/'), [
'rum_8.0.0',
'rum_test_data',
]);
await syntheticsRunner.loadTestData(
path.join(__dirname, '../../synthetics/e2e/fixtures/es_archiver/'),
`${REPO_ROOT}/x-pack/plugins/observability_solution/ux/e2e/fixtures/`,
['rum_8.0.0', 'rum_test_data']
);
await syntheticsRunner.loadTestData(
`${REPO_ROOT}/x-pack/plugins/observability_solution/synthetics/e2e/fixtures/es_archiver/`,
['full_heartbeat', 'browser']
);
await syntheticsRunner.loadTestFiles(async () => {

View file

@ -1,155 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable no-console */
import Url from 'url';
import { run as syntheticsRun } from '@elastic/synthetics';
import { PromiseType } from 'utility-types';
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
import { EsArchiver } from '@kbn/es-archiver';
import { esArchiverUnload } from './tasks/es_archiver';
import { TestReporter } from './test_reporter';
export interface ArgParams {
headless: boolean;
match?: string;
pauseOnError: boolean;
}
export class SyntheticsRunner {
public getService: any;
public kibanaUrl: string;
private elasticsearchUrl: string;
public testFilesLoaded: boolean = false;
public params: ArgParams;
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
constructor(getService: any, params: ArgParams) {
this.getService = getService;
this.kibanaUrl = this.getKibanaUrl();
this.elasticsearchUrl = this.getElasticsearchUrl();
this.params = params;
}
async setup() {
await this.createTestUsers();
}
async createTestUsers() {
await createApmUsers({
elasticsearch: { node: this.elasticsearchUrl, username: 'elastic', password: 'changeme' },
kibana: { hostname: this.kibanaUrl },
});
}
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
console.log('Loading test files');
await callback(reload);
this.loadTestFilesCallback = callback;
this.testFilesLoaded = true;
console.log('Successfully loaded test files');
}
async loadTestData(e2eDir: string, dataArchives: string[]) {
try {
console.log('Loading esArchiver...');
const esArchiver: EsArchiver = this.getService('esArchiver');
const promises = dataArchives.map((archive) => {
if (archive === 'synthetics_data') {
return esArchiver.load(e2eDir + archive, {
docsOnly: true,
skipExisting: true,
});
}
return esArchiver.load(e2eDir + archive, { skipExisting: true });
});
await Promise.all([...promises]);
} catch (e) {
console.log(e);
}
}
getKibanaUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.kibana.protocol'),
hostname: config.get('servers.kibana.hostname'),
port: config.get('servers.kibana.port'),
});
}
getElasticsearchUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.elasticsearch.protocol'),
hostname: config.get('servers.elasticsearch.hostname'),
port: config.get('servers.elasticsearch.port'),
});
}
async run() {
if (!this.testFilesLoaded) {
throw new Error('Test files not loaded');
}
const { headless, match, pauseOnError } = this.params;
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
console.log(`Running ${noOfRuns} times`);
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
for (let i = 0; i < noOfRuns; i++) {
results = await syntheticsRun({
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
playwrightOptions: {
headless,
chromiumSandbox: false,
timeout: 60 * 1000,
viewport: {
height: 900,
width: 1600,
},
recordVideo: {
dir: '.journeys/videos',
},
},
grepOpts: { match: match === 'undefined' ? '' : match },
pauseOnError,
screenshots: 'only-on-failure',
reporter: TestReporter,
});
if (noOfRuns > 1) {
// need to reload again since runner resets the journeys
await this.loadTestFiles(this.loadTestFilesCallback!, true);
}
}
await this.assertResults(results);
}
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
Object.entries(results).forEach(([_journey, result]) => {
if (result.status !== 'succeeded') {
process.exitCode = 1;
process.exit();
}
});
}
cleanUp() {
console.log('Removing esArchiver...');
esArchiverUnload('full_heartbeat');
esArchiverUnload('browser');
}
}

View file

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Path from 'path';
import { execSync } from 'child_process';
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
export const esArchiverLoad = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../scripts/es_archiver load "${path}" --config ../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverUnload = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../scripts/es_archiver unload "${path}" --config ../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverResetKibana = () => {
execSync(
`node ../../../../scripts/es_archiver empty-kibana-index --config ../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};

View file

@ -6,5 +6,9 @@
"outDir": "target/types",
"types": ["node"]
},
"kbn_references": ["@kbn/test", "@kbn/apm-plugin", "@kbn/es-archiver"]
"kbn_references": [
"@kbn/test",
"@kbn/repo-info",
"@kbn/observability-synthetics-test-data",
]
}

View file

@ -7,7 +7,7 @@
import { Page } from '@elastic/synthetics';
import { loginPageProvider } from '@kbn/synthetics-e2e/page_objects/login';
import { utilsPageProvider } from '@kbn/synthetics-e2e/page_objects/utils';
import { recordVideo } from '@kbn/synthetics-e2e/helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
export function sloAppPageProvider({ page, kibanaUrl }: { page: Page; kibanaUrl: string }) {
page.setDefaultTimeout(60 * 1000);

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { FtrConfigProviderContext } from '@kbn/test';
import { SyntheticsRunner, argv } from '@kbn/synthetics-e2e';
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
const { headless, grep, bail: pauseOnError } = argv;

View file

@ -12,5 +12,6 @@
"@kbn/ftr-common-functional-services",
"@kbn/data-forge",
"@kbn/synthetics-e2e",
"@kbn/observability-synthetics-test-data",
]
}

View file

@ -10,8 +10,8 @@ import { CA_CERT_PATH } from '@kbn/dev-utils';
import { get } from 'lodash';
import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services';
import { readKibanaConfig } from '@kbn/observability-synthetics-test-data';
import { readKibanaConfig } from './tasks/read_kibana_config';
const MANIFEST_KEY = 'xpack.uptime.service.manifestUrl';
const SERVICE_PASSWORD = 'xpack.uptime.service.password';
const SERVICE_USERNAME = 'xpack.uptime.service.username';

View file

@ -1,229 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
import { Reporter, ReporterOptions } from '@elastic/synthetics';
import {
JourneyEndResult,
JourneyStartResult,
StepEndResult,
} from '@elastic/synthetics/dist/common_types';
import { yellow, green, cyan, red, bold } from 'chalk';
// eslint-disable-next-line no-console
const log = console.log;
import { performance } from 'perf_hooks';
import * as fs from 'fs';
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
import { join } from 'path';
function renderError(error: any) {
let output = '';
const outer = indent('');
const inner = indent(outer);
const container = outer + '---\n';
output += container;
let stack = error.stack;
if (stack) {
output += inner + 'stack: |-\n';
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
const lines = String(stack).split('\n');
for (const line of lines) {
output += inner + ' ' + line + '\n';
}
}
output += container;
return red(output);
}
function renderDuration(durationMs: number) {
return Number(durationMs).toFixed(0);
}
export class TestReporter implements Reporter {
metrics = {
succeeded: 0,
failed: 0,
skipped: 0,
};
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
constructor(options: ReporterOptions = {}) {}
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
if (process.env.CI) {
this.write(`\n--- Journey: ${journey.name}`);
} else {
this.write(bold(`\n Journey: ${journey.name}`));
}
}
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
const { status, end, start, error } = result;
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
this.metrics[status]++;
if (!this.journeys.has(journey.name)) {
this.journeys.set(journey.name, []);
}
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
}
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
if (total === 0 && error) {
this.write(renderError(error));
}
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
this.write(message);
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
const { data, step } = screenshot;
if (status === 'failed') {
await (async () => {
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
encoding: 'base64',
});
})();
}
});
}
onEnd() {
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.some((step) => step.status === 'failed')
);
if (failedJourneys.length > 0) {
failedJourneys.forEach(([journeyName, steps]) => {
if (process.env.CI) {
const name = red(`Journey: ${journeyName} 🥵`);
this.write(`\n+++ ${name}`);
steps.forEach((stepResult) => {
const { status, end, start, error, name: stepName } = stepResult;
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
});
}
});
}
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.every((step) => step.status === 'succeeded')
);
successfulJourneys.forEach(([journeyName, steps]) => {
try {
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
} catch (e) {
// eslint-disable-next-line no-console
console.log(
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
);
}
});
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
let message = '\n';
if (total === 0) {
message = 'No tests found!';
message += ` (${renderDuration(now())} ms) \n`;
this.write(message);
return;
}
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
message += failed > 0 ? red(` ${failed} failed`) : '';
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
this.write(message);
}
write(message: any) {
if (typeof message === 'object') {
message = JSON.stringify(message);
}
log(message + '\n');
}
}
const SEPARATOR = '\n';
function indent(lines: string, tab = ' ') {
return lines.replace(/^/gm, tab);
}
const NO_UTF8_SUPPORT = process.platform === 'win32';
const symbols = {
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
skipped: cyan('-'),
progress: cyan('>'),
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
};
function now() {
return performance.now();
}
function findPWLogsIndexes(msgOrStack: string): [number, number] {
let startIndex = 0;
let endIndex = 0;
if (!msgOrStack) {
return [startIndex, endIndex];
}
const lines = String(msgOrStack).split(SEPARATOR);
const logStart = /[=]{3,} logs [=]{3,}/;
const logEnd = /[=]{10,}/;
lines.forEach((line, index) => {
if (logStart.test(line)) {
startIndex = index;
} else if (logEnd.test(line)) {
endIndex = index;
}
});
return [startIndex, endIndex];
}
function rewriteErrorStack(stack: string, indexes: [number, number]) {
const [start, end] = indexes;
/**
* Do not rewrite if its not a playwright error
*/
if (start === 0 && end === 0) {
return stack;
}
const linesToKeep = start + 3;
if (start > 0 && linesToKeep < end) {
const lines = stack.split(SEPARATOR);
return lines
.slice(0, linesToKeep)
.concat(...lines.slice(end))
.join(SEPARATOR);
}
return stack;
}

View file

@ -5,7 +5,5 @@
* 2.0.
*/
export { SyntheticsRunner } from './helpers/synthetics_runner';
export { argv } from './helpers/parse_args_params';
export { loginPageProvider } from './page_objects/login';
export { utilsPageProvider } from './page_objects/utils';

View file

@ -7,7 +7,7 @@
import { journey, step, expect, Page } from '@elastic/synthetics';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { recordVideo } from '../../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';
import { byTestId, assertText } from '../../helpers/utils';

View file

@ -7,7 +7,7 @@
import { journey, step, expect, before } from '@elastic/synthetics';
import { SYNTHETICS_API_URLS } from '@kbn/synthetics-plugin/common/constants';
import { recordVideo } from '../../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
journey('ProjectAPIKeys', async ({ page }) => {
recordVideo(page);

View file

@ -7,7 +7,7 @@
import { expect, Page } from '@elastic/synthetics';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { FormMonitorType } from '@kbn/synthetics-plugin/common/runtime_types/monitor_management';
import { recordVideo } from '../../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { loginPageProvider } from '../../page_objects/login';
import { utilsPageProvider } from '../../page_objects/utils';

View file

@ -4,10 +4,9 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
import { FtrConfigProviderContext } from '@kbn/test';
import path from 'path';
import { argv } from '../helpers/parse_args_params';
import { SyntheticsRunner } from '../helpers/synthetics_runner';
const { headless, grep, bail: pauseOnError } = argv;

View file

@ -12,10 +12,7 @@
"@kbn/dev-utils",
"@kbn/ux-plugin/e2e",
"@kbn/ftr-common-functional-services",
"@kbn/apm-plugin",
"@kbn/es-archiver",
"@kbn/synthetics-plugin",
"@kbn/repo-info",
"@kbn/observability-synthetics-test-data",
"@kbn/ftr-common-functional-ui-services"
]

View file

@ -9,7 +9,7 @@ import { FtrConfigProviderContext } from '@kbn/test';
import { CA_CERT_PATH } from '@kbn/dev-utils';
import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services';
import { readKibanaConfig } from './tasks/read_kibana_config';
import { readKibanaConfig } from '@kbn/observability-synthetics-test-data';
const MANIFEST_KEY = 'xpack.uptime.service.manifestUrl';
const SERVICE_PASSWORD = 'xpack.uptime.service.password';
const SERVICE_USERNAME = 'xpack.uptime.service.username';

View file

@ -1,28 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import yargs from 'yargs';
const { argv } = yargs(process.argv.slice(2))
.option('headless', {
default: true,
type: 'boolean',
description: 'Start in headless mode',
})
.option('bail', {
default: false,
type: 'boolean',
description: 'Pause on error',
})
.option('grep', {
default: undefined,
type: 'string',
description: 'run only journeys with a name or tags that matches the glob',
})
.help();
export { argv };

View file

@ -1,32 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fs from 'fs';
import Runner from '@elastic/synthetics/dist/core/runner';
import { after, Page } from '@elastic/synthetics';
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
// @ts-ignore
export const runner: Runner = global[SYNTHETICS_RUNNER];
export const recordVideo = (page: Page, postfix = '') => {
after(async () => {
try {
const videoFilePath = await page.video()?.path();
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
const newVideoPath = videoFilePath?.replace(
pathToVideo!,
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
);
fs.renameSync(videoFilePath!, newVideoPath!);
} catch (e) {
// eslint-disable-next-line no-console
console.log('Error while renaming video file', e);
}
});
};

View file

@ -1,155 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable no-console */
import Url from 'url';
import { run as syntheticsRun } from '@elastic/synthetics';
import { PromiseType } from 'utility-types';
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
import { EsArchiver } from '@kbn/es-archiver';
import { esArchiverUnload } from '../tasks/es_archiver';
import { TestReporter } from './test_reporter';
export interface ArgParams {
headless: boolean;
match?: string;
pauseOnError: boolean;
}
export class SyntheticsRunner {
public getService: any;
public kibanaUrl: string;
private elasticsearchUrl: string;
public testFilesLoaded: boolean = false;
public params: ArgParams;
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
constructor(getService: any, params: ArgParams) {
this.getService = getService;
this.kibanaUrl = this.getKibanaUrl();
this.elasticsearchUrl = this.getElasticsearchUrl();
this.params = params;
}
async setup() {
await this.createTestUsers();
}
async createTestUsers() {
await createApmUsers({
elasticsearch: { node: this.elasticsearchUrl, username: 'elastic', password: 'changeme' },
kibana: { hostname: this.kibanaUrl },
});
}
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
console.log('Loading test files');
await callback(reload);
this.loadTestFilesCallback = callback;
this.testFilesLoaded = true;
console.log('Successfully loaded test files');
}
async loadTestData(e2eDir: string, dataArchives: string[]) {
try {
console.log('Loading esArchiver...');
const esArchiver: EsArchiver = this.getService('esArchiver');
const promises = dataArchives.map((archive) => {
if (archive === 'synthetics_data') {
return esArchiver.load(e2eDir + archive, {
docsOnly: true,
skipExisting: true,
});
}
return esArchiver.load(e2eDir + archive, { skipExisting: true });
});
await Promise.all([...promises]);
} catch (e) {
console.log(e);
}
}
getKibanaUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.kibana.protocol'),
hostname: config.get('servers.kibana.hostname'),
port: config.get('servers.kibana.port'),
});
}
getElasticsearchUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.elasticsearch.protocol'),
hostname: config.get('servers.elasticsearch.hostname'),
port: config.get('servers.elasticsearch.port'),
});
}
async run() {
if (!this.testFilesLoaded) {
throw new Error('Test files not loaded');
}
const { headless, match, pauseOnError } = this.params;
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
console.log(`Running ${noOfRuns} times`);
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
for (let i = 0; i < noOfRuns; i++) {
results = await syntheticsRun({
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
playwrightOptions: {
headless,
chromiumSandbox: false,
timeout: 60 * 1000,
viewport: {
height: 900,
width: 1600,
},
recordVideo: {
dir: '.journeys/videos',
},
},
grepOpts: { match: match === 'undefined' ? '' : match },
pauseOnError,
screenshots: 'only-on-failure',
reporter: TestReporter,
});
if (noOfRuns > 1) {
// need to reload again since runner resets the journeys
await this.loadTestFiles(this.loadTestFilesCallback!, true);
}
}
await this.assertResults(results);
}
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
Object.entries(results).forEach(([_journey, result]) => {
if (result.status !== 'succeeded') {
process.exitCode = 1;
process.exit();
}
});
}
cleanUp() {
console.log('Removing esArchiver...');
esArchiverUnload('full_heartbeat');
esArchiverUnload('browser');
}
}

View file

@ -1,229 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
import { Reporter, ReporterOptions } from '@elastic/synthetics';
import {
JourneyEndResult,
JourneyStartResult,
StepEndResult,
} from '@elastic/synthetics/dist/common_types';
import { yellow, green, cyan, red, bold } from 'chalk';
// eslint-disable-next-line no-console
const log = console.log;
import { performance } from 'perf_hooks';
import * as fs from 'fs';
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
import { join } from 'path';
function renderError(error: any) {
let output = '';
const outer = indent('');
const inner = indent(outer);
const container = outer + '---\n';
output += container;
let stack = error.stack;
if (stack) {
output += inner + 'stack: |-\n';
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
const lines = String(stack).split('\n');
for (const line of lines) {
output += inner + ' ' + line + '\n';
}
}
output += container;
return red(output);
}
function renderDuration(durationMs: number) {
return Number(durationMs).toFixed(0);
}
export class TestReporter implements Reporter {
metrics = {
succeeded: 0,
failed: 0,
skipped: 0,
};
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
constructor(options: ReporterOptions = {}) {}
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
if (process.env.CI) {
this.write(`\n--- Journey: ${journey.name}`);
} else {
this.write(bold(`\n Journey: ${journey.name}`));
}
}
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
const { status, end, start, error } = result;
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
this.metrics[status]++;
if (!this.journeys.has(journey.name)) {
this.journeys.set(journey.name, []);
}
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
}
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
if (total === 0 && error) {
this.write(renderError(error));
}
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
this.write(message);
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
const { data, step } = screenshot;
if (status === 'failed') {
await (async () => {
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
encoding: 'base64',
});
})();
}
});
}
onEnd() {
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.some((step) => step.status === 'failed')
);
if (failedJourneys.length > 0) {
failedJourneys.forEach(([journeyName, steps]) => {
if (process.env.CI) {
const name = red(`Journey: ${journeyName} 🥵`);
this.write(`\n+++ ${name}`);
steps.forEach((stepResult) => {
const { status, end, start, error, name: stepName } = stepResult;
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
});
}
});
}
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.every((step) => step.status === 'succeeded')
);
successfulJourneys.forEach(([journeyName, steps]) => {
try {
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
} catch (e) {
// eslint-disable-next-line no-console
console.log(
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
);
}
});
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
let message = '\n';
if (total === 0) {
message = 'No tests found!';
message += ` (${renderDuration(now())} ms) \n`;
this.write(message);
return;
}
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
message += failed > 0 ? red(` ${failed} failed`) : '';
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
this.write(message);
}
write(message: any) {
if (typeof message === 'object') {
message = JSON.stringify(message);
}
log(message + '\n');
}
}
const SEPARATOR = '\n';
function indent(lines: string, tab = ' ') {
return lines.replace(/^/gm, tab);
}
const NO_UTF8_SUPPORT = process.platform === 'win32';
const symbols = {
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
skipped: cyan('-'),
progress: cyan('>'),
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
};
function now() {
return performance.now();
}
function findPWLogsIndexes(msgOrStack: string): [number, number] {
let startIndex = 0;
let endIndex = 0;
if (!msgOrStack) {
return [startIndex, endIndex];
}
const lines = String(msgOrStack).split(SEPARATOR);
const logStart = /[=]{3,} logs [=]{3,}/;
const logEnd = /[=]{10,}/;
lines.forEach((line, index) => {
if (logStart.test(line)) {
startIndex = index;
} else if (logEnd.test(line)) {
endIndex = index;
}
});
return [startIndex, endIndex];
}
function rewriteErrorStack(stack: string, indexes: [number, number]) {
const [start, end] = indexes;
/**
* Do not rewrite if its not a playwright error
*/
if (start === 0 && end === 0) {
return stack;
}
const linesToKeep = start + 3;
if (start > 0 && linesToKeep < end) {
const lines = stack.split(SEPARATOR);
return lines
.slice(0, linesToKeep)
.concat(...lines.slice(end))
.join(SEPARATOR);
}
return stack;
}

View file

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Path from 'path';
import { execSync } from 'child_process';
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
export const esArchiverLoad = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../../scripts/es_archiver load "${path}" --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverUnload = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../../scripts/es_archiver unload "${path}" --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverResetKibana = () => {
execSync(
`node ../../../../../scripts/es_archiver empty-kibana-index --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};

View file

@ -1,22 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import path from 'path';
import fs from 'fs';
import yaml from 'js-yaml';
export type KibanaConfig = ReturnType<typeof readKibanaConfig>;
export const readKibanaConfig = () => {
const kibanaConfigDir = path.join(__filename, '../../../../../../../config');
const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml');
const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml');
return (yaml.load(
fs.readFileSync(fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, 'utf8')
) || {}) as Record<string, string>;
};

View file

@ -13,7 +13,7 @@
"@kbn/ux-plugin/e2e",
"@kbn/ftr-common-functional-services",
"@kbn/apm-plugin",
"@kbn/es-archiver",
"@kbn/ftr-common-functional-ui-services"
"@kbn/ftr-common-functional-ui-services",
"@kbn/observability-synthetics-test-data"
]
}

View file

@ -6,13 +6,13 @@
*/
import { journey, step, before } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import {
assertNotText,
assertText,
byTestId,
waitForLoadingToFinish,
} from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { settingsPageProvider } from '../../page_objects/settings';

View file

@ -7,8 +7,8 @@
import { journey, step, expect, before } from '@elastic/synthetics';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, assertText, waitForLoadingToFinish } from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { loginPageProvider } from '../../../page_objects/login';
journey('StatusFlyoutInAlertingApp', async ({ page, params }) => {

View file

@ -6,8 +6,8 @@
*/
import { journey, step, before, expect } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, assertText, waitForLoadingToFinish } from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { loginPageProvider } from '../../../page_objects/login';
journey('TlsFlyoutInAlertingApp', async ({ page, params }) => {

View file

@ -7,8 +7,8 @@
import { journey, step, expect, before } from '@elastic/synthetics';
import { callKibana } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/helpers/call_kibana';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../../helpers/utils';
import { recordVideo } from '../../helpers/record_video';
import { loginPageProvider } from '../../page_objects/login';
journey('DataViewPermissions', async ({ page, params }) => {

View file

@ -6,7 +6,7 @@
*/
import { journey, step, before, Page } from '@elastic/synthetics';
import { recordVideo } from '../../../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { makeChecksWithStatus } from '../../../helpers/make_checks';
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';

View file

@ -7,8 +7,8 @@
import { journey, step, expect, before, Page } from '@elastic/synthetics';
import { noop } from 'lodash';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, delay } from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
const dateRangeStart = '2019-09-10T12:40:08.078Z';

View file

@ -6,8 +6,8 @@
*/
import { journey, step, before, Page } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId } from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
const dateRangeStart = '2019-09-10T12:40:08.078Z';

View file

@ -6,8 +6,8 @@
*/
import { journey, step, expect, before, Page } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, delay } from '../../../helpers/utils';
import { recordVideo } from '../../../helpers/record_video';
import { makeChecksWithStatus } from '../../../helpers/make_checks';
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';

View file

@ -7,7 +7,7 @@
import { journey, step, expect } from '@elastic/synthetics';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { recordVideo } from '../../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { loginPageProvider } from '../../page_objects/login';
journey('StepsDuration', async ({ page, params }) => {

View file

@ -6,8 +6,8 @@
*/
import { journey, step, before } from '@elastic/synthetics';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { byTestId, waitForLoadingToFinish } from '../../helpers/utils';
import { recordVideo } from '../../helpers/record_video';
journey('UptimeOverview', ({ page, params }) => {
recordVideo(page);

View file

@ -6,8 +6,7 @@
*/
import { FtrConfigProviderContext } from '@kbn/test';
import path from 'path';
import { argv } from '../helpers/parse_args_params';
import { SyntheticsRunner } from '../helpers/synthetics_runner';
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
const { headless, grep, bail: pauseOnError } = argv;

View file

@ -1,33 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import yargs from 'yargs';
const { argv } = yargs(process.argv.slice(2))
.option('headless', {
default: true,
type: 'boolean',
description: 'Start in headless mode',
})
.option('bail', {
default: false,
type: 'boolean',
description: 'Pause on error',
})
.option('watch', {
default: false,
type: 'boolean',
description: 'Runs the server in watch mode, restarting on changes',
})
.option('grep', {
default: undefined,
type: 'string',
description: 'run only journeys with a name or tags that matches the glob',
})
.help();
export { argv };

View file

@ -1,32 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fs from 'fs';
import Runner from '@elastic/synthetics/dist/core/runner';
import { after, Page } from '@elastic/synthetics';
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
// @ts-ignore
export const runner: Runner = global[SYNTHETICS_RUNNER];
export const recordVideo = (page: Page, postfix = '') => {
after(async () => {
try {
const videoFilePath = await page.video()?.path();
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
const newVideoPath = videoFilePath?.replace(
pathToVideo!,
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
);
fs.renameSync(videoFilePath!, newVideoPath!);
} catch (e) {
// eslint-disable-next-line no-console
console.log('Error while renaming video file', e);
}
});
};

View file

@ -1,159 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable no-console */
import Url from 'url';
import { run as syntheticsRun } from '@elastic/synthetics';
import { PromiseType } from 'utility-types';
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
import { EsArchiver } from '@kbn/es-archiver';
import { esArchiverUnload } from '../tasks/es_archiver';
import { TestReporter } from './test_reporter';
export interface ArgParams {
headless: boolean;
match?: string;
pauseOnError: boolean;
}
export class SyntheticsRunner {
public getService: any;
public kibanaUrl: string;
private elasticsearchUrl: string;
public testFilesLoaded: boolean = false;
public params: ArgParams;
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
constructor(getService: any, params: ArgParams) {
this.getService = getService;
this.kibanaUrl = this.getKibanaUrl();
this.elasticsearchUrl = this.getElasticsearchUrl();
this.params = params;
}
async setup() {
await this.createTestUsers();
}
async createTestUsers() {
await createApmUsers({
elasticsearch: {
node: this.elasticsearchUrl,
username: 'elastic',
password: 'changeme',
},
kibana: { hostname: this.kibanaUrl },
});
}
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
console.log('Loading test files');
await callback(reload);
this.loadTestFilesCallback = callback;
this.testFilesLoaded = true;
console.log('Successfully loaded test files');
}
async loadTestData(e2eDir: string, dataArchives: string[]) {
try {
console.log('Loading esArchiver...');
const esArchiver: EsArchiver = this.getService('esArchiver');
const promises = dataArchives.map((archive) => {
if (archive === 'synthetics_data') {
return esArchiver.load(e2eDir + archive, {
docsOnly: true,
skipExisting: true,
});
}
return esArchiver.load(e2eDir + archive, { skipExisting: true });
});
await Promise.all([...promises]);
} catch (e) {
console.log(e);
}
}
getKibanaUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.kibana.protocol'),
hostname: config.get('servers.kibana.hostname'),
port: config.get('servers.kibana.port'),
});
}
getElasticsearchUrl() {
const config = this.getService('config');
return Url.format({
protocol: config.get('servers.elasticsearch.protocol'),
hostname: config.get('servers.elasticsearch.hostname'),
port: config.get('servers.elasticsearch.port'),
});
}
async run() {
if (!this.testFilesLoaded) {
throw new Error('Test files not loaded');
}
const { headless, match, pauseOnError } = this.params;
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
console.log(`Running ${noOfRuns} times`);
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
for (let i = 0; i < noOfRuns; i++) {
results = await syntheticsRun({
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
playwrightOptions: {
headless,
chromiumSandbox: false,
timeout: 60 * 1000,
viewport: {
height: 900,
width: 1600,
},
recordVideo: {
dir: '.journeys/videos',
},
},
grepOpts: { match: match === 'undefined' ? '' : match },
pauseOnError,
screenshots: 'only-on-failure',
reporter: TestReporter,
});
if (noOfRuns > 1) {
// need to reload again since runner resets the journeys
await this.loadTestFiles(this.loadTestFilesCallback!, true);
}
}
await this.assertResults(results);
}
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
Object.entries(results).forEach(([_journey, result]) => {
if (result.status !== 'succeeded') {
process.exitCode = 1;
process.exit();
}
});
}
cleanUp() {
console.log('Removing esArchiver...');
esArchiverUnload('full_heartbeat');
esArchiverUnload('browser');
}
}

View file

@ -1,229 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
import { Reporter, ReporterOptions } from '@elastic/synthetics';
import {
JourneyEndResult,
JourneyStartResult,
StepEndResult,
} from '@elastic/synthetics/dist/common_types';
import { yellow, green, cyan, red, bold } from 'chalk';
// eslint-disable-next-line no-console
const log = console.log;
import { performance } from 'perf_hooks';
import * as fs from 'fs';
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
import { join } from 'path';
function renderError(error: any) {
let output = '';
const outer = indent('');
const inner = indent(outer);
const container = outer + '---\n';
output += container;
let stack = error.stack;
if (stack) {
output += inner + 'stack: |-\n';
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
const lines = String(stack).split('\n');
for (const line of lines) {
output += inner + ' ' + line + '\n';
}
}
output += container;
return red(output);
}
function renderDuration(durationMs: number) {
return Number(durationMs).toFixed(0);
}
export class TestReporter implements Reporter {
metrics = {
succeeded: 0,
failed: 0,
skipped: 0,
};
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
constructor(options: ReporterOptions = {}) {}
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
if (process.env.CI) {
this.write(`\n--- Journey: ${journey.name}`);
} else {
this.write(bold(`\n Journey: ${journey.name}`));
}
}
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
const { status, end, start, error } = result;
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
this.metrics[status]++;
if (!this.journeys.has(journey.name)) {
this.journeys.set(journey.name, []);
}
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
}
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
if (total === 0 && error) {
this.write(renderError(error));
}
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
this.write(message);
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
const { data, step } = screenshot;
if (status === 'failed') {
await (async () => {
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
encoding: 'base64',
});
})();
}
});
}
onEnd() {
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.some((step) => step.status === 'failed')
);
if (failedJourneys.length > 0) {
failedJourneys.forEach(([journeyName, steps]) => {
if (process.env.CI) {
const name = red(`Journey: ${journeyName} 🥵`);
this.write(`\n+++ ${name}`);
steps.forEach((stepResult) => {
const { status, end, start, error, name: stepName } = stepResult;
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
(end - start) * 1000
)} ms)`;
this.write(indent(message));
if (error) {
this.write(renderError(error));
}
});
}
});
}
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
steps.every((step) => step.status === 'succeeded')
);
successfulJourneys.forEach(([journeyName, steps]) => {
try {
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
} catch (e) {
// eslint-disable-next-line no-console
console.log(
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
);
}
});
const { failed, succeeded, skipped } = this.metrics;
const total = failed + succeeded + skipped;
let message = '\n';
if (total === 0) {
message = 'No tests found!';
message += ` (${renderDuration(now())} ms) \n`;
this.write(message);
return;
}
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
message += failed > 0 ? red(` ${failed} failed`) : '';
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
this.write(message);
}
write(message: any) {
if (typeof message === 'object') {
message = JSON.stringify(message);
}
log(message + '\n');
}
}
const SEPARATOR = '\n';
function indent(lines: string, tab = ' ') {
return lines.replace(/^/gm, tab);
}
const NO_UTF8_SUPPORT = process.platform === 'win32';
const symbols = {
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
skipped: cyan('-'),
progress: cyan('>'),
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
};
function now() {
return performance.now();
}
function findPWLogsIndexes(msgOrStack: string): [number, number] {
let startIndex = 0;
let endIndex = 0;
if (!msgOrStack) {
return [startIndex, endIndex];
}
const lines = String(msgOrStack).split(SEPARATOR);
const logStart = /[=]{3,} logs [=]{3,}/;
const logEnd = /[=]{10,}/;
lines.forEach((line, index) => {
if (logStart.test(line)) {
startIndex = index;
} else if (logEnd.test(line)) {
endIndex = index;
}
});
return [startIndex, endIndex];
}
function rewriteErrorStack(stack: string, indexes: [number, number]) {
const [start, end] = indexes;
/**
* Do not rewrite if its not a playwright error
*/
if (start === 0 && end === 0) {
return stack;
}
const linesToKeep = start + 3;
if (start > 0 && linesToKeep < end) {
const lines = stack.split(SEPARATOR);
return lines
.slice(0, linesToKeep)
.concat(...lines.slice(end))
.join(SEPARATOR);
}
return stack;
}

View file

@ -6,7 +6,7 @@
*/
import { journey, step, expect, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -7,7 +7,7 @@
import { journey, step, expect, before } from '@elastic/synthetics';
import { Client } from '@elastic/elasticsearch';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { loginToKibana, waitForLoadingToFinish } from './utils';
const addTestTransaction = async (params: any) => {

View file

@ -6,7 +6,7 @@
*/
import { journey, step, expect, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, expect, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, expect, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, expect, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, before, expect } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,7 +6,7 @@
*/
import { journey, step, before } from '@elastic/synthetics';
import { recordVideo } from '../helpers/record_video';
import { recordVideo } from '@kbn/observability-synthetics-test-data';
import { UXDashboardDatePicker } from '../page_objects/date_picker';
import { byLensTestId, loginToKibana, waitForLoadingToFinish } from './utils';

View file

@ -6,8 +6,7 @@
*/
import { FtrConfigProviderContext } from '@kbn/test';
import path from 'path';
import { argv } from './helpers/parse_args_params';
import { SyntheticsRunner } from './helpers/synthetics_runner';
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
const { headless, grep, bail: pauseOnError } = argv;

View file

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Path from 'path';
import { execSync } from 'child_process';
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
export const esArchiverLoad = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../../scripts/es_archiver load "${path}" --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverUnload = (folder: string) => {
const path = Path.join(ES_ARCHIVE_DIR, folder);
execSync(
`node ../../../../../scripts/es_archiver unload "${path}" --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};
export const esArchiverResetKibana = () => {
execSync(
`node ../../../../../scripts/es_archiver empty-kibana-index --config ../../../../test/functional/config.base.js`,
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
);
};

View file

@ -6,5 +6,8 @@
"outDir": "target/types",
"types": ["node"]
},
"kbn_references": ["@kbn/test", "@kbn/apm-plugin", "@kbn/es-archiver"]
"kbn_references": [
"@kbn/test",
"@kbn/observability-synthetics-test-data",
]
}