mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[Synthetics] Clean up e2e test helpers !! (#203812)
## Summary Clean up e2e test helpers
This commit is contained in:
parent
f4795cdcd7
commit
0203bba44f
64 changed files with 129 additions and 1541 deletions
|
@ -6,3 +6,4 @@
|
|||
*/
|
||||
|
||||
export { makeUpSummary, makeDownSummary } from './src/make_summaries';
|
||||
export * from './src/e2e';
|
||||
|
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { expect, Page } from '@elastic/synthetics';
|
||||
|
||||
export async function waitForLoadingToFinish({ page }: { page: Page }) {
|
||||
while (true) {
|
||||
if (!(await page.isVisible(byTestId('kbnLoadingMessage'), { timeout: 5000 }))) break;
|
||||
await page.waitForTimeout(1000);
|
||||
}
|
||||
}
|
||||
|
||||
export async function loginToKibana({
|
||||
page,
|
||||
user,
|
||||
}: {
|
||||
page: Page;
|
||||
user?: { username: string; password: string };
|
||||
}) {
|
||||
await page.fill('[data-test-subj=loginUsername]', user?.username ?? 'elastic', {
|
||||
timeout: 60 * 1000,
|
||||
});
|
||||
|
||||
await page.fill('[data-test-subj=loginPassword]', user?.password ?? 'changeme');
|
||||
|
||||
await page.click('[data-test-subj=loginSubmit]');
|
||||
|
||||
await waitForLoadingToFinish({ page });
|
||||
}
|
||||
|
||||
export const byTestId = (testId: string) => {
|
||||
return `[data-test-subj=${testId}]`;
|
||||
};
|
||||
|
||||
export const assertText = async ({ page, text }: { page: Page; text: string }) => {
|
||||
const element = await page.waitForSelector(`text=${text}`);
|
||||
expect(await element.isVisible()).toBeTruthy();
|
||||
};
|
||||
|
||||
export const assertNotText = async ({ page, text }: { page: Page; text: string }) => {
|
||||
expect(await page.$(`text=${text}`)).toBeFalsy();
|
||||
};
|
||||
|
||||
export const getQuerystring = (params: object) => {
|
||||
return Object.entries(params)
|
||||
.map(([key, value]) => encodeURIComponent(key) + '=' + encodeURIComponent(value))
|
||||
.join('&');
|
||||
};
|
||||
|
||||
export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export const TIMEOUT_60_SEC = {
|
||||
timeout: 60 * 1000,
|
||||
};
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { recordVideo } from './helpers/record_video';
|
||||
export { SyntheticsRunner } from './helpers/synthetics_runner';
|
||||
export { argv } from './helpers/parse_args_params';
|
||||
export { readKibanaConfig } from './tasks/read_kibana_config';
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
import Path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
|
||||
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
|
||||
|
||||
|
@ -16,7 +17,7 @@ const NODE_TLS_REJECT_UNAUTHORIZED = '1';
|
|||
export const esArchiverLoad = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver load "${path}" --config ../../../test/functional/config.base.js`,
|
||||
`node ${REPO_ROOT}/scripts/es_archiver load "${path}" --config ${REPO_ROOT}/test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
@ -24,14 +25,7 @@ export const esArchiverLoad = (folder: string) => {
|
|||
export const esArchiverUnload = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver unload "${path}" --config ../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverResetKibana = () => {
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver empty-kibana-index --config ../../../test/functional/config.base.js`,
|
||||
`node ${REPO_ROOT}/scripts/es_archiver unload "${path}" --config ${REPO_ROOT}/test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
|
@ -16,5 +16,8 @@
|
|||
"target/**/*"
|
||||
],
|
||||
"kbn_references": [
|
||||
"@kbn/apm-plugin",
|
||||
"@kbn/es-archiver",
|
||||
"@kbn/repo-info",
|
||||
]
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
|
||||
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
|
||||
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
|
||||
|
||||
|
|
|
@ -6,10 +6,10 @@
|
|||
*/
|
||||
|
||||
import { journey, step } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import moment from 'moment';
|
||||
import { recordVideo } from '../record_video';
|
||||
import { createExploratoryViewUrl } from '../../public/components/shared/exploratory_view/configurations/exploratory_view_url';
|
||||
import { byTestId, loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
|
||||
import { loginToKibana, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../utils';
|
||||
|
||||
journey('Step Duration series', async ({ page, params }) => {
|
||||
recordVideo(page);
|
||||
|
@ -56,7 +56,8 @@ journey('Step Duration series', async ({ page, params }) => {
|
|||
await page.click('[aria-label="Remove report metric"]');
|
||||
await page.click('button:has-text("Select report metric")');
|
||||
await page.click('button:has-text("Step duration")');
|
||||
await page.click(byTestId('seriesBreakdown'));
|
||||
await page.waitForSelector('[data-test-subj=seriesBreakdown]');
|
||||
await page.getByTestId('seriesBreakdown').click();
|
||||
await page.click('button[role="option"]:has-text("Step name")');
|
||||
await page.click('.euiComboBox__inputWrap');
|
||||
await page.click('[role="combobox"][placeholder="Search Monitor name"]');
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import yargs from 'yargs';
|
||||
|
||||
const { argv } = yargs(process.argv.slice(2))
|
||||
.option('headless', {
|
||||
default: true,
|
||||
type: 'boolean',
|
||||
description: 'Start in headless mode',
|
||||
})
|
||||
.option('bail', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
description: 'Pause on error',
|
||||
})
|
||||
.option('watch', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
description: 'Runs the server in watch mode, restarting on changes',
|
||||
})
|
||||
.option('grep', {
|
||||
default: undefined,
|
||||
type: 'string',
|
||||
description: 'run only journeys with a name or tags that matches the glob',
|
||||
})
|
||||
.help();
|
||||
|
||||
export { argv };
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import Runner from '@elastic/synthetics/dist/core/runner';
|
||||
import { after, Page } from '@elastic/synthetics';
|
||||
|
||||
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
|
||||
|
||||
// @ts-ignore
|
||||
export const runner: Runner = global[SYNTHETICS_RUNNER];
|
||||
|
||||
export const recordVideo = (page: Page, postfix = '') => {
|
||||
after(async () => {
|
||||
try {
|
||||
const videoFilePath = await page.video()?.path();
|
||||
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
|
||||
const newVideoPath = videoFilePath?.replace(
|
||||
pathToVideo!,
|
||||
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
|
||||
);
|
||||
fs.renameSync(videoFilePath!, newVideoPath!);
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Error while renaming video file', e);
|
||||
}
|
||||
});
|
||||
};
|
|
@ -6,8 +6,8 @@
|
|||
*/
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import path from 'path';
|
||||
import { SyntheticsRunner } from './synthetics_runner';
|
||||
import { argv } from './parse_args_params';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
const { headless, grep, bail: pauseOnError } = argv;
|
||||
|
||||
|
@ -24,13 +24,12 @@ async function runE2ETests({ readConfigFile }: FtrConfigProviderContext) {
|
|||
});
|
||||
|
||||
await syntheticsRunner.setup();
|
||||
|
||||
await syntheticsRunner.loadTestData(path.join(__dirname, '../../ux/e2e/fixtures/'), [
|
||||
'rum_8.0.0',
|
||||
'rum_test_data',
|
||||
]);
|
||||
await syntheticsRunner.loadTestData(
|
||||
path.join(__dirname, '../../synthetics/e2e/fixtures/es_archiver/'),
|
||||
`${REPO_ROOT}/x-pack/plugins/observability_solution/ux/e2e/fixtures/`,
|
||||
['rum_8.0.0', 'rum_test_data']
|
||||
);
|
||||
await syntheticsRunner.loadTestData(
|
||||
`${REPO_ROOT}/x-pack/plugins/observability_solution/synthetics/e2e/fixtures/es_archiver/`,
|
||||
['full_heartbeat', 'browser']
|
||||
);
|
||||
await syntheticsRunner.loadTestFiles(async () => {
|
||||
|
|
|
@ -1,155 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import Url from 'url';
|
||||
import { run as syntheticsRun } from '@elastic/synthetics';
|
||||
import { PromiseType } from 'utility-types';
|
||||
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
|
||||
|
||||
import { EsArchiver } from '@kbn/es-archiver';
|
||||
import { esArchiverUnload } from './tasks/es_archiver';
|
||||
import { TestReporter } from './test_reporter';
|
||||
|
||||
export interface ArgParams {
|
||||
headless: boolean;
|
||||
match?: string;
|
||||
pauseOnError: boolean;
|
||||
}
|
||||
|
||||
export class SyntheticsRunner {
|
||||
public getService: any;
|
||||
public kibanaUrl: string;
|
||||
private elasticsearchUrl: string;
|
||||
|
||||
public testFilesLoaded: boolean = false;
|
||||
|
||||
public params: ArgParams;
|
||||
|
||||
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
|
||||
|
||||
constructor(getService: any, params: ArgParams) {
|
||||
this.getService = getService;
|
||||
this.kibanaUrl = this.getKibanaUrl();
|
||||
this.elasticsearchUrl = this.getElasticsearchUrl();
|
||||
this.params = params;
|
||||
}
|
||||
|
||||
async setup() {
|
||||
await this.createTestUsers();
|
||||
}
|
||||
|
||||
async createTestUsers() {
|
||||
await createApmUsers({
|
||||
elasticsearch: { node: this.elasticsearchUrl, username: 'elastic', password: 'changeme' },
|
||||
kibana: { hostname: this.kibanaUrl },
|
||||
});
|
||||
}
|
||||
|
||||
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
|
||||
console.log('Loading test files');
|
||||
await callback(reload);
|
||||
this.loadTestFilesCallback = callback;
|
||||
this.testFilesLoaded = true;
|
||||
console.log('Successfully loaded test files');
|
||||
}
|
||||
|
||||
async loadTestData(e2eDir: string, dataArchives: string[]) {
|
||||
try {
|
||||
console.log('Loading esArchiver...');
|
||||
|
||||
const esArchiver: EsArchiver = this.getService('esArchiver');
|
||||
|
||||
const promises = dataArchives.map((archive) => {
|
||||
if (archive === 'synthetics_data') {
|
||||
return esArchiver.load(e2eDir + archive, {
|
||||
docsOnly: true,
|
||||
skipExisting: true,
|
||||
});
|
||||
}
|
||||
return esArchiver.load(e2eDir + archive, { skipExisting: true });
|
||||
});
|
||||
|
||||
await Promise.all([...promises]);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
|
||||
getKibanaUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.kibana.protocol'),
|
||||
hostname: config.get('servers.kibana.hostname'),
|
||||
port: config.get('servers.kibana.port'),
|
||||
});
|
||||
}
|
||||
|
||||
getElasticsearchUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.elasticsearch.protocol'),
|
||||
hostname: config.get('servers.elasticsearch.hostname'),
|
||||
port: config.get('servers.elasticsearch.port'),
|
||||
});
|
||||
}
|
||||
|
||||
async run() {
|
||||
if (!this.testFilesLoaded) {
|
||||
throw new Error('Test files not loaded');
|
||||
}
|
||||
const { headless, match, pauseOnError } = this.params;
|
||||
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
|
||||
console.log(`Running ${noOfRuns} times`);
|
||||
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
|
||||
for (let i = 0; i < noOfRuns; i++) {
|
||||
results = await syntheticsRun({
|
||||
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
|
||||
playwrightOptions: {
|
||||
headless,
|
||||
chromiumSandbox: false,
|
||||
timeout: 60 * 1000,
|
||||
viewport: {
|
||||
height: 900,
|
||||
width: 1600,
|
||||
},
|
||||
recordVideo: {
|
||||
dir: '.journeys/videos',
|
||||
},
|
||||
},
|
||||
grepOpts: { match: match === 'undefined' ? '' : match },
|
||||
pauseOnError,
|
||||
screenshots: 'only-on-failure',
|
||||
reporter: TestReporter,
|
||||
});
|
||||
if (noOfRuns > 1) {
|
||||
// need to reload again since runner resets the journeys
|
||||
await this.loadTestFiles(this.loadTestFilesCallback!, true);
|
||||
}
|
||||
}
|
||||
|
||||
await this.assertResults(results);
|
||||
}
|
||||
|
||||
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
|
||||
Object.entries(results).forEach(([_journey, result]) => {
|
||||
if (result.status !== 'succeeded') {
|
||||
process.exitCode = 1;
|
||||
process.exit();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cleanUp() {
|
||||
console.log('Removing esArchiver...');
|
||||
esArchiverUnload('full_heartbeat');
|
||||
esArchiverUnload('browser');
|
||||
}
|
||||
}
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
|
||||
|
||||
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
|
||||
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
|
||||
|
||||
export const esArchiverLoad = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver load "${path}" --config ../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverUnload = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver unload "${path}" --config ../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverResetKibana = () => {
|
||||
execSync(
|
||||
`node ../../../../scripts/es_archiver empty-kibana-index --config ../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
|
@ -6,5 +6,9 @@
|
|||
"outDir": "target/types",
|
||||
"types": ["node"]
|
||||
},
|
||||
"kbn_references": ["@kbn/test", "@kbn/apm-plugin", "@kbn/es-archiver"]
|
||||
"kbn_references": [
|
||||
"@kbn/test",
|
||||
"@kbn/repo-info",
|
||||
"@kbn/observability-synthetics-test-data",
|
||||
]
|
||||
}
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
import { Page } from '@elastic/synthetics';
|
||||
import { loginPageProvider } from '@kbn/synthetics-e2e/page_objects/login';
|
||||
import { utilsPageProvider } from '@kbn/synthetics-e2e/page_objects/utils';
|
||||
import { recordVideo } from '@kbn/synthetics-e2e/helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
export function sloAppPageProvider({ page, kibanaUrl }: { page: Page; kibanaUrl: string }) {
|
||||
page.setDefaultTimeout(60 * 1000);
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import { SyntheticsRunner, argv } from '@kbn/synthetics-e2e';
|
||||
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
const { headless, grep, bail: pauseOnError } = argv;
|
||||
|
||||
|
|
|
@ -12,5 +12,6 @@
|
|||
"@kbn/ftr-common-functional-services",
|
||||
"@kbn/data-forge",
|
||||
"@kbn/synthetics-e2e",
|
||||
"@kbn/observability-synthetics-test-data",
|
||||
]
|
||||
}
|
||||
|
|
|
@ -10,8 +10,8 @@ import { CA_CERT_PATH } from '@kbn/dev-utils';
|
|||
import { get } from 'lodash';
|
||||
import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
|
||||
import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services';
|
||||
import { readKibanaConfig } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
import { readKibanaConfig } from './tasks/read_kibana_config';
|
||||
const MANIFEST_KEY = 'xpack.uptime.service.manifestUrl';
|
||||
const SERVICE_PASSWORD = 'xpack.uptime.service.password';
|
||||
const SERVICE_USERNAME = 'xpack.uptime.service.username';
|
||||
|
|
|
@ -1,229 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
|
||||
import { Reporter, ReporterOptions } from '@elastic/synthetics';
|
||||
import {
|
||||
JourneyEndResult,
|
||||
JourneyStartResult,
|
||||
StepEndResult,
|
||||
} from '@elastic/synthetics/dist/common_types';
|
||||
|
||||
import { yellow, green, cyan, red, bold } from 'chalk';
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
const log = console.log;
|
||||
|
||||
import { performance } from 'perf_hooks';
|
||||
import * as fs from 'fs';
|
||||
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
|
||||
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
|
||||
import { join } from 'path';
|
||||
|
||||
function renderError(error: any) {
|
||||
let output = '';
|
||||
const outer = indent('');
|
||||
const inner = indent(outer);
|
||||
const container = outer + '---\n';
|
||||
output += container;
|
||||
let stack = error.stack;
|
||||
if (stack) {
|
||||
output += inner + 'stack: |-\n';
|
||||
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
|
||||
const lines = String(stack).split('\n');
|
||||
for (const line of lines) {
|
||||
output += inner + ' ' + line + '\n';
|
||||
}
|
||||
}
|
||||
output += container;
|
||||
return red(output);
|
||||
}
|
||||
|
||||
function renderDuration(durationMs: number) {
|
||||
return Number(durationMs).toFixed(0);
|
||||
}
|
||||
|
||||
export class TestReporter implements Reporter {
|
||||
metrics = {
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
|
||||
|
||||
constructor(options: ReporterOptions = {}) {}
|
||||
|
||||
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
|
||||
if (process.env.CI) {
|
||||
this.write(`\n--- Journey: ${journey.name}`);
|
||||
} else {
|
||||
this.write(bold(`\n Journey: ${journey.name}`));
|
||||
}
|
||||
}
|
||||
|
||||
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
|
||||
const { status, end, start, error } = result;
|
||||
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
this.metrics[status]++;
|
||||
if (!this.journeys.has(journey.name)) {
|
||||
this.journeys.set(journey.name, []);
|
||||
}
|
||||
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
|
||||
}
|
||||
|
||||
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
if (total === 0 && error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
|
||||
this.write(message);
|
||||
|
||||
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
|
||||
|
||||
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
|
||||
const { data, step } = screenshot;
|
||||
|
||||
if (status === 'failed') {
|
||||
await (async () => {
|
||||
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
|
||||
encoding: 'base64',
|
||||
});
|
||||
})();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onEnd() {
|
||||
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.some((step) => step.status === 'failed')
|
||||
);
|
||||
|
||||
if (failedJourneys.length > 0) {
|
||||
failedJourneys.forEach(([journeyName, steps]) => {
|
||||
if (process.env.CI) {
|
||||
const name = red(`Journey: ${journeyName} 🥵`);
|
||||
this.write(`\n+++ ${name}`);
|
||||
steps.forEach((stepResult) => {
|
||||
const { status, end, start, error, name: stepName } = stepResult;
|
||||
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.every((step) => step.status === 'succeeded')
|
||||
);
|
||||
|
||||
successfulJourneys.forEach(([journeyName, steps]) => {
|
||||
try {
|
||||
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
|
||||
let message = '\n';
|
||||
if (total === 0) {
|
||||
message = 'No tests found!';
|
||||
message += ` (${renderDuration(now())} ms) \n`;
|
||||
this.write(message);
|
||||
return;
|
||||
}
|
||||
|
||||
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
|
||||
message += failed > 0 ? red(` ${failed} failed`) : '';
|
||||
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
|
||||
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
|
||||
this.write(message);
|
||||
}
|
||||
|
||||
write(message: any) {
|
||||
if (typeof message === 'object') {
|
||||
message = JSON.stringify(message);
|
||||
}
|
||||
log(message + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
const SEPARATOR = '\n';
|
||||
|
||||
function indent(lines: string, tab = ' ') {
|
||||
return lines.replace(/^/gm, tab);
|
||||
}
|
||||
|
||||
const NO_UTF8_SUPPORT = process.platform === 'win32';
|
||||
const symbols = {
|
||||
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
|
||||
skipped: cyan('-'),
|
||||
progress: cyan('>'),
|
||||
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
|
||||
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
|
||||
};
|
||||
|
||||
function now() {
|
||||
return performance.now();
|
||||
}
|
||||
|
||||
function findPWLogsIndexes(msgOrStack: string): [number, number] {
|
||||
let startIndex = 0;
|
||||
let endIndex = 0;
|
||||
if (!msgOrStack) {
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
const lines = String(msgOrStack).split(SEPARATOR);
|
||||
const logStart = /[=]{3,} logs [=]{3,}/;
|
||||
const logEnd = /[=]{10,}/;
|
||||
lines.forEach((line, index) => {
|
||||
if (logStart.test(line)) {
|
||||
startIndex = index;
|
||||
} else if (logEnd.test(line)) {
|
||||
endIndex = index;
|
||||
}
|
||||
});
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
|
||||
function rewriteErrorStack(stack: string, indexes: [number, number]) {
|
||||
const [start, end] = indexes;
|
||||
/**
|
||||
* Do not rewrite if its not a playwright error
|
||||
*/
|
||||
if (start === 0 && end === 0) {
|
||||
return stack;
|
||||
}
|
||||
const linesToKeep = start + 3;
|
||||
if (start > 0 && linesToKeep < end) {
|
||||
const lines = stack.split(SEPARATOR);
|
||||
return lines
|
||||
.slice(0, linesToKeep)
|
||||
.concat(...lines.slice(end))
|
||||
.join(SEPARATOR);
|
||||
}
|
||||
return stack;
|
||||
}
|
|
@ -5,7 +5,5 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export { SyntheticsRunner } from './helpers/synthetics_runner';
|
||||
export { argv } from './helpers/parse_args_params';
|
||||
export { loginPageProvider } from './page_objects/login';
|
||||
export { utilsPageProvider } from './page_objects/utils';
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { journey, step, expect, Page } from '@elastic/synthetics';
|
||||
import { RetryService } from '@kbn/ftr-common-functional-services';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { syntheticsAppPageProvider } from '../page_objects/synthetics_app';
|
||||
import { byTestId, assertText } from '../../helpers/utils';
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { SYNTHETICS_API_URLS } from '@kbn/synthetics-plugin/common/constants';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
journey('ProjectAPIKeys', async ({ page }) => {
|
||||
recordVideo(page);
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
import { expect, Page } from '@elastic/synthetics';
|
||||
import { RetryService } from '@kbn/ftr-common-functional-services';
|
||||
import { FormMonitorType } from '@kbn/synthetics-plugin/common/runtime_types/monitor_management';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { loginPageProvider } from '../../page_objects/login';
|
||||
import { utilsPageProvider } from '../../page_objects/utils';
|
||||
|
||||
|
|
|
@ -4,10 +4,9 @@
|
|||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import path from 'path';
|
||||
import { argv } from '../helpers/parse_args_params';
|
||||
import { SyntheticsRunner } from '../helpers/synthetics_runner';
|
||||
|
||||
const { headless, grep, bail: pauseOnError } = argv;
|
||||
|
||||
|
|
|
@ -12,10 +12,7 @@
|
|||
"@kbn/dev-utils",
|
||||
"@kbn/ux-plugin/e2e",
|
||||
"@kbn/ftr-common-functional-services",
|
||||
"@kbn/apm-plugin",
|
||||
"@kbn/es-archiver",
|
||||
"@kbn/synthetics-plugin",
|
||||
"@kbn/repo-info",
|
||||
"@kbn/observability-synthetics-test-data",
|
||||
"@kbn/ftr-common-functional-ui-services"
|
||||
]
|
||||
|
|
|
@ -9,7 +9,7 @@ import { FtrConfigProviderContext } from '@kbn/test';
|
|||
import { CA_CERT_PATH } from '@kbn/dev-utils';
|
||||
import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
|
||||
import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services';
|
||||
import { readKibanaConfig } from './tasks/read_kibana_config';
|
||||
import { readKibanaConfig } from '@kbn/observability-synthetics-test-data';
|
||||
const MANIFEST_KEY = 'xpack.uptime.service.manifestUrl';
|
||||
const SERVICE_PASSWORD = 'xpack.uptime.service.password';
|
||||
const SERVICE_USERNAME = 'xpack.uptime.service.username';
|
||||
|
|
|
@ -1,28 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import yargs from 'yargs';
|
||||
|
||||
const { argv } = yargs(process.argv.slice(2))
|
||||
.option('headless', {
|
||||
default: true,
|
||||
type: 'boolean',
|
||||
description: 'Start in headless mode',
|
||||
})
|
||||
.option('bail', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
description: 'Pause on error',
|
||||
})
|
||||
.option('grep', {
|
||||
default: undefined,
|
||||
type: 'string',
|
||||
description: 'run only journeys with a name or tags that matches the glob',
|
||||
})
|
||||
.help();
|
||||
|
||||
export { argv };
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import Runner from '@elastic/synthetics/dist/core/runner';
|
||||
import { after, Page } from '@elastic/synthetics';
|
||||
|
||||
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
|
||||
|
||||
// @ts-ignore
|
||||
export const runner: Runner = global[SYNTHETICS_RUNNER];
|
||||
|
||||
export const recordVideo = (page: Page, postfix = '') => {
|
||||
after(async () => {
|
||||
try {
|
||||
const videoFilePath = await page.video()?.path();
|
||||
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
|
||||
const newVideoPath = videoFilePath?.replace(
|
||||
pathToVideo!,
|
||||
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
|
||||
);
|
||||
fs.renameSync(videoFilePath!, newVideoPath!);
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Error while renaming video file', e);
|
||||
}
|
||||
});
|
||||
};
|
|
@ -1,155 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import Url from 'url';
|
||||
import { run as syntheticsRun } from '@elastic/synthetics';
|
||||
import { PromiseType } from 'utility-types';
|
||||
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
|
||||
|
||||
import { EsArchiver } from '@kbn/es-archiver';
|
||||
import { esArchiverUnload } from '../tasks/es_archiver';
|
||||
import { TestReporter } from './test_reporter';
|
||||
|
||||
export interface ArgParams {
|
||||
headless: boolean;
|
||||
match?: string;
|
||||
pauseOnError: boolean;
|
||||
}
|
||||
|
||||
export class SyntheticsRunner {
|
||||
public getService: any;
|
||||
public kibanaUrl: string;
|
||||
private elasticsearchUrl: string;
|
||||
|
||||
public testFilesLoaded: boolean = false;
|
||||
|
||||
public params: ArgParams;
|
||||
|
||||
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
|
||||
|
||||
constructor(getService: any, params: ArgParams) {
|
||||
this.getService = getService;
|
||||
this.kibanaUrl = this.getKibanaUrl();
|
||||
this.elasticsearchUrl = this.getElasticsearchUrl();
|
||||
this.params = params;
|
||||
}
|
||||
|
||||
async setup() {
|
||||
await this.createTestUsers();
|
||||
}
|
||||
|
||||
async createTestUsers() {
|
||||
await createApmUsers({
|
||||
elasticsearch: { node: this.elasticsearchUrl, username: 'elastic', password: 'changeme' },
|
||||
kibana: { hostname: this.kibanaUrl },
|
||||
});
|
||||
}
|
||||
|
||||
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
|
||||
console.log('Loading test files');
|
||||
await callback(reload);
|
||||
this.loadTestFilesCallback = callback;
|
||||
this.testFilesLoaded = true;
|
||||
console.log('Successfully loaded test files');
|
||||
}
|
||||
|
||||
async loadTestData(e2eDir: string, dataArchives: string[]) {
|
||||
try {
|
||||
console.log('Loading esArchiver...');
|
||||
|
||||
const esArchiver: EsArchiver = this.getService('esArchiver');
|
||||
|
||||
const promises = dataArchives.map((archive) => {
|
||||
if (archive === 'synthetics_data') {
|
||||
return esArchiver.load(e2eDir + archive, {
|
||||
docsOnly: true,
|
||||
skipExisting: true,
|
||||
});
|
||||
}
|
||||
return esArchiver.load(e2eDir + archive, { skipExisting: true });
|
||||
});
|
||||
|
||||
await Promise.all([...promises]);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
|
||||
getKibanaUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.kibana.protocol'),
|
||||
hostname: config.get('servers.kibana.hostname'),
|
||||
port: config.get('servers.kibana.port'),
|
||||
});
|
||||
}
|
||||
|
||||
getElasticsearchUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.elasticsearch.protocol'),
|
||||
hostname: config.get('servers.elasticsearch.hostname'),
|
||||
port: config.get('servers.elasticsearch.port'),
|
||||
});
|
||||
}
|
||||
|
||||
async run() {
|
||||
if (!this.testFilesLoaded) {
|
||||
throw new Error('Test files not loaded');
|
||||
}
|
||||
const { headless, match, pauseOnError } = this.params;
|
||||
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
|
||||
console.log(`Running ${noOfRuns} times`);
|
||||
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
|
||||
for (let i = 0; i < noOfRuns; i++) {
|
||||
results = await syntheticsRun({
|
||||
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
|
||||
playwrightOptions: {
|
||||
headless,
|
||||
chromiumSandbox: false,
|
||||
timeout: 60 * 1000,
|
||||
viewport: {
|
||||
height: 900,
|
||||
width: 1600,
|
||||
},
|
||||
recordVideo: {
|
||||
dir: '.journeys/videos',
|
||||
},
|
||||
},
|
||||
grepOpts: { match: match === 'undefined' ? '' : match },
|
||||
pauseOnError,
|
||||
screenshots: 'only-on-failure',
|
||||
reporter: TestReporter,
|
||||
});
|
||||
if (noOfRuns > 1) {
|
||||
// need to reload again since runner resets the journeys
|
||||
await this.loadTestFiles(this.loadTestFilesCallback!, true);
|
||||
}
|
||||
}
|
||||
|
||||
await this.assertResults(results);
|
||||
}
|
||||
|
||||
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
|
||||
Object.entries(results).forEach(([_journey, result]) => {
|
||||
if (result.status !== 'succeeded') {
|
||||
process.exitCode = 1;
|
||||
process.exit();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cleanUp() {
|
||||
console.log('Removing esArchiver...');
|
||||
esArchiverUnload('full_heartbeat');
|
||||
esArchiverUnload('browser');
|
||||
}
|
||||
}
|
|
@ -1,229 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
|
||||
import { Reporter, ReporterOptions } from '@elastic/synthetics';
|
||||
import {
|
||||
JourneyEndResult,
|
||||
JourneyStartResult,
|
||||
StepEndResult,
|
||||
} from '@elastic/synthetics/dist/common_types';
|
||||
|
||||
import { yellow, green, cyan, red, bold } from 'chalk';
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
const log = console.log;
|
||||
|
||||
import { performance } from 'perf_hooks';
|
||||
import * as fs from 'fs';
|
||||
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
|
||||
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
|
||||
import { join } from 'path';
|
||||
|
||||
function renderError(error: any) {
|
||||
let output = '';
|
||||
const outer = indent('');
|
||||
const inner = indent(outer);
|
||||
const container = outer + '---\n';
|
||||
output += container;
|
||||
let stack = error.stack;
|
||||
if (stack) {
|
||||
output += inner + 'stack: |-\n';
|
||||
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
|
||||
const lines = String(stack).split('\n');
|
||||
for (const line of lines) {
|
||||
output += inner + ' ' + line + '\n';
|
||||
}
|
||||
}
|
||||
output += container;
|
||||
return red(output);
|
||||
}
|
||||
|
||||
function renderDuration(durationMs: number) {
|
||||
return Number(durationMs).toFixed(0);
|
||||
}
|
||||
|
||||
export class TestReporter implements Reporter {
|
||||
metrics = {
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
|
||||
|
||||
constructor(options: ReporterOptions = {}) {}
|
||||
|
||||
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
|
||||
if (process.env.CI) {
|
||||
this.write(`\n--- Journey: ${journey.name}`);
|
||||
} else {
|
||||
this.write(bold(`\n Journey: ${journey.name}`));
|
||||
}
|
||||
}
|
||||
|
||||
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
|
||||
const { status, end, start, error } = result;
|
||||
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
this.metrics[status]++;
|
||||
if (!this.journeys.has(journey.name)) {
|
||||
this.journeys.set(journey.name, []);
|
||||
}
|
||||
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
|
||||
}
|
||||
|
||||
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
if (total === 0 && error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
|
||||
this.write(message);
|
||||
|
||||
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
|
||||
|
||||
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
|
||||
const { data, step } = screenshot;
|
||||
|
||||
if (status === 'failed') {
|
||||
await (async () => {
|
||||
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
|
||||
encoding: 'base64',
|
||||
});
|
||||
})();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onEnd() {
|
||||
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.some((step) => step.status === 'failed')
|
||||
);
|
||||
|
||||
if (failedJourneys.length > 0) {
|
||||
failedJourneys.forEach(([journeyName, steps]) => {
|
||||
if (process.env.CI) {
|
||||
const name = red(`Journey: ${journeyName} 🥵`);
|
||||
this.write(`\n+++ ${name}`);
|
||||
steps.forEach((stepResult) => {
|
||||
const { status, end, start, error, name: stepName } = stepResult;
|
||||
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.every((step) => step.status === 'succeeded')
|
||||
);
|
||||
|
||||
successfulJourneys.forEach(([journeyName, steps]) => {
|
||||
try {
|
||||
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
|
||||
let message = '\n';
|
||||
if (total === 0) {
|
||||
message = 'No tests found!';
|
||||
message += ` (${renderDuration(now())} ms) \n`;
|
||||
this.write(message);
|
||||
return;
|
||||
}
|
||||
|
||||
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
|
||||
message += failed > 0 ? red(` ${failed} failed`) : '';
|
||||
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
|
||||
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
|
||||
this.write(message);
|
||||
}
|
||||
|
||||
write(message: any) {
|
||||
if (typeof message === 'object') {
|
||||
message = JSON.stringify(message);
|
||||
}
|
||||
log(message + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
const SEPARATOR = '\n';
|
||||
|
||||
function indent(lines: string, tab = ' ') {
|
||||
return lines.replace(/^/gm, tab);
|
||||
}
|
||||
|
||||
const NO_UTF8_SUPPORT = process.platform === 'win32';
|
||||
const symbols = {
|
||||
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
|
||||
skipped: cyan('-'),
|
||||
progress: cyan('>'),
|
||||
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
|
||||
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
|
||||
};
|
||||
|
||||
function now() {
|
||||
return performance.now();
|
||||
}
|
||||
|
||||
function findPWLogsIndexes(msgOrStack: string): [number, number] {
|
||||
let startIndex = 0;
|
||||
let endIndex = 0;
|
||||
if (!msgOrStack) {
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
const lines = String(msgOrStack).split(SEPARATOR);
|
||||
const logStart = /[=]{3,} logs [=]{3,}/;
|
||||
const logEnd = /[=]{10,}/;
|
||||
lines.forEach((line, index) => {
|
||||
if (logStart.test(line)) {
|
||||
startIndex = index;
|
||||
} else if (logEnd.test(line)) {
|
||||
endIndex = index;
|
||||
}
|
||||
});
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
|
||||
function rewriteErrorStack(stack: string, indexes: [number, number]) {
|
||||
const [start, end] = indexes;
|
||||
/**
|
||||
* Do not rewrite if its not a playwright error
|
||||
*/
|
||||
if (start === 0 && end === 0) {
|
||||
return stack;
|
||||
}
|
||||
const linesToKeep = start + 3;
|
||||
if (start > 0 && linesToKeep < end) {
|
||||
const lines = stack.split(SEPARATOR);
|
||||
return lines
|
||||
.slice(0, linesToKeep)
|
||||
.concat(...lines.slice(end))
|
||||
.join(SEPARATOR);
|
||||
}
|
||||
return stack;
|
||||
}
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
|
||||
|
||||
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
|
||||
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
|
||||
|
||||
export const esArchiverLoad = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver load "${path}" --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverUnload = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver unload "${path}" --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverResetKibana = () => {
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver empty-kibana-index --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
|
@ -1,22 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
export type KibanaConfig = ReturnType<typeof readKibanaConfig>;
|
||||
|
||||
export const readKibanaConfig = () => {
|
||||
const kibanaConfigDir = path.join(__filename, '../../../../../../../config');
|
||||
const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml');
|
||||
const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml');
|
||||
|
||||
return (yaml.load(
|
||||
fs.readFileSync(fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, 'utf8')
|
||||
) || {}) as Record<string, string>;
|
||||
};
|
|
@ -13,7 +13,7 @@
|
|||
"@kbn/ux-plugin/e2e",
|
||||
"@kbn/ftr-common-functional-services",
|
||||
"@kbn/apm-plugin",
|
||||
"@kbn/es-archiver",
|
||||
"@kbn/ftr-common-functional-ui-services"
|
||||
"@kbn/ftr-common-functional-ui-services",
|
||||
"@kbn/observability-synthetics-test-data"
|
||||
]
|
||||
}
|
||||
|
|
|
@ -6,13 +6,13 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import {
|
||||
assertNotText,
|
||||
assertText,
|
||||
byTestId,
|
||||
waitForLoadingToFinish,
|
||||
} from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
|
||||
import { settingsPageProvider } from '../../page_objects/settings';
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { RetryService } from '@kbn/ftr-common-functional-services';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, assertText, waitForLoadingToFinish } from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { loginPageProvider } from '../../../page_objects/login';
|
||||
|
||||
journey('StatusFlyoutInAlertingApp', async ({ page, params }) => {
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before, expect } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, assertText, waitForLoadingToFinish } from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { loginPageProvider } from '../../../page_objects/login';
|
||||
|
||||
journey('TlsFlyoutInAlertingApp', async ({ page, params }) => {
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { callKibana } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/helpers/call_kibana';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, TIMEOUT_60_SEC, waitForLoadingToFinish } from '../../helpers/utils';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
import { loginPageProvider } from '../../page_objects/login';
|
||||
|
||||
journey('DataViewPermissions', async ({ page, params }) => {
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before, Page } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { makeChecksWithStatus } from '../../../helpers/make_checks';
|
||||
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
|
||||
import { journey, step, expect, before, Page } from '@elastic/synthetics';
|
||||
import { noop } from 'lodash';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, delay } from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
|
||||
|
||||
const dateRangeStart = '2019-09-10T12:40:08.078Z';
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before, Page } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId } from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
|
||||
|
||||
const dateRangeStart = '2019-09-10T12:40:08.078Z';
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before, Page } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, delay } from '../../../helpers/utils';
|
||||
import { recordVideo } from '../../../helpers/record_video';
|
||||
import { makeChecksWithStatus } from '../../../helpers/make_checks';
|
||||
import { monitorDetailsPageProvider } from '../../page_objects/monitor_details';
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { journey, step, expect } from '@elastic/synthetics';
|
||||
import { RetryService } from '@kbn/ftr-common-functional-services';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { loginPageProvider } from '../../page_objects/login';
|
||||
|
||||
journey('StepsDuration', async ({ page, params }) => {
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { byTestId, waitForLoadingToFinish } from '../../helpers/utils';
|
||||
import { recordVideo } from '../../helpers/record_video';
|
||||
|
||||
journey('UptimeOverview', ({ page, params }) => {
|
||||
recordVideo(page);
|
||||
|
|
|
@ -6,8 +6,7 @@
|
|||
*/
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import path from 'path';
|
||||
import { argv } from '../helpers/parse_args_params';
|
||||
import { SyntheticsRunner } from '../helpers/synthetics_runner';
|
||||
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
const { headless, grep, bail: pauseOnError } = argv;
|
||||
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import yargs from 'yargs';
|
||||
|
||||
const { argv } = yargs(process.argv.slice(2))
|
||||
.option('headless', {
|
||||
default: true,
|
||||
type: 'boolean',
|
||||
description: 'Start in headless mode',
|
||||
})
|
||||
.option('bail', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
description: 'Pause on error',
|
||||
})
|
||||
.option('watch', {
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
description: 'Runs the server in watch mode, restarting on changes',
|
||||
})
|
||||
.option('grep', {
|
||||
default: undefined,
|
||||
type: 'string',
|
||||
description: 'run only journeys with a name or tags that matches the glob',
|
||||
})
|
||||
.help();
|
||||
|
||||
export { argv };
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import Runner from '@elastic/synthetics/dist/core/runner';
|
||||
import { after, Page } from '@elastic/synthetics';
|
||||
|
||||
const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
|
||||
|
||||
// @ts-ignore
|
||||
export const runner: Runner = global[SYNTHETICS_RUNNER];
|
||||
|
||||
export const recordVideo = (page: Page, postfix = '') => {
|
||||
after(async () => {
|
||||
try {
|
||||
const videoFilePath = await page.video()?.path();
|
||||
const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
|
||||
const newVideoPath = videoFilePath?.replace(
|
||||
pathToVideo!,
|
||||
postfix ? runner.currentJourney!.name + `-${postfix}` : runner.currentJourney!.name
|
||||
);
|
||||
fs.renameSync(videoFilePath!, newVideoPath!);
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Error while renaming video file', e);
|
||||
}
|
||||
});
|
||||
};
|
|
@ -1,159 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import Url from 'url';
|
||||
import { run as syntheticsRun } from '@elastic/synthetics';
|
||||
import { PromiseType } from 'utility-types';
|
||||
import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
|
||||
|
||||
import { EsArchiver } from '@kbn/es-archiver';
|
||||
import { esArchiverUnload } from '../tasks/es_archiver';
|
||||
import { TestReporter } from './test_reporter';
|
||||
|
||||
export interface ArgParams {
|
||||
headless: boolean;
|
||||
match?: string;
|
||||
pauseOnError: boolean;
|
||||
}
|
||||
|
||||
export class SyntheticsRunner {
|
||||
public getService: any;
|
||||
public kibanaUrl: string;
|
||||
private elasticsearchUrl: string;
|
||||
|
||||
public testFilesLoaded: boolean = false;
|
||||
|
||||
public params: ArgParams;
|
||||
|
||||
private loadTestFilesCallback?: (reload?: boolean) => Promise<void>;
|
||||
|
||||
constructor(getService: any, params: ArgParams) {
|
||||
this.getService = getService;
|
||||
this.kibanaUrl = this.getKibanaUrl();
|
||||
this.elasticsearchUrl = this.getElasticsearchUrl();
|
||||
this.params = params;
|
||||
}
|
||||
|
||||
async setup() {
|
||||
await this.createTestUsers();
|
||||
}
|
||||
|
||||
async createTestUsers() {
|
||||
await createApmUsers({
|
||||
elasticsearch: {
|
||||
node: this.elasticsearchUrl,
|
||||
username: 'elastic',
|
||||
password: 'changeme',
|
||||
},
|
||||
kibana: { hostname: this.kibanaUrl },
|
||||
});
|
||||
}
|
||||
|
||||
async loadTestFiles(callback: (reload?: boolean) => Promise<void>, reload = false) {
|
||||
console.log('Loading test files');
|
||||
await callback(reload);
|
||||
this.loadTestFilesCallback = callback;
|
||||
this.testFilesLoaded = true;
|
||||
console.log('Successfully loaded test files');
|
||||
}
|
||||
|
||||
async loadTestData(e2eDir: string, dataArchives: string[]) {
|
||||
try {
|
||||
console.log('Loading esArchiver...');
|
||||
|
||||
const esArchiver: EsArchiver = this.getService('esArchiver');
|
||||
|
||||
const promises = dataArchives.map((archive) => {
|
||||
if (archive === 'synthetics_data') {
|
||||
return esArchiver.load(e2eDir + archive, {
|
||||
docsOnly: true,
|
||||
skipExisting: true,
|
||||
});
|
||||
}
|
||||
return esArchiver.load(e2eDir + archive, { skipExisting: true });
|
||||
});
|
||||
|
||||
await Promise.all([...promises]);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
|
||||
getKibanaUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.kibana.protocol'),
|
||||
hostname: config.get('servers.kibana.hostname'),
|
||||
port: config.get('servers.kibana.port'),
|
||||
});
|
||||
}
|
||||
|
||||
getElasticsearchUrl() {
|
||||
const config = this.getService('config');
|
||||
|
||||
return Url.format({
|
||||
protocol: config.get('servers.elasticsearch.protocol'),
|
||||
hostname: config.get('servers.elasticsearch.hostname'),
|
||||
port: config.get('servers.elasticsearch.port'),
|
||||
});
|
||||
}
|
||||
|
||||
async run() {
|
||||
if (!this.testFilesLoaded) {
|
||||
throw new Error('Test files not loaded');
|
||||
}
|
||||
const { headless, match, pauseOnError } = this.params;
|
||||
const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
|
||||
console.log(`Running ${noOfRuns} times`);
|
||||
let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
|
||||
for (let i = 0; i < noOfRuns; i++) {
|
||||
results = await syntheticsRun({
|
||||
params: { kibanaUrl: this.kibanaUrl, getService: this.getService },
|
||||
playwrightOptions: {
|
||||
headless,
|
||||
chromiumSandbox: false,
|
||||
timeout: 60 * 1000,
|
||||
viewport: {
|
||||
height: 900,
|
||||
width: 1600,
|
||||
},
|
||||
recordVideo: {
|
||||
dir: '.journeys/videos',
|
||||
},
|
||||
},
|
||||
grepOpts: { match: match === 'undefined' ? '' : match },
|
||||
pauseOnError,
|
||||
screenshots: 'only-on-failure',
|
||||
reporter: TestReporter,
|
||||
});
|
||||
if (noOfRuns > 1) {
|
||||
// need to reload again since runner resets the journeys
|
||||
await this.loadTestFiles(this.loadTestFilesCallback!, true);
|
||||
}
|
||||
}
|
||||
|
||||
await this.assertResults(results);
|
||||
}
|
||||
|
||||
assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
|
||||
Object.entries(results).forEach(([_journey, result]) => {
|
||||
if (result.status !== 'succeeded') {
|
||||
process.exitCode = 1;
|
||||
process.exit();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cleanUp() {
|
||||
console.log('Removing esArchiver...');
|
||||
esArchiverUnload('full_heartbeat');
|
||||
esArchiverUnload('browser');
|
||||
}
|
||||
}
|
|
@ -1,229 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Journey, Step } from '@elastic/synthetics/dist/dsl';
|
||||
import { Reporter, ReporterOptions } from '@elastic/synthetics';
|
||||
import {
|
||||
JourneyEndResult,
|
||||
JourneyStartResult,
|
||||
StepEndResult,
|
||||
} from '@elastic/synthetics/dist/common_types';
|
||||
|
||||
import { yellow, green, cyan, red, bold } from 'chalk';
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
const log = console.log;
|
||||
|
||||
import { performance } from 'perf_hooks';
|
||||
import * as fs from 'fs';
|
||||
import { gatherScreenshots } from '@elastic/synthetics/dist/reporters/json';
|
||||
import { CACHE_PATH } from '@elastic/synthetics/dist/helpers';
|
||||
import { join } from 'path';
|
||||
|
||||
function renderError(error: any) {
|
||||
let output = '';
|
||||
const outer = indent('');
|
||||
const inner = indent(outer);
|
||||
const container = outer + '---\n';
|
||||
output += container;
|
||||
let stack = error.stack;
|
||||
if (stack) {
|
||||
output += inner + 'stack: |-\n';
|
||||
stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
|
||||
const lines = String(stack).split('\n');
|
||||
for (const line of lines) {
|
||||
output += inner + ' ' + line + '\n';
|
||||
}
|
||||
}
|
||||
output += container;
|
||||
return red(output);
|
||||
}
|
||||
|
||||
function renderDuration(durationMs: number) {
|
||||
return Number(durationMs).toFixed(0);
|
||||
}
|
||||
|
||||
export class TestReporter implements Reporter {
|
||||
metrics = {
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
|
||||
journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
|
||||
|
||||
constructor(options: ReporterOptions = {}) {}
|
||||
|
||||
onJourneyStart(journey: Journey, {}: JourneyStartResult) {
|
||||
if (process.env.CI) {
|
||||
this.write(`\n--- Journey: ${journey.name}`);
|
||||
} else {
|
||||
this.write(bold(`\n Journey: ${journey.name}`));
|
||||
}
|
||||
}
|
||||
|
||||
onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
|
||||
const { status, end, start, error } = result;
|
||||
const message = `${symbols[status]} Step: '${step.name}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
this.metrics[status]++;
|
||||
if (!this.journeys.has(journey.name)) {
|
||||
this.journeys.set(journey.name, []);
|
||||
}
|
||||
this.journeys.get(journey.name)?.push({ name: step.name, ...result });
|
||||
}
|
||||
|
||||
async onJourneyEnd(journey: Journey, { error, start, end, status }: JourneyEndResult) {
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
if (total === 0 && error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
const message = `${symbols[status]} Took (${renderDuration(end - start)} seconds)`;
|
||||
this.write(message);
|
||||
|
||||
await fs.promises.mkdir('.journeys/failed_steps', { recursive: true });
|
||||
|
||||
await gatherScreenshots(join(CACHE_PATH, 'screenshots'), async (screenshot) => {
|
||||
const { data, step } = screenshot;
|
||||
|
||||
if (status === 'failed') {
|
||||
await (async () => {
|
||||
await fs.promises.writeFile(join('.journeys/failed_steps/', `${step.name}.jpg`), data, {
|
||||
encoding: 'base64',
|
||||
});
|
||||
})();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
onEnd() {
|
||||
const failedJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.some((step) => step.status === 'failed')
|
||||
);
|
||||
|
||||
if (failedJourneys.length > 0) {
|
||||
failedJourneys.forEach(([journeyName, steps]) => {
|
||||
if (process.env.CI) {
|
||||
const name = red(`Journey: ${journeyName} 🥵`);
|
||||
this.write(`\n+++ ${name}`);
|
||||
steps.forEach((stepResult) => {
|
||||
const { status, end, start, error, name: stepName } = stepResult;
|
||||
const message = `${symbols[status]} Step: '${stepName}' ${status} (${renderDuration(
|
||||
(end - start) * 1000
|
||||
)} ms)`;
|
||||
this.write(indent(message));
|
||||
if (error) {
|
||||
this.write(renderError(error));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const successfulJourneys = Array.from(this.journeys.entries()).filter(([, steps]) =>
|
||||
steps.every((step) => step.status === 'succeeded')
|
||||
);
|
||||
|
||||
successfulJourneys.forEach(([journeyName, steps]) => {
|
||||
try {
|
||||
fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
'Failed to delete video file for path ' + '.journeys/videos/' + journeyName + '.webm'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const { failed, succeeded, skipped } = this.metrics;
|
||||
const total = failed + succeeded + skipped;
|
||||
|
||||
let message = '\n';
|
||||
if (total === 0) {
|
||||
message = 'No tests found!';
|
||||
message += ` (${renderDuration(now())} ms) \n`;
|
||||
this.write(message);
|
||||
return;
|
||||
}
|
||||
|
||||
message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
|
||||
message += failed > 0 ? red(` ${failed} failed`) : '';
|
||||
message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
|
||||
message += ` (${renderDuration(now() / 1000)} seconds) \n`;
|
||||
this.write(message);
|
||||
}
|
||||
|
||||
write(message: any) {
|
||||
if (typeof message === 'object') {
|
||||
message = JSON.stringify(message);
|
||||
}
|
||||
log(message + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
const SEPARATOR = '\n';
|
||||
|
||||
function indent(lines: string, tab = ' ') {
|
||||
return lines.replace(/^/gm, tab);
|
||||
}
|
||||
|
||||
const NO_UTF8_SUPPORT = process.platform === 'win32';
|
||||
const symbols = {
|
||||
warning: yellow(NO_UTF8_SUPPORT ? '!' : '⚠'),
|
||||
skipped: cyan('-'),
|
||||
progress: cyan('>'),
|
||||
succeeded: green(NO_UTF8_SUPPORT ? 'ok' : '✓'),
|
||||
failed: red(NO_UTF8_SUPPORT ? 'x' : '✖'),
|
||||
};
|
||||
|
||||
function now() {
|
||||
return performance.now();
|
||||
}
|
||||
|
||||
function findPWLogsIndexes(msgOrStack: string): [number, number] {
|
||||
let startIndex = 0;
|
||||
let endIndex = 0;
|
||||
if (!msgOrStack) {
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
const lines = String(msgOrStack).split(SEPARATOR);
|
||||
const logStart = /[=]{3,} logs [=]{3,}/;
|
||||
const logEnd = /[=]{10,}/;
|
||||
lines.forEach((line, index) => {
|
||||
if (logStart.test(line)) {
|
||||
startIndex = index;
|
||||
} else if (logEnd.test(line)) {
|
||||
endIndex = index;
|
||||
}
|
||||
});
|
||||
return [startIndex, endIndex];
|
||||
}
|
||||
|
||||
function rewriteErrorStack(stack: string, indexes: [number, number]) {
|
||||
const [start, end] = indexes;
|
||||
/**
|
||||
* Do not rewrite if its not a playwright error
|
||||
*/
|
||||
if (start === 0 && end === 0) {
|
||||
return stack;
|
||||
}
|
||||
const linesToKeep = start + 3;
|
||||
if (start > 0 && linesToKeep < end) {
|
||||
const lines = stack.split(SEPARATOR);
|
||||
return lines
|
||||
.slice(0, linesToKeep)
|
||||
.concat(...lines.slice(end))
|
||||
.join(SEPARATOR);
|
||||
}
|
||||
return stack;
|
||||
}
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
const addTestTransaction = async (params: any) => {
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, expect, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before, expect } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { journey, step, before } from '@elastic/synthetics';
|
||||
import { recordVideo } from '../helpers/record_video';
|
||||
import { recordVideo } from '@kbn/observability-synthetics-test-data';
|
||||
import { UXDashboardDatePicker } from '../page_objects/date_picker';
|
||||
import { byLensTestId, loginToKibana, waitForLoadingToFinish } from './utils';
|
||||
|
||||
|
|
|
@ -6,8 +6,7 @@
|
|||
*/
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import path from 'path';
|
||||
import { argv } from './helpers/parse_args_params';
|
||||
import { SyntheticsRunner } from './helpers/synthetics_runner';
|
||||
import { SyntheticsRunner, argv } from '@kbn/observability-synthetics-test-data';
|
||||
|
||||
const { headless, grep, bail: pauseOnError } = argv;
|
||||
|
||||
|
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const ES_ARCHIVE_DIR = './fixtures/es_archiver';
|
||||
|
||||
// Otherwise execSync would inject NODE_TLS_REJECT_UNAUTHORIZED=0 and node would abort if used over https
|
||||
const NODE_TLS_REJECT_UNAUTHORIZED = '1';
|
||||
|
||||
export const esArchiverLoad = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver load "${path}" --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverUnload = (folder: string) => {
|
||||
const path = Path.join(ES_ARCHIVE_DIR, folder);
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver unload "${path}" --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
||||
|
||||
export const esArchiverResetKibana = () => {
|
||||
execSync(
|
||||
`node ../../../../../scripts/es_archiver empty-kibana-index --config ../../../../test/functional/config.base.js`,
|
||||
{ env: { ...process.env, NODE_TLS_REJECT_UNAUTHORIZED }, stdio: 'inherit' }
|
||||
);
|
||||
};
|
|
@ -6,5 +6,8 @@
|
|||
"outDir": "target/types",
|
||||
"types": ["node"]
|
||||
},
|
||||
"kbn_references": ["@kbn/test", "@kbn/apm-plugin", "@kbn/es-archiver"]
|
||||
"kbn_references": [
|
||||
"@kbn/test",
|
||||
"@kbn/observability-synthetics-test-data",
|
||||
]
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue