mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[kbn-journey] improve CI output by saving server logs into files (#175293)
## Summary This PR refactors `run_performance` script logic that we use to run single user performance journeys: - simplify script by keeping only 2 functions: `startES` & `runFunctionalTest` - write ES and Kibana logs (for warmup phase) to files instead of console output; - upload files with server log as build artifacts - do not retry journey TEST phase run on failure, it doubles EBT metrics and does not provide much value: journey should be simple and stable. - do not retry step running journeys, most of the failures are due to journey flakiness and we just skip them Testing performance pipeline https://buildkite.com/elastic/kibana-single-user-performance/builds/12552 Before: <img width="1243" alt="image" src="c4635986-6552-4842-abf9-640f09630674"> After <img width="1243" alt="Screenshot 2024-02-02 at 13 56 23" src="47ff6a76-d8dd-44dd-93a5-2c49ef87e808"> --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dzmitry Lemechko
GitHub
parent
97b5b7de88
commit
e04708fa3f
5 changed files with 138 additions and 105 deletions
|
|
@ -22,12 +22,6 @@ steps:
|
|||
depends_on: build
|
||||
key: tests
|
||||
timeout_in_minutes: 90
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 2
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- label: '📈 Report performance metrics to ci-stats'
|
||||
command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
|
||||
|
|
|
|||
|
|
@ -30,3 +30,12 @@ if [ -d "$JOURNEY_SCREENSHOTS_DIR" ]; then
|
|||
buildkite-agent artifact upload "**/*fullscreen*.png"
|
||||
cd "$KIBANA_DIR"
|
||||
fi
|
||||
|
||||
echo "--- Upload server logs"
|
||||
JOURNEY_SERVER_LOGS_REL_PATH=".ftr/journey_server_logs"
|
||||
JOURNEY_SERVER_LOGS_DIR="${KIBANA_DIR}/${JOURNEY_SERVER_LOGS_REL_PATH}"
|
||||
if [ -d "$JOURNEY_SERVER_LOGS_DIR" ]; then
|
||||
cd "$KIBANA_DIR"
|
||||
tar -czf server-logs.tar.gz $JOURNEY_SERVER_LOGS_REL_PATH/**/*
|
||||
buildkite-agent artifact upload server-logs.tar.gz
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ export const FLAG_OPTIONS: FlagOptions = {
|
|||
'exclude-tag',
|
||||
'include',
|
||||
'exclude',
|
||||
'writeLogsToPath',
|
||||
],
|
||||
alias: {
|
||||
updateAll: 'u',
|
||||
|
|
@ -47,6 +48,7 @@ export const FLAG_OPTIONS: FlagOptions = {
|
|||
--kibana-install-dir Run Kibana from existing install directory instead of from source
|
||||
--bail Stop the test run at the first failure
|
||||
--logToFile Write the log output from Kibana/ES to files instead of to stdout
|
||||
--writeLogsToPath Write the log output from Kibana/ES to files in specified path
|
||||
--dry-run Report tests without executing them
|
||||
--updateBaselines Replace baseline screenshots with whatever is generated from the test
|
||||
--updateSnapshots Replace inline and file snapshots with whatever is generated from the test
|
||||
|
|
@ -73,6 +75,12 @@ export function parseFlags(flags: FlagsReader) {
|
|||
|
||||
const esVersionString = flags.string('es-version');
|
||||
|
||||
const logsDir = flags.path('writeLogsToPath')
|
||||
? Path.resolve(REPO_ROOT, flags.path('writeLogsToPath')!)
|
||||
: flags.boolean('logToFile')
|
||||
? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
configs,
|
||||
esVersion: esVersionString ? new EsVersion(esVersionString) : EsVersion.getDefault(),
|
||||
|
|
@ -80,9 +88,7 @@ export function parseFlags(flags: FlagsReader) {
|
|||
dryRun: flags.boolean('dry-run'),
|
||||
updateBaselines: flags.boolean('updateBaselines') || flags.boolean('updateAll'),
|
||||
updateSnapshots: flags.boolean('updateSnapshots') || flags.boolean('updateAll'),
|
||||
logsDir: flags.boolean('logToFile')
|
||||
? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
|
||||
: undefined,
|
||||
logsDir,
|
||||
esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']),
|
||||
esServerlessImage: flags.string('esServerlessImage'),
|
||||
installDir: flags.path('kibana-install-dir'),
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@
|
|||
|
||||
import { createFlagError } from '@kbn/dev-cli-errors';
|
||||
import { run } from '@kbn/dev-cli-runner';
|
||||
import { ProcRunner } from '@kbn/dev-proc-runner';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
|
|
@ -19,6 +21,92 @@ export interface Journey {
|
|||
path: string;
|
||||
}
|
||||
|
||||
interface EsRunProps {
|
||||
procRunner: ProcRunner;
|
||||
log: ToolingLog;
|
||||
logsDir?: string;
|
||||
}
|
||||
|
||||
interface TestRunProps extends EsRunProps {
|
||||
journey: Journey;
|
||||
phase: 'TEST' | 'WARMUP';
|
||||
kibanaInstallDir: string | undefined;
|
||||
}
|
||||
|
||||
const readFilesRecursively = (dir: string, callback: Function) => {
|
||||
const files = fs.readdirSync(dir);
|
||||
files.forEach((file) => {
|
||||
const filePath = path.join(dir, file);
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.isDirectory()) {
|
||||
readFilesRecursively(filePath, callback);
|
||||
} else if (stat.isFile()) {
|
||||
callback(filePath);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
async function startEs(props: EsRunProps) {
|
||||
const { procRunner, log, logsDir } = props;
|
||||
await procRunner.run('es', {
|
||||
cmd: 'node',
|
||||
args: [
|
||||
'scripts/es',
|
||||
'snapshot',
|
||||
'--license=trial',
|
||||
// Temporarily disabling APM
|
||||
// ...(JOURNEY_APM_CONFIG.active
|
||||
// ? [
|
||||
// '-E',
|
||||
// 'tracing.apm.enabled=true',
|
||||
// '-E',
|
||||
// 'tracing.apm.agent.transaction_sample_rate=1.0',
|
||||
// '-E',
|
||||
// `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
|
||||
// '-E',
|
||||
// `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
|
||||
// '-E',
|
||||
// `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
|
||||
// ]
|
||||
// : []),
|
||||
`--writeLogsToPath=${logsDir}/es-cluster.log`,
|
||||
],
|
||||
cwd: REPO_ROOT,
|
||||
wait: /kbn\/es setup complete/,
|
||||
});
|
||||
|
||||
log.info(`✅ ES is ready and will run in the background`);
|
||||
}
|
||||
|
||||
async function runFunctionalTest(props: TestRunProps) {
|
||||
const { procRunner, journey, phase, kibanaInstallDir, logsDir } = props;
|
||||
await procRunner.run('functional-tests', {
|
||||
cmd: 'node',
|
||||
args: [
|
||||
'scripts/functional_tests',
|
||||
['--config', journey.path],
|
||||
kibanaInstallDir ? ['--kibana-install-dir', kibanaInstallDir] : [],
|
||||
// save Kibana logs in file instead of console output; only for "warmup" phase
|
||||
logsDir ? ['--writeLogsToPath', logsDir] : [],
|
||||
'--debug',
|
||||
'--bail',
|
||||
].flat(),
|
||||
cwd: REPO_ROOT,
|
||||
wait: true,
|
||||
env: {
|
||||
// Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values.
|
||||
...Object.fromEntries(
|
||||
Object.keys(process.env).flatMap((k) =>
|
||||
k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : []
|
||||
)
|
||||
),
|
||||
TEST_PERFORMANCE_PHASE: phase,
|
||||
TEST_ES_URL: 'http://elastic:changeme@localhost:9200',
|
||||
TEST_ES_DISABLE_STARTUP: 'true',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
run(
|
||||
async ({ log, flagsReader, procRunner }) => {
|
||||
const skipWarmup = flagsReader.boolean('skip-warmup');
|
||||
|
|
@ -33,19 +121,23 @@ run(
|
|||
throw createFlagError('--journey-path must be an existing path');
|
||||
}
|
||||
|
||||
let journeys: Journey[] = [];
|
||||
const journeys: Journey[] = [];
|
||||
|
||||
if (journeyPath) {
|
||||
journeys = fs.statSync(journeyPath).isDirectory()
|
||||
? fs.readdirSync(journeyPath).map((fileName) => {
|
||||
return { name: fileName, path: path.resolve(journeyPath, fileName) };
|
||||
})
|
||||
: [{ name: path.parse(journeyPath).name, path: journeyPath }];
|
||||
if (journeyPath && fs.statSync(journeyPath).isFile()) {
|
||||
journeys.push({ name: path.parse(journeyPath).name, path: journeyPath });
|
||||
} else {
|
||||
const journeyBasePath = path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
|
||||
journeys = fs.readdirSync(journeyBasePath).map((name) => {
|
||||
return { name, path: path.join(journeyBasePath, name) };
|
||||
});
|
||||
// default dir is x-pack/performance/journeys
|
||||
const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
|
||||
readFilesRecursively(dir, (filePath: string) =>
|
||||
journeys.push({
|
||||
name: path.parse(filePath).name,
|
||||
path: path.resolve(dir, filePath),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (journeys.length === 0) {
|
||||
throw new Error('No journeys found');
|
||||
}
|
||||
|
||||
log.info(
|
||||
|
|
@ -56,11 +148,24 @@ run(
|
|||
|
||||
for (const journey of journeys) {
|
||||
try {
|
||||
await startEs();
|
||||
// create folder to store ES/Kibana server logs
|
||||
const logsDir = path.resolve(REPO_ROOT, `.ftr/journey_server_logs/${journey.name}`);
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
process.stdout.write(`--- Running journey: ${journey.name} [start ES, warmup run]\n`);
|
||||
await startEs({ procRunner, log, logsDir });
|
||||
if (!skipWarmup) {
|
||||
await runWarmup(journey, kibanaInstallDir);
|
||||
// Set the phase to WARMUP, this will prevent the FTR from starting Kibana with opt-in telemetry and save logs to file
|
||||
await runFunctionalTest({
|
||||
procRunner,
|
||||
log,
|
||||
journey,
|
||||
phase: 'WARMUP',
|
||||
kibanaInstallDir,
|
||||
logsDir,
|
||||
});
|
||||
}
|
||||
await runTest(journey, kibanaInstallDir);
|
||||
process.stdout.write(`--- Running journey: ${journey.name} [collect metrics]\n`);
|
||||
await runFunctionalTest({ procRunner, log, journey, phase: 'TEST', kibanaInstallDir });
|
||||
} catch (e) {
|
||||
log.error(e);
|
||||
failedJourneys.push(journey.name);
|
||||
|
|
@ -69,88 +174,6 @@ run(
|
|||
}
|
||||
}
|
||||
|
||||
async function runFunctionalTest(
|
||||
configPath: string,
|
||||
phase: 'TEST' | 'WARMUP',
|
||||
kibanaBuildDir: string | undefined
|
||||
) {
|
||||
await procRunner.run('functional-tests', {
|
||||
cmd: 'node',
|
||||
args: [
|
||||
'scripts/functional_tests',
|
||||
['--config', configPath],
|
||||
kibanaBuildDir ? ['--kibana-install-dir', kibanaBuildDir] : [],
|
||||
'--debug',
|
||||
'--bail',
|
||||
].flat(),
|
||||
cwd: REPO_ROOT,
|
||||
wait: true,
|
||||
env: {
|
||||
// Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values.
|
||||
...Object.fromEntries(
|
||||
Object.keys(process.env).flatMap((k) =>
|
||||
k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : []
|
||||
)
|
||||
),
|
||||
TEST_PERFORMANCE_PHASE: phase,
|
||||
TEST_ES_URL: 'http://elastic:changeme@localhost:9200',
|
||||
TEST_ES_DISABLE_STARTUP: 'true',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function startEs() {
|
||||
process.stdout.write(`--- Starting ES\n`);
|
||||
await procRunner.run('es', {
|
||||
cmd: 'node',
|
||||
args: [
|
||||
'scripts/es',
|
||||
'snapshot',
|
||||
'--license=trial',
|
||||
// Temporarily disabling APM
|
||||
// ...(JOURNEY_APM_CONFIG.active
|
||||
// ? [
|
||||
// '-E',
|
||||
// 'tracing.apm.enabled=true',
|
||||
// '-E',
|
||||
// 'tracing.apm.agent.transaction_sample_rate=1.0',
|
||||
// '-E',
|
||||
// `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
|
||||
// '-E',
|
||||
// `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
|
||||
// '-E',
|
||||
// `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
|
||||
// ]
|
||||
// : []),
|
||||
],
|
||||
cwd: REPO_ROOT,
|
||||
wait: /kbn\/es setup complete/,
|
||||
});
|
||||
|
||||
log.info(`✅ ES is ready and will run in the background`);
|
||||
}
|
||||
|
||||
async function runWarmup(journey: Journey, kibanaBuildDir: string | undefined) {
|
||||
try {
|
||||
process.stdout.write(`--- Running warmup: ${journey.name}\n`);
|
||||
// Set the phase to WARMUP, this will prevent the functional test server from starting Elasticsearch, opt in to telemetry, etc.
|
||||
await runFunctionalTest(journey.path, 'WARMUP', kibanaBuildDir);
|
||||
} catch (e) {
|
||||
log.warning(`Warmup for ${journey.name} failed`);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTest(journey: Journey, kibanaBuildDir: string | undefined) {
|
||||
try {
|
||||
process.stdout.write(`--- Running ${journey.name}\n`);
|
||||
await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir);
|
||||
} catch (e) {
|
||||
log.warning(`Journey ${journey.name} failed. Retrying once...`);
|
||||
await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir);
|
||||
}
|
||||
}
|
||||
|
||||
if (failedJourneys.length > 0) {
|
||||
throw new Error(`${failedJourneys.length} journeys failed: ${failedJourneys.join(',')}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,5 +41,6 @@
|
|||
"@kbn/config-schema",
|
||||
"@kbn/core-test-helpers-so-type-serializer",
|
||||
"@kbn/core-test-helpers-kbn-server",
|
||||
"@kbn/dev-proc-runner",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue