mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
Performance runner in js (#146129)
## Summary Rewrite the performance journey runner using TypeScript, to avoid dangling ES\node processes during test execution. Results are stable with this runner, as verified on CI  Co-authored-by: Spencer <email@spalger.com> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Liza Katz
GitHub
parent
f8849c56bd
commit
783ea146a7
4 changed files with 113 additions and 117 deletions
|
|
@ -7,117 +7,13 @@ source .buildkite/scripts/common/util.sh
|
|||
is_test_execution_step
|
||||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
|
||||
# These tests are running on static workers so we have to make sure we delete previous build of Kibana
|
||||
rm -rf "$KIBANA_BUILD_LOCATION"
|
||||
.buildkite/scripts/download_build_artifacts.sh
|
||||
|
||||
function is_running {
|
||||
kill -0 "$1" &>/dev/null
|
||||
}
|
||||
|
||||
# unset env vars defined in other parts of CI for automatic APM collection of
|
||||
# Kibana. We manage APM config in our FTR config and performance service, and
|
||||
# APM treats config in the ENV with a very high precedence.
|
||||
unset ELASTIC_APM_ENVIRONMENT
|
||||
unset ELASTIC_APM_TRANSACTION_SAMPLE_RATE
|
||||
unset ELASTIC_APM_SERVER_URL
|
||||
unset ELASTIC_APM_SECRET_TOKEN
|
||||
unset ELASTIC_APM_ACTIVE
|
||||
unset ELASTIC_APM_CONTEXT_PROPAGATION_ONLY
|
||||
unset ELASTIC_APM_ACTIVE
|
||||
unset ELASTIC_APM_SERVER_URL
|
||||
unset ELASTIC_APM_SECRET_TOKEN
|
||||
unset ELASTIC_APM_GLOBAL_LABELS
|
||||
|
||||
# `kill $esPid` doesn't work, seems that kbn-es doesn't listen to signals correctly, this does work
|
||||
trap 'killall node -q' EXIT
|
||||
|
||||
export TEST_ES_URL=http://elastic:changeme@localhost:9200
|
||||
export TEST_ES_DISABLE_STARTUP=true
|
||||
|
||||
echo "--- determining which journeys to run"
|
||||
|
||||
journeys=$(buildkite-agent meta-data get "failed-journeys" --default '')
|
||||
if [ "$journeys" != "" ]; then
|
||||
echo "re-running failed journeys:${journeys}"
|
||||
else
|
||||
paths=()
|
||||
for path in x-pack/performance/journeys/*; do
|
||||
paths+=("$path")
|
||||
done
|
||||
journeys=$(printf "%s\n" "${paths[@]}")
|
||||
echo "running discovered journeys:${journeys}"
|
||||
fi
|
||||
|
||||
# track failed journeys here which might get written to metadata
|
||||
failedJourneys=()
|
||||
|
||||
while read -r journey; do
|
||||
if [ "$journey" == "" ]; then
|
||||
continue;
|
||||
fi
|
||||
|
||||
echo "--- $journey - 🔎 Start es"
|
||||
|
||||
node scripts/es snapshot&
|
||||
export esPid=$!
|
||||
|
||||
# Pings the es server every second for up to 2 minutes until it is green
|
||||
curl \
|
||||
--fail \
|
||||
--silent \
|
||||
--retry 120 \
|
||||
--retry-delay 1 \
|
||||
--retry-connrefused \
|
||||
-XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" \
|
||||
> /dev/null
|
||||
|
||||
echo "✅ ES is ready and will run in the background"
|
||||
|
||||
phases=("WARMUP" "TEST")
|
||||
status=0
|
||||
for phase in "${phases[@]}"; do
|
||||
echo "--- $journey - $phase"
|
||||
|
||||
export TEST_PERFORMANCE_PHASE="$phase"
|
||||
|
||||
set +e
|
||||
node scripts/functional_tests \
|
||||
--config "$journey" \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--debug \
|
||||
--bail
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [ $status -ne 0 ]; then
|
||||
failedJourneys+=("$journey")
|
||||
echo "^^^ +++"
|
||||
echo "❌ FTR failed with status code: $status"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# remove trap, we're manually shutting down
|
||||
trap - EXIT;
|
||||
|
||||
echo "--- $journey - 🔎 Shutdown ES"
|
||||
killall node
|
||||
echo "waiting for $esPid to exit gracefully";
|
||||
|
||||
timeout=30 #seconds
|
||||
dur=0
|
||||
while is_running $esPid; do
|
||||
sleep 1;
|
||||
((dur=dur+1))
|
||||
if [ $dur -ge $timeout ]; then
|
||||
echo "es still running after $dur seconds, killing ES and node forcefully";
|
||||
killall -SIGKILL java
|
||||
killall -SIGKILL node
|
||||
sleep 5;
|
||||
fi
|
||||
done
|
||||
done <<< "$journeys"
|
||||
echo "--- Running performance tests"
|
||||
node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"
|
||||
|
||||
echo "--- Upload journey step screenshots"
|
||||
JOURNEY_SCREENSHOTS_DIR="${KIBANA_DIR}/data/journey_screenshots"
|
||||
|
|
@ -126,11 +22,3 @@ if [ -d "$JOURNEY_SCREENSHOTS_DIR" ]; then
|
|||
buildkite-agent artifact upload "**/*fullscreen*.png"
|
||||
cd "$KIBANA_DIR"
|
||||
fi
|
||||
|
||||
echo "--- report/record failed journeys"
|
||||
if [ "${failedJourneys[*]}" != "" ]; then
|
||||
buildkite-agent meta-data set "failed-journeys" "$(printf "%s\n" "${failedJourneys[@]}")"
|
||||
|
||||
echo "failed journeys: ${failedJourneys[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ export function makeFtrConfigProvider(
|
|||
|
||||
kbnTestServer: {
|
||||
...baseConfig.kbnTestServer,
|
||||
// delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution
|
||||
// delay shutdown to ensure that APM can report the data it collects during test execution
|
||||
delayShutdown: process.env.TEST_PERFORMANCE_PHASE === 'TEST' ? 15_000 : 0,
|
||||
|
||||
serverArgs: [
|
||||
|
|
@ -93,7 +93,7 @@ export function makeFtrConfigProvider(
|
|||
],
|
||||
|
||||
env: {
|
||||
ELASTIC_APM_ACTIVE: process.env.TEST_PERFORMANCE_PHASE ? 'true' : 'false',
|
||||
ELASTIC_APM_ACTIVE: 'true',
|
||||
ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
|
||||
ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
|
||||
ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
|
||||
|
|
|
|||
10
scripts/run_performance.js
Normal file
10
scripts/run_performance.js
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
require('../src/setup_node_env');
|
||||
require('../src/dev/performance/run_performance_cli');
|
||||
98
src/dev/performance/run_performance_cli.ts
Normal file
98
src/dev/performance/run_performance_cli.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { run } from '@kbn/dev-cli-runner';
|
||||
import { REPO_ROOT } from '@kbn/utils';
|
||||
import Fsp from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
run(
|
||||
async ({ log, flagsReader, procRunner }) => {
|
||||
async function runFunctionalTest(journey: string, phase: 'TEST' | 'WARMUP') {
|
||||
// Pass in a clean APM environment, so that FTR can later
|
||||
// set it's own values.
|
||||
const cleanApmEnv = {
|
||||
ELASTIC_APM_TRANSACTION_SAMPLE_RATE: undefined,
|
||||
ELASTIC_APM_SERVER_URL: undefined,
|
||||
ELASTIC_APM_SECRET_TOKEN: undefined,
|
||||
ELASTIC_APM_ACTIVE: undefined,
|
||||
ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: undefined,
|
||||
ELASTIC_APM_GLOBAL_LABELS: undefined,
|
||||
};
|
||||
|
||||
await procRunner.run('functional-tests', {
|
||||
cmd: 'node',
|
||||
args: [
|
||||
'scripts/functional_tests',
|
||||
['--config', path.join(journeyBasePath, journey)],
|
||||
['--kibana-install-dir', kibanaInstallDir],
|
||||
'--debug',
|
||||
'--bail',
|
||||
].flat(),
|
||||
cwd: REPO_ROOT,
|
||||
wait: true,
|
||||
env: {
|
||||
TEST_PERFORMANCE_PHASE: phase,
|
||||
TEST_ES_URL: 'http://elastic:changeme@localhost:9200',
|
||||
TEST_ES_DISABLE_STARTUP: 'true',
|
||||
...cleanApmEnv,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function startEs() {
|
||||
process.stdout.write(`--- Starting ES\n`);
|
||||
await procRunner.run('es', {
|
||||
cmd: 'node',
|
||||
args: ['scripts/es', 'snapshot'],
|
||||
cwd: REPO_ROOT,
|
||||
wait: /kbn\/es setup complete/,
|
||||
});
|
||||
|
||||
log.info(`✅ ES is ready and will run in the background`);
|
||||
}
|
||||
|
||||
async function runWarmup(journey: string) {
|
||||
try {
|
||||
process.stdout.write(`--- Running warmup ${journey}\n`);
|
||||
// Set the phase to WARMUP, this will prevent the functional test server from starting Elasticsearch, opt in to telemetry, etc.
|
||||
await runFunctionalTest(journey, 'WARMUP');
|
||||
} catch (e) {
|
||||
log.warning(`Warmup for ${journey} failed`);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTest(journey: string) {
|
||||
try {
|
||||
process.stdout.write(`--- Running test ${journey}\n`);
|
||||
await runFunctionalTest(journey, 'TEST');
|
||||
} catch (e) {
|
||||
log.warning(`Journey ${journey} failed. Retrying once...`);
|
||||
await runFunctionalTest(journey, 'TEST');
|
||||
}
|
||||
}
|
||||
|
||||
const journeyBasePath = path.resolve(REPO_ROOT, 'x-pack/performance/journeys/');
|
||||
const kibanaInstallDir = flagsReader.requiredPath('kibana-install-dir');
|
||||
const journeys = await Fsp.readdir(journeyBasePath);
|
||||
log.info(`Found ${journeys.length} journeys to run`);
|
||||
|
||||
for (const journey of journeys) {
|
||||
await startEs();
|
||||
await runWarmup(journey);
|
||||
await runTest(journey);
|
||||
await procRunner.stop('es');
|
||||
}
|
||||
},
|
||||
{
|
||||
flags: {
|
||||
string: ['kibana-install-dir'],
|
||||
},
|
||||
}
|
||||
);
|
||||
Loading…
Add table
Add a link
Reference in a new issue