[scalability testing] add json to track skipped journeys (#151629)

## Summary

Since developers are adding more api capacity tests, we need an easy way
to quickly skip the failing/unstable ones.

We run these tests on CI by passing path to the root test folder with
--journey-path flag:

0e18843e03/.buildkite/scripts/steps/scalability/benchmarking.sh (L82-L83)

The idea is that when there is a need to skip test, it should be added
to `x-pack/test/scalability/disabled_scalability_tests.json`:
```
[
   "x-pack/test/scalability/apis/api.telemetry.cluster_stats.json"
]
```

Using json array file seems like an easy option (similar to jest
config), but I'm open for suggestions.
This commit is contained in:
Dzmitry Lemechko 2023-02-21 17:58:50 +01:00 committed by GitHub
parent acfc15f343
commit 64d57c5c14
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 2 deletions

View file

@ -40,13 +40,36 @@ run(
})
: [{ name: path.parse(journeyPath).name, path: journeyPath }];
const skippedFilePath = 'x-pack/test/scalability/disabled_scalability_tests.json';
const skipped: string[] = JSON.parse(
fs.readFileSync(path.resolve(REPO_ROOT, skippedFilePath), 'utf8')
).map((relativePath: string) => path.resolve(REPO_ROOT, relativePath));
let filtered: Journey[] = [];
if (skipped.length === 0) {
filtered = journeys;
} else {
for (const journey of journeys) {
if (skipped.includes(journey.path)) {
log.warning(`Journey '${journey.name} is skipped'`);
} else {
filtered.push(journey);
}
}
}
if (filtered.length === 0) {
log.info(`No journeys found, check skipped list in '${skippedFilePath}'`);
return;
}
log.info(
`Found ${journeys.length} journeys to run: ${JSON.stringify(journeys.map((j) => j.name))}`
`Found ${filtered.length} journeys to run: ${JSON.stringify(filtered.map((j) => j.name))}`
);
const failedJourneys = [];
for (const journey of journeys) {
for (const journey of filtered) {
try {
process.stdout.write(`--- Running scalability journey: ${journey.name}\n`);
await runScalabilityJourney(journey.path, kibanaInstallDir);

View file

@ -0,0 +1,2 @@
[
]