mirror of
https://github.com/elastic/kibana.git
synced 2025-04-25 02:09:32 -04:00
[ftr] fix scripts/functional_tests to respect cli flags (#155734)
## Summary This [failure](https://buildkite.com/elastic/kibana-on-merge/builds/29091#01879988-ecd2-4e4f-bfb4-108939f145a1) clearly shows that `--bail` flag is ignored when passed to `scripts/functional_tests.js` script, and since `scripts/functional_tests.js --help` list these flags I think we need to fix it: ``` --include-tag Tags that suites must include to be run, can be included multiple times --exclude-tag Tags that suites must NOT include to be run, can be included multiple times --include Files that must included to be run, can be included multiple times --exclude Files that must NOT be included to be run, can be included multiple times --grep Pattern to select which tests to run --bail Stop the test run at the first failure --dry-run Report tests without executing them --updateBaselines Replace baseline screenshots with whatever is generated from the test --updateSnapshots Replace inline and file snapshots with whatever is generated from the test ``` I was able to reproduce it locally: 1. Break [test/functional/apps/console/_console.ts](test/functional/apps/console/_console.ts) by adding `expect(1).to.be(2);` in the first `it` function 2. Run `node scripts/functional_tests.js --bail --config test/functional/apps/console/config.ts` Actual: Tests continue to run after failure Expected: Stop tests after first failure It turned out `scripts/functional_test_runner.js` respects the flags so I just copied the logic from [packages/kbn-test/src/functional_test_runner/cli.ts](https://github.com/elastic/kibana/blob/main/packages/kbn-test/src/functional_test_runner/cli.ts#L41-L63) Let me know if you think we need to add jest tests. Tested: ``` node scripts/functional_tests.js --bail --config test/functional/apps/console/config.ts --grep "multiple requests output" ```
This commit is contained in:
parent
947b75741c
commit
4211e03a5f
2 changed files with 26 additions and 5 deletions
|
@ -78,12 +78,12 @@ export function parseFlags(flags: FlagsReader) {
|
||||||
installDir: flags.path('kibana-install-dir'),
|
installDir: flags.path('kibana-install-dir'),
|
||||||
grep: flags.string('grep'),
|
grep: flags.string('grep'),
|
||||||
suiteTags: {
|
suiteTags: {
|
||||||
include: flags.arrayOfStrings('include-tag'),
|
include: flags.arrayOfStrings('include-tag') ?? [],
|
||||||
exclude: flags.arrayOfStrings('exclude-tag'),
|
exclude: flags.arrayOfStrings('exclude-tag') ?? [],
|
||||||
},
|
},
|
||||||
suiteFilters: {
|
suiteFilters: {
|
||||||
include: flags.arrayOfPaths('include'),
|
include: flags.arrayOfPaths('include') ?? [],
|
||||||
exclude: flags.arrayOfPaths('exclude'),
|
exclude: flags.arrayOfPaths('exclude') ?? [],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,6 +36,27 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
|
||||||
log.warning('❗️❗️❗️');
|
log.warning('❗️❗️❗️');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const settingOverrides = {
|
||||||
|
mochaOpts: {
|
||||||
|
bail: options.bail,
|
||||||
|
dryRun: options.dryRun,
|
||||||
|
grep: options.grep,
|
||||||
|
},
|
||||||
|
kbnTestServer: {
|
||||||
|
installDir: options.installDir,
|
||||||
|
},
|
||||||
|
suiteFiles: {
|
||||||
|
include: options.suiteFilters.include,
|
||||||
|
exclude: options.suiteFilters.exclude,
|
||||||
|
},
|
||||||
|
suiteTags: {
|
||||||
|
include: options.suiteTags.include,
|
||||||
|
exclude: options.suiteTags.exclude,
|
||||||
|
},
|
||||||
|
updateBaselines: options.updateBaselines,
|
||||||
|
updateSnapshots: options.updateSnapshots,
|
||||||
|
};
|
||||||
|
|
||||||
for (const [i, path] of options.configs.entries()) {
|
for (const [i, path] of options.configs.entries()) {
|
||||||
await log.indent(0, async () => {
|
await log.indent(0, async () => {
|
||||||
if (options.configs.length > 1) {
|
if (options.configs.length > 1) {
|
||||||
|
@ -43,7 +64,7 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
|
||||||
log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, path)}`);
|
log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, path)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const config = await readConfigFile(log, options.esVersion, path);
|
const config = await readConfigFile(log, options.esVersion, path, settingOverrides);
|
||||||
|
|
||||||
const hasTests = await checkForEnabledTestsInFtrConfig({
|
const hasTests = await checkForEnabledTestsInFtrConfig({
|
||||||
config,
|
config,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue