kibana/x-pack/test/security_solution_api_integration/scripts/index.js
Konrad Szwarc f96d55a4f5
[EDR Workflows] MKI API tests (#187560)
This pull request introduces two changes to our existing API integration
tests:
1. It restructures the files to follow the security solution-wide
standard.
2. It adds our API integration tests to the periodic MKI pipeline.
[Example
build](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/818)

**Change of Structure:**
All tests have been moved to
`x-pack/test/security_solution_api_integration/test_suites/edr_workflows`
and are grouped by feature and then by licensing.
![Screenshot 2024-07-10 at 11 52
42](223c9138-8702-42f2-a801-a35be87304cb)

**MKI:**
Due to the nature of our tests – their dependence on switching users
and/or modifying internal indices – only 3 out of 7 test suites qualify
to be run in MKI. I've added all test suites to
`.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml`.
However, the ones that would be skipped are commented out to avoid
consuming resources without providing any value.

**Testing for Regression:**
I've noticed that the `@skipInServerlessMKI` tag is not working as
expected. Tests tagged with `@serverless @skipInServerlessMKI Test Name`
were not being run in the PR pipelines. The grep pattern we were using
in individual configs and in
`x-pack/test/security_solution_api_integration/scripts/index.js`
(`'/^(?!.*@skipInServerless).*@serverless.*/'`) would also match
`@skipInServerlessMKI`.

I've modified the pattern to look for a full word, expecting it to be at
the beginning or end of a string, and to be followed or not followed by
a whitespace. We could use unit tests for these grep patterns 😄

Here is a screenshot of the new regex being tested:

![Screenshot 2024-07-10 at 12 09
28](8b9dd49a-3ca5-458d-9567-ad938847f169)

This led me to double-check whether all our API integration tests are
being executed in both PR and MKI pipelines, all seems to be in place:

**MKI:**
1. Artifacts -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81ae-47c3-a867-b16de4bfa20e/262-380)
- 0 tests executed due to `@skipInServerlessMKI` present in all top
describe of each test file
2. Authentication -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b0-4824-a658-3a881607eb56)
- 0 tests executed due to `@skipInServerlessMKI` present in all top
describe of each test file
3. Metadata -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b1-4730-9b51-512b1b554f64/261-386)
- 0 tests executed due to `@skipInServerlessMKI` present in all top
describe of each test file
4. Package -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b3-418c-ad33-0cd7dd68ad46/261-370)
- 0 tests executed due to `@skipInServerlessMKI` present in all top
describe of each test file
5. Policy Response -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b4-4034-a575-3ddbdde42e24/261-422)
- all tests were executed
6. Resolver -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b5-482d-8023-e1f819d3c56e/261-711)
- all but the tests with `@skipInServerless` were executed
7. Response actions -
[buildkite](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-defend-workflows/builds/817#01909bfb-81b7-4561-83a3-6896523cff8f/262-403)
- only one file was executed due to the second one being tagged as
`@skipInServerlessMKI`

**PR:**

All tests are accounted for and executed as expected, no regression.
package suite was never executed since it's `.skip`


policy_response/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-675b-4fab-a787-e5e472711fb0/3394)
policy_response/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-678f-49c6-ae4f-aee3738713c2/3446)
authentication/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-6768-4330-9b6c-8328a46a5a99/2352)
authentication/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-67b3-4b9b-ba31-110f737a1f3f/1970)
resolver/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-6759-49a3-8cb7-4b0097cf8975/6266)
resolver/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-676a-4dfe-bc19-6fd50e42980a/3302)
metadata/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-676c-49ff-b0a5-cf7acc9c5506/4827)
metadata/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-679e-45e8-aa52-5672baf344df/3000)
response_actions/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-67ad-4826-bb58-4b6330fef338/2760)
response_actions/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-67b7-4a6a-a37d-d138a7054a41/9654)
artifacts/ess
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-67d6-4926-a93f-b193ab2859be/1158)
artifacts/serverless
[buildkite](https://buildkite.com/elastic/kibana-pull-request/builds/220548#01909c0b-672e-4350-8820-c7fd8d7ef010/2328)

---------

Co-authored-by: Angela Chuang <yi-chun.chuang@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Angela Chuang <6295984+angorayc@users.noreply.github.com>
2024-07-12 14:41:41 +02:00

85 lines
2.3 KiB
JavaScript

/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
const { spawn } = require('child_process');
const [, , type, area, licenseFolder, domain, projectType, environment, ...args] = process.argv;
const commandUsage = `
Usage: node index.js <type> <area> <licenseFolder> <domain> <projectType> <environment> [args]
Arguments:
type: server | runner
environment: serverlessEnv | essEnv | qaPeriodicEnv | qaEnv. Mandatory for runner type
area, domain, licenseFolder, projectType, environment are required arguments to locate the config file with below path
: ./test_suites/<area>/<domain>/<licenseFolder>/configs/<projectType>.config.ts
`;
if (!type || !area || !licenseFolder || !domain || !projectType) {
console.error(commandUsage);
}
const configPath = `./test_suites/${area}/${domain}/${licenseFolder}/configs/${projectType}.config.ts`;
const command =
type === 'server'
? '../../scripts/functional_tests_server.js'
: '../../scripts/functional_test_runner';
let grepArgs = [];
if (type !== 'server') {
switch (environment) {
case 'serverlessEnv':
grepArgs = ['--grep', '/^(?!.*(^|\\s)@skipInServerless(\\s|$)).*@serverless.*/'];
break;
case 'essEnv':
grepArgs = ['--grep', '/^(?!.*@skipInEss).*@ess.*/'];
break;
case 'qaPeriodicEnv':
grepArgs = ['--grep', '/^(?!.*@skipInServerless|.*@skipInServerlessMKI).*@serverless.*/'];
break;
case 'qaEnv':
grepArgs = ['--grep', '/^(?!.*@skipInServerless|.*@skipInServerlessMKI).*@serverlessQA.*/'];
break;
default:
console.error(
`Unsupported environment: ${environment}.
${commandUsage}
`
);
process.exit(1);
}
}
console.log(
"Spawning child process with command: 'node',",
command,
'--config',
configPath,
...grepArgs,
...args
);
const child = spawn('node', [command, '--config', configPath, ...grepArgs, ...args], {
stdio: 'inherit',
});
child.on('close', (code) => {
console.log(`[index.js] child process closed with code ${code}`);
});
// Listen for process exit
child.on('exit', (code) => {
console.log(`[index.js] child process exited with code ${code}`);
process.exit(code);
});