github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[security_solution] Smarter retries for Cypress (#179585)

Browse source 
## Summary

This PR introduces a significant improvement in the way we handle
failing tests within our Cypress test suite. Previously, when a test
spec failed during a job, our approach was to retry the entire set of
specs, which was not only time-consuming but also inefficient. This
process often resulted in unnecessary reruns of tests that had already
passed, leading to increased resource consumption and longer feedback
cycles for developers.

With the changes introduced in this PR, we now target a more efficient
and logical approach by retrying only the specific spec that failed,
rather than the entire suite. This focused retry logic means that if a
job encounters a failing test, only that particular test will be rerun.
This adjustment significantly reduces the overall execution time of our
test suite and minimizes the consumption of valuable Builtkie resources.

Key benefits of this change include:

- **Reduced Test Execution Time**: By avoiding unnecessary reruns of
passing tests, we significantly cut down the total time spent on test
executions.
- **Improved Resource Utilization**: This change ensures a more
judicious use of our CI/CD resources, allowing for more efficient
processing of jobs and reducing potential bottlenecks in our testing
pipeline.
- **Faster Feedback Loops**: Developers will receive quicker feedback on
the status of their tests, enabling them to address failures more
promptly and efficiently.
- **Increased Test Suite Reliability**: By focusing on retrying only the
failing tests, we can more accurately identify flaky tests and work
towards improving the stability of our test suite.
This commit is contained in:
Patryk Kopyciński 2024-03-29 16:28:36 +01:00 • committed by GitHub
parent c02c0b3c53
commit aa6a905ffd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 465 additions and 423 deletions
Download patch file Download diff file Expand all files Collapse all files

32
.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
View file

@ -66,7 +66,7 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
@ -79,7 +79,7 @@ steps:
parallelism: 4
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
@ -92,7 +92,7 @@ steps:
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
@ -102,10 +102,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 8
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
@ -115,10 +115,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 4
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
@ -128,10 +128,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
@ -141,10 +141,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
@ -157,7 +157,7 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
@ -167,10 +167,10 @@ steps:
queue: n2-4-virt
depends_on: build
timeout_in_minutes: 60
parallelism: 10
parallelism: 12
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
@ -180,10 +180,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 7
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- wait: ~

68
.buildkite/pipelines/on_merge.yml
View file

@ -88,7 +88,7 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
@ -97,10 +97,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 4
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
@ -112,7 +112,7 @@ steps:
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
@ -121,10 +121,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 8
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
@ -133,10 +133,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 4
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh
@ -145,10 +145,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 8
parallelism: 4
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh
@ -160,7 +160,7 @@ steps:
parallelism: 6
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
@ -169,10 +169,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
@ -184,7 +184,7 @@ steps:
parallelism: 6
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
@ -193,10 +193,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 8
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
@ -208,7 +208,7 @@ steps:
parallelism: 6
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
@ -220,7 +220,7 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
@ -232,7 +232,7 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh
@ -244,7 +244,7 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_explore.sh
@ -253,10 +253,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 4
parallelism: 3
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
@ -265,10 +265,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 8
parallelism: 7
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/threat_intelligence.sh
@ -277,10 +277,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 2
parallelism: 1
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/osquery_cypress.sh
@ -289,10 +289,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
@ -301,10 +301,10 @@ steps:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 6
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/defend_workflows.sh
@ -314,10 +314,10 @@ steps:
depends_on:
- build
timeout_in_minutes: 60
parallelism: 16
parallelism: 20
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
@ -327,10 +327,10 @@ steps:
depends_on:
- build
timeout_in_minutes: 60
parallelism: 10
parallelism: 14
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'

10
.buildkite/pipelines/on_merge_unsupported_ftrs.yml
View file

@ -35,7 +35,7 @@ steps:
limit: 1
- command: .buildkite/scripts/steps/functional/profiling_cypress.sh
label: 'Profling Cypress Tests'
label: 'Profiling Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
@ -68,10 +68,10 @@ steps:
queue: n2-4-virt
depends_on: build
timeout_in_minutes: 60
parallelism: 16
parallelism: 20
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
@ -80,8 +80,8 @@ steps:
queue: n2-4-virt
depends_on: build
timeout_in_minutes: 60
parallelism: 10
parallelism: 14
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

2
.buildkite/pipelines/pull_request/response_ops.yml
View file

@ -7,7 +7,7 @@ steps:
- build
- quick_checks
timeout_in_minutes: 120
parallelism: 4
parallelism: 9
retry:
automatic:
- exit_status: '*'

8
.buildkite/pipelines/pull_request/security_solution/ai_assistant.yml
View file

@ -10,8 +10,8 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
label: 'AI Assistant - Security Solution Cypress Tests'
@ -24,5 +24,5 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

10
.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml
View file

@ -7,10 +7,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 16
parallelism: 20
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
@ -21,10 +21,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 10
parallelism: 14
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
# status_exception: Native role management is not enabled in this Elasticsearch instance
@ -36,5 +36,5 @@ steps:
# timeout_in_minutes: 60
# retry:
# automatic:
# - exit_status: '*'
# - exit_status: '-1'
# limit: 1

20
.buildkite/pipelines/pull_request/security_solution/detection_engine.yml
View file

@ -1,4 +1,4 @@
steps:
steps:
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
label: 'Serverless Detection Engine - Security Solution Cypress Tests'
agents:
@ -7,10 +7,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
@ -21,12 +21,12 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
label: 'Detection Engine - Security Solution Cypress Tests'
agents:
@ -35,10 +35,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 8
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
@ -49,8 +49,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

4
.buildkite/pipelines/pull_request/security_solution/entity_analytics.yml
View file

@ -10,7 +10,7 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh
@ -24,5 +24,5 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

12
.buildkite/pipelines/pull_request/security_solution/explore.yml
View file

@ -1,4 +1,4 @@
steps:
steps:
- command: .buildkite/scripts/steps/functional/security_solution_explore.sh
label: 'Explore - Security Solution Cypress Tests'
agents:
@ -7,10 +7,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 4
parallelism: 3
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
@ -21,8 +21,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 4
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

12
.buildkite/pipelines/pull_request/security_solution/investigations.yml
View file

@ -7,12 +7,12 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 8
parallelism: 7
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
label: 'Serverless Investigations - Security Solution Cypress Tests'
agents:
@ -21,8 +21,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 8
parallelism: 7
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

8
.buildkite/pipelines/pull_request/security_solution/osquery_cypress.yml
View file

@ -7,10 +7,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
@ -21,8 +21,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

20
.buildkite/pipelines/pull_request/security_solution/rule_management.yml
View file

@ -1,4 +1,4 @@
steps:
steps:
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
label: 'Serverless Rule Management - Security Solution Cypress Tests'
agents:
@ -7,10 +7,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 8
parallelism: 5
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
@ -21,10 +21,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 4
parallelism: 2
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh
@ -35,10 +35,10 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 8
parallelism: 4
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh
@ -49,8 +49,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 6
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

4
.buildkite/pipelines/pull_request/security_solution/threat_intelligence.yml
View file

@ -7,8 +7,8 @@ steps:
- build
- quick_checks
timeout_in_minutes: 60
parallelism: 2
parallelism: 1
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

6
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml
View file

@ -47,7 +47,7 @@ steps:
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
@ -67,5 +67,5 @@ steps:
parallelism: 6
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

4
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml
View file

@ -47,5 +47,5 @@ steps:
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

2
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml
View file

@ -47,5 +47,5 @@ steps:
parallelism: 4
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1

4
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml
View file

@ -47,5 +47,5 @@ steps:
parallelism: 1
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

4
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml
View file

@ -47,5 +47,5 @@ steps:
parallelism: 8
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

6
.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml
View file

@ -47,7 +47,7 @@ steps:
parallelism: 8
retry:
automatic:
- exit_status: '*'
- exit_status: '-1'
limit: 1
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules
@ -67,5 +67,5 @@ steps:
parallelism: 4
retry:
automatic:
- exit_status: '*'
limit: 1
- exit_status: '-1'
limit: 1

410
x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts
View file

@ -204,69 +204,72 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
_.pull(fleetServerPorts, fleetServerPort);
};
await pMap(
files,
async (filePath) => {
let result:
| CypressCommandLine.CypressRunResult
| CypressCommandLine.CypressFailedRunResult
| undefined;
await withProcRunner(log, async (procs) => {
const abortCtrl = new AbortController();
const failedSpecFilePaths: string[] = [];
const onEarlyExit = (msg: string) => {
log.error(msg);
abortCtrl.abort();
};
const runSpecs = async (filePaths: string[]) =>
pMap(
filePaths,
async (filePath) => {
let result:
| CypressCommandLine.CypressRunResult
| CypressCommandLine.CypressFailedRunResult
| undefined;
await withProcRunner(log, async (procs) => {
const abortCtrl = new AbortController();
const esPort: number = getEsPort();
const kibanaPort: number = getKibanaPort();
const fleetServerPort: number = getFleetServerPort();
const specFileFTRConfig = parseTestFileConfig(filePath);
const ftrConfigFilePath = path.resolve(
_.isArray(argv.ftrConfigFile) ? _.last(argv.ftrConfigFile) : argv.ftrConfigFile
);
const config = await getFTRConfig({
log,
esPort,
kibanaPort,
fleetServerPort,
ftrConfigFilePath,
specFilePath: filePath,
specFileFTRConfig,
isOpen,
});
const createUrlFromFtrConfig = (
type: 'elasticsearch' | 'kibana' | 'fleetserver',
withAuth: boolean = false
): string => {
const getKeyPath = (keyPath: string = ''): string => {
return `servers.${type}${keyPath ? `.${keyPath}` : ''}`;
const onEarlyExit = (msg: string) => {
log.error(msg);
abortCtrl.abort();
};
if (!config.get(getKeyPath())) {
throw new Error(`Unable to create URL for ${type}. Not found in FTR config at `);
}
const esPort: number = getEsPort();
const kibanaPort: number = getKibanaPort();
const fleetServerPort: number = getFleetServerPort();
const specFileFTRConfig = parseTestFileConfig(filePath);
const ftrConfigFilePath = path.resolve(
_.isArray(argv.ftrConfigFile) ? _.last(argv.ftrConfigFile) : argv.ftrConfigFile
);
const url = new URL('http://localhost');
const config = await getFTRConfig({
log,
esPort,
kibanaPort,
fleetServerPort,
ftrConfigFilePath,
specFilePath: filePath,
specFileFTRConfig,
isOpen,
});
url.port = config.get(getKeyPath('port'));
url.protocol = config.get(getKeyPath('protocol'));
url.hostname = config.get(getKeyPath('hostname'));
const createUrlFromFtrConfig = (
type: 'elasticsearch' | 'kibana' | 'fleetserver',
withAuth: boolean = false
): string => {
const getKeyPath = (keyPath: string = ''): string => {
return `servers.${type}${keyPath ? `.${keyPath}` : ''}`;
};
if (withAuth) {
url.username = config.get(getKeyPath('username'));
url.password = config.get(getKeyPath('password'));
}
if (!config.get(getKeyPath())) {
throw new Error(`Unable to create URL for ${type}. Not found in FTR config at `);
}
return url.toString().replace(/\/$/, '');
};
const url = new URL('http://localhost');
const baseUrl = createUrlFromFtrConfig('kibana');
url.port = config.get(getKeyPath('port'));
url.protocol = config.get(getKeyPath('protocol'));
url.hostname = config.get(getKeyPath('hostname'));
log.info(`
if (withAuth) {
url.username = config.get(getKeyPath('username'));
url.password = config.get(getKeyPath('password'));
}
return url.toString().replace(/\/$/, '');
};
const baseUrl = createUrlFromFtrConfig('kibana');
log.info(`
----------------------------------------------
Cypress FTR setup for file: ${filePath}:
----------------------------------------------
@ -286,116 +289,116 @@ ${JSON.stringify(
----------------------------------------------
`);
const lifecycle = new Lifecycle(log);
const lifecycle = new Lifecycle(log);
const providers = new ProviderCollection(log, [
...readProviderSpec('Service', {
lifecycle: () => lifecycle,
log: () => log,
config: () => config,
}),
...readProviderSpec('Service', config.get('services')),
]);
const options = {
installDir: process.env.KIBANA_INSTALL_DIR,
ci: process.env.CI,
};
const shutdownEs = await pRetry(
async () =>
runElasticsearch({
config,
log,
name: `ftr-${esPort}`,
esFrom: config.get('esTestCluster')?.from || 'snapshot',
onEarlyExit,
const providers = new ProviderCollection(log, [
...readProviderSpec('Service', {
lifecycle: () => lifecycle,
log: () => log,
config: () => config,
}),
{ retries: 2, forever: false }
);
...readProviderSpec('Service', config.get('services')),
]);
await runKibanaServer({
procs,
config,
installDir: options?.installDir,
extraKbnOpts:
options?.installDir || options?.ci || !isOpen
? []
: ['--dev', '--no-dev-config', '--no-dev-credentials'],
onEarlyExit,
inspect: argv.inspect,
});
const options = {
installDir: process.env.KIBANA_INSTALL_DIR,
ci: process.env.CI,
};
// Setup fleet if Cypress config requires it
let fleetServer: void | StartedFleetServer;
if (cypressConfigFile.env?.WITH_FLEET_SERVER) {
log.info(`Setting up fleet-server for this Cypress config`);
const shutdownEs = await pRetry(
async () =>
runElasticsearch({
config,
log,
name: `ftr-${esPort}`,
esFrom: config.get('esTestCluster')?.from || 'snapshot',
onEarlyExit,
}),
{ retries: 2, forever: false }
);
const kbnClient = createKbnClient({
url: baseUrl,
username: config.get('servers.kibana.username'),
password: config.get('servers.kibana.password'),
await runKibanaServer({
procs,
config,
installDir: options?.installDir,
extraKbnOpts:
options?.installDir || options?.ci || !isOpen
? []
: ['--dev', '--no-dev-config', '--no-dev-credentials'],
onEarlyExit,
inspect: argv.inspect,
});
// Setup fleet if Cypress config requires it
let fleetServer: void | StartedFleetServer;
if (cypressConfigFile.env?.WITH_FLEET_SERVER) {
log.info(`Setting up fleet-server for this Cypress config`);
const kbnClient = createKbnClient({
url: baseUrl,
username: config.get('servers.kibana.username'),
password: config.get('servers.kibana.password'),
log,
});
fleetServer = await startFleetServer({
kbnClient,
logger: log,
port:
fleetServerPort ?? config.has('servers.fleetserver.port')
? (config.get('servers.fleetserver.port') as number)
: undefined,
// `force` is needed to ensure that any currently running fleet server (perhaps left
// over from an interrupted run) is killed and a new one restarted
force: true,
});
}
await providers.loadAll();
const functionalTestRunner = new FunctionalTestRunner(
log,
config,
EsVersion.getDefault()
);
const ftrEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), {
retries: 1,
});
fleetServer = await startFleetServer({
kbnClient,
logger: log,
port:
fleetServerPort ?? config.has('servers.fleetserver.port')
? (config.get('servers.fleetserver.port') as number)
: undefined,
// `force` is needed to ensure that any currently running fleet server (perhaps left
// over from an interrupted run) is killed and a new one restarted
force: true,
});
}
log.debug(
`Env. variables returned by [functionalTestRunner.run()]:\n`,
JSON.stringify(ftrEnv, null, 2)
);
await providers.loadAll();
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
...ftrEnv,
const functionalTestRunner = new FunctionalTestRunner(
log,
config,
EsVersion.getDefault()
);
// NOTE:
// ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
// points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
ELASTICSEARCH_URL:
ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_URL_WITH_AUTH: createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_USERNAME:
ftrEnv.ELASTICSEARCH_USERNAME ?? config.get('servers.elasticsearch.username'),
ELASTICSEARCH_PASSWORD:
ftrEnv.ELASTICSEARCH_PASSWORD ?? config.get('servers.elasticsearch.password'),
const ftrEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), {
retries: 1,
});
FLEET_SERVER_URL: createUrlFromFtrConfig('fleetserver'),
log.debug(
`Env. variables returned by [functionalTestRunner.run()]:\n`,
JSON.stringify(ftrEnv, null, 2)
);
KIBANA_URL: baseUrl,
KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
KIBANA_USERNAME: config.get('servers.kibana.username'),
KIBANA_PASSWORD: config.get('servers.kibana.password'),
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
...ftrEnv,
IS_SERVERLESS: config.get('serverless'),
// NOTE:
// ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
// points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
ELASTICSEARCH_URL:
ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_URL_WITH_AUTH: createUrlFromFtrConfig('elasticsearch', true),
ELASTICSEARCH_USERNAME:
ftrEnv.ELASTICSEARCH_USERNAME ?? config.get('servers.elasticsearch.username'),
ELASTICSEARCH_PASSWORD:
ftrEnv.ELASTICSEARCH_PASSWORD ?? config.get('servers.elasticsearch.password'),
...argv.env,
};
FLEET_SERVER_URL: createUrlFromFtrConfig('fleetserver'),
KIBANA_URL: baseUrl,
KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
KIBANA_USERNAME: config.get('servers.kibana.username'),
KIBANA_PASSWORD: config.get('servers.kibana.password'),
IS_SERVERLESS: config.get('serverless'),
...argv.env,
};
log.info(`
log.info(`
----------------------------------------------
Cypress run ENV for file: ${filePath}:
----------------------------------------------
@ -405,65 +408,86 @@ ${JSON.stringify(cyCustomEnv, null, 2)}
----------------------------------------------
`);
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
config: {
e2e: {
baseUrl,
},
env: cyCustomEnv,
},
});
} else {
try {
result = await cypress.run({
browser: 'electron',
spec: filePath,
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
reporter: argv.reporter as string,
reporterOptions: argv.reporterOptions,
headed: argv.headed as boolean,
config: {
e2e: {
baseUrl,
},
numTestsKeptInMemory: 0,
env: cyCustomEnv,
},
});
} catch (error) {
result = error;
} else {
try {
result = await cypress.run({
browser: 'electron',
spec: filePath,
configFile: cypressConfigFilePath,
reporter: argv.reporter as string,
reporterOptions: argv.reporterOptions,
headed: argv.headed as boolean,
config: {
e2e: {
baseUrl,
},
numTestsKeptInMemory: 0,
env: cyCustomEnv,
},
});
if ((result as CypressCommandLine.CypressRunResult)?.totalFailed) {
failedSpecFilePaths.push(filePath);
}
} catch (error) {
result = error;
failedSpecFilePaths.push(filePath);
}
}
}
if (fleetServer) {
await fleetServer.stop();
}
if (fleetServer) {
await fleetServer.stop();
}
await procs.stop('kibana');
await shutdownEs();
cleanupServerPorts({ esPort, kibanaPort, fleetServerPort });
await procs.stop('kibana');
await shutdownEs();
cleanupServerPorts({ esPort, kibanaPort, fleetServerPort });
return result;
});
return result;
});
return result;
},
{
concurrency: 1,
}
).then((results) => {
renderSummaryTable(results as CypressCommandLine.CypressRunResult[]);
const hasFailedTests = _.some(
results,
(result) =>
(result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
(result as CypressCommandLine.CypressRunResult)?.totalFailed
},
{
concurrency: 1,
}
);
if (hasFailedTests) {
throw createFailError('Not all tests passed');
}
});
const initialResults = await runSpecs(files);
// If there are failed tests, retry them
const retryResults = await runSpecs([...failedSpecFilePaths]);
renderSummaryTable([
// Don't include failed specs from initial run in results
..._.filter(
initialResults,
(initialResult: CypressCommandLine.CypressRunResult) =>
initialResult?.runs &&
_.some(
initialResult?.runs,
(runResult) => !failedSpecFilePaths.includes(runResult.spec.absolute)
)
),
...retryResults,
] as CypressCommandLine.CypressRunResult[]);
const hasFailedTests = _.some(
// only fail the job if retry failed as well
retryResults,
(result) =>
(result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
(result as CypressCommandLine.CypressRunResult)?.totalFailed
);
if (hasFailedTests) {
throw createFailError('Not all tests passed');
}
},
{
flags: {

240
x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts
View file

@ -525,153 +525,171 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
return process.exit(0);
}
const results = await pMap(
files,
async (filePath) => {
let result:
| CypressCommandLine.CypressRunResult
| CypressCommandLine.CypressFailedRunResult
| undefined;
await withProcRunner(log, async (procs) => {
const id = crypto.randomBytes(8).toString('hex');
const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`;
const failedSpecFilePaths: string[] = [];
const productTypes = isOpen
? getProductTypes(tier, endpointAddon, cloudAddon)
: (parseTestFileConfig(filePath).productTypes as ProductType[]);
const runSpecs = (filePaths: string[]) =>
pMap(
filePaths,
async (filePath) => {
let result:
| CypressCommandLine.CypressRunResult
| CypressCommandLine.CypressFailedRunResult
| undefined;
await withProcRunner(log, async (procs) => {
const id = crypto.randomBytes(8).toString('hex');
const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`;
if (!API_KEY) {
log.info('API KEY to create project could not be retrieved.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
const productTypes = isOpen
? getProductTypes(tier, endpointAddon, cloudAddon)
: (parseTestFileConfig(filePath).productTypes as ProductType[]);
log.info(`${id}: Creating project ${PROJECT_NAME}...`);
// Creating project for the test to run
const project = await createSecurityProject(PROJECT_NAME, API_KEY, productTypes);
if (!API_KEY) {
log.info('API KEY to create project could not be retrieved.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
if (!project) {
log.info('Failed to create project.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
log.info(`${id}: Creating project ${PROJECT_NAME}...`);
// Creating project for the test to run
const project = await createSecurityProject(PROJECT_NAME, API_KEY, productTypes);
context.addCleanupTask(() => {
const command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`;
exec(command);
});
if (!project) {
log.info('Failed to create project.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
// Reset credentials for elastic user
const credentials = await resetCredentials(project.id, id, API_KEY);
context.addCleanupTask(() => {
const command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`;
exec(command);
});
if (!credentials) {
log.info('Credentials could not be reset.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
// Reset credentials for elastic user
const credentials = await resetCredentials(project.id, id, API_KEY);
// Wait for project to be initialized
await waitForProjectInitialized(project.id, API_KEY);
if (!credentials) {
log.info('Credentials could not be reset.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
// Base64 encode the credentials in order to invoke ES and KB APIs
const auth = btoa(`${credentials.username}:${credentials.password}`);
// Wait for project to be initialized
await waitForProjectInitialized(project.id, API_KEY);
// Wait for elasticsearch status to go green.
await waitForEsStatusGreen(project.es_url, auth, id);
// Base64 encode the credentials in order to invoke ES and KB APIs
const auth = btoa(`${credentials.username}:${credentials.password}`);
// Wait until Kibana is available
await waitForKibanaAvailable(project.kb_url, auth, id);
// Wait for elasticsearch status to go green.
await waitForEsStatusGreen(project.es_url, auth, id);
// Wait for Elasticsearch to be accessible
await waitForEsAccess(project.es_url, auth, id);
// Wait until Kibana is available
await waitForKibanaAvailable(project.kb_url, auth, id);
// Wait until application is ready
await waitForKibanaLogin(project.kb_url, credentials);
// Wait for Elasticsearch to be accessible
await waitForEsAccess(project.es_url, auth, id);
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
CYPRESS_BASE_URL: project.kb_url,
// Wait until application is ready
await waitForKibanaLogin(project.kb_url, credentials);
ELASTICSEARCH_URL: project.es_url,
ELASTICSEARCH_USERNAME: credentials.username,
ELASTICSEARCH_PASSWORD: credentials.password,
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
CYPRESS_BASE_URL: project.kb_url,
KIBANA_URL: project.kb_url,
KIBANA_USERNAME: credentials.username,
KIBANA_PASSWORD: credentials.password,
ELASTICSEARCH_URL: project.es_url,
ELASTICSEARCH_USERNAME: credentials.username,
ELASTICSEARCH_PASSWORD: credentials.password,
// Both CLOUD_SERVERLESS and IS_SERVERLESS are used by the cypress tests.
CLOUD_SERVERLESS: true,
IS_SERVERLESS: true,
// TEST_CLOUD is used by SvlUserManagerProvider to define if testing against cloud.
TEST_CLOUD: 1,
};
KIBANA_URL: project.kb_url,
KIBANA_USERNAME: credentials.username,
KIBANA_PASSWORD: credentials.password,
if (process.env.DEBUG && !process.env.CI) {
log.info(`
// Both CLOUD_SERVERLESS and IS_SERVERLESS are used by the cypress tests.
CLOUD_SERVERLESS: true,
IS_SERVERLESS: true,
// TEST_CLOUD is used by SvlUserManagerProvider to define if testing against cloud.
TEST_CLOUD: 1,
};
if (process.env.DEBUG && !process.env.CI) {
log.info(`
----------------------------------------------
Cypress run ENV for file: ${filePath}:
----------------------------------------------
${JSON.stringify(cyCustomEnv, null, 2)}
----------------------------------------------
`);
}
process.env.TEST_CLOUD_HOST_NAME = new URL(BASE_ENV_URL).hostname;
}
process.env.TEST_CLOUD_HOST_NAME = new URL(BASE_ENV_URL).hostname;
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
config: {
e2e: {
baseUrl: project.kb_url,
},
env: cyCustomEnv,
},
});
} else {
try {
result = await cypress.run({
browser: 'electron',
spec: filePath,
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
reporter: argv.reporter as string,
reporterOptions: argv.reporterOptions,
headed: argv.headed as boolean,
config: {
e2e: {
baseUrl: project.kb_url,
},
numTestsKeptInMemory: 0,
env: cyCustomEnv,
},
});
// Delete serverless project
log.info(`${id} : Deleting project ${PROJECT_NAME}...`);
await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY);
} catch (error) {
result = error;
} else {
try {
result = await cypress.run({
browser: 'electron',
spec: filePath,
configFile: cypressConfigFilePath,
reporter: argv.reporter as string,
reporterOptions: argv.reporterOptions,
headed: argv.headed as boolean,
config: {
e2e: {
baseUrl: project.kb_url,
},
numTestsKeptInMemory: 0,
env: cyCustomEnv,
},
});
// Delete serverless project
log.info(`${id} : Deleting project ${PROJECT_NAME}...`);
await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY);
} catch (error) {
result = error;
}
}
}
return result;
});
return result;
});
return result;
},
{
concurrency: PARALLEL_COUNT,
}
);
if (results) {
renderSummaryTable(results as CypressCommandLine.CypressRunResult[]);
const hasFailedTests = _.some(
results,
(result) =>
(result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
(result as CypressCommandLine.CypressRunResult)?.totalFailed
},
{
concurrency: PARALLEL_COUNT,
}
);
if (hasFailedTests) {
throw createFailError('Not all tests passed');
}
const initialResults = await runSpecs(files);
// If there are failed tests, retry them
const retryResults = await runSpecs([...failedSpecFilePaths]);
renderSummaryTable([
// Don't include failed specs from initial run in results
..._.filter(
initialResults,
(initialResult: CypressCommandLine.CypressRunResult) =>
initialResult?.runs &&
_.some(
initialResult?.runs,
(runResult) => !failedSpecFilePaths.includes(runResult.spec.absolute)
)
),
...retryResults,
] as CypressCommandLine.CypressRunResult[]);
const hasFailedTests = _.some(
// only fail the job if retry failed as well
retryResults,
(result) =>
(result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
(result as CypressCommandLine.CypressRunResult)?.totalFailed
);
if (hasFailedTests) {
throw createFailError('Not all tests passed');
}
},
{

2
x-pack/test/security_solution_cypress/package.json
View file

@ -15,7 +15,7 @@
"cypress:detection_engine:run:ess":"yarn cypress:ess --spec './cypress/e2e/detection_response/detection_engine/!(exceptions)/**/*.cy.ts'",
"cypress:detection_engine:exceptions:run:ess": "yarn cypress:ess --spec './cypress/e2e/detection_response/detection_engine/exceptions/**/*.cy.ts'",
"cypress:ai_assistant:run:ess":"yarn cypress:ess --spec './cypress/e2e/ai_assistant/**/*.cy.ts'",
"cypress:run:respops:ess": "yarn cypress:ess --spec './cypress/e2e/(detection_response)/**/*.cy.ts'",
"cypress:run:respops:ess": "yarn cypress:ess --spec './cypress/e2e/detection_response/**/*.cy.ts'",
"cypress:investigations:run:ess": "yarn cypress:ess --spec './cypress/e2e/investigations/**/*.cy.ts'",
"cypress:explore:run:ess": "yarn cypress:ess --spec './cypress/e2e/explore/**/*.cy.ts'",
"cypress:changed-specs-only:ess": "yarn cypress:ess --changed-specs-only --env burn=5",