[Security Solution] Fix junit report transformation (#165541)

**Fixes:** https://github.com/elastic/kibana/issues/165546

## Summary

This PR fixes junit report transformation for Security Solution build steps (Cypress writes reports in `mochawesome` format and `yarn junit:transform` transforms it to junit format).

## Details

After refactoring it turned out `yarn junit:merge` was moved from `cypress:*` yarn scripts to build step `*.sh` files in `.buildkite/scripts/steps/functional` folder. This way a command to run Cypress tests changed from

```sh
yarn cypress:run:ess
```

to

```sh
yarn cypress:run:ess; status=$?; yarn junit:merge && exit $status
```

In first case any test failure do not lead to early exist and all following commands are executed as yarn runs scripts without preserving shell settings. In the second case failing tests lead to `yarn cypress:run:ess` exit with non-zero exit code so the shell script exits immediately without giving a chance for `yarn junit:merge` to execute.

This problem is solved by disabling early exit on error via `set +e`.

On top of that using of `artifact_paths` config option is redundant as [post command script](https://github.com/elastic/kibana/blob/main/.buildkite/scripts/lifecycle/post_command.sh#L16) upload build artefact via a bildkite agent.

#### [Failed build example](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3049#_)
This commit is contained in:
Maxim Palenov 2023-09-05 07:40:32 +02:00 committed by GitHub
parent 148d341d70
commit deed5e4679
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 16 additions and 44 deletions

View file

@ -96,8 +96,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"
- command: .buildkite/scripts/steps/functional/osquery_cypress.sh
label: 'Osquery Cypress Tests'
@ -110,8 +108,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-osquery/**/*"
- command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
label: 'Synthetics @elastic/synthetics Tests'

View file

@ -146,8 +146,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"
# status_exception: Native role management is not enabled in this Elasticsearch instance
# - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
@ -161,8 +159,6 @@ steps:
# automatic:
# - exit_status: '*'
# limit: 1
# artifact_paths:
# - "target/kibana-security-solution/**/*"
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
label: 'Serverless Security Investigations Cypress Tests'
@ -176,8 +172,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
label: 'Serverless Security Explore Cypress Tests'
@ -191,8 +185,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"
- command: .buildkite/scripts/steps/lint.sh
label: 'Linting'

View file

@ -10,8 +10,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"
- command: .buildkite/scripts/steps/functional/defend_workflows_vagrant.sh
label: 'Defend Workflows Endpoint Cypress Tests'
@ -24,5 +22,3 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"

View file

@ -10,8 +10,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-osquery/**/*"
- command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh
label: 'Osquery Cypress Tests, burning changed specs'
@ -22,8 +20,6 @@ steps:
soft_fail: true
retry:
automatic: false
artifact_paths:
- "target/kibana-osquery/**/*"
# Error: self-signed certificate in certificate chain
# - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
@ -37,5 +33,3 @@ steps:
# automatic:
# - exit_status: '*'
# limit: 1
# artifact_paths:
# - "target/kibana-osquery/**/*"

View file

@ -10,5 +10,3 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"

View file

@ -9,5 +9,3 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-solution/**/*"

View file

@ -10,8 +10,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- 'target/kibana-security-solution/**/*'
- command: .buildkite/scripts/steps/functional/security_solution_explore.sh
label: 'Explore - Security Solution Cypress Tests'
@ -24,8 +22,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- 'target/kibana-security-solution/**/*'
- command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
label: 'Investigations - Security Solution Cypress Tests'
@ -38,8 +34,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- 'target/kibana-security-solution/**/*'
- command: .buildkite/scripts/steps/functional/security_solution_burn.sh
label: 'Security Solution Cypress tests, burning changed specs'
@ -51,8 +45,6 @@ steps:
retry:
automatic: false
soft_fail: true
artifact_paths:
- 'target/kibana-security-solution/**/*'
- command: .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
label: 'Security Solution OpenAPI codegen'

View file

@ -10,5 +10,3 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-threat-intelligence/**/*"

View file

@ -110,8 +110,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-serverless/**/*"
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
label: 'Serverless Explore - Security Solution Cypress Tests'
@ -124,8 +122,6 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-serverless/**/*"
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
label: 'Serverless Investigations - Security Solution Cypress Tests'
@ -138,5 +134,3 @@ steps:
automatic:
- exit_status: '*'
limit: 1
artifact_paths:
- "target/kibana-security-serverless/**/*"

View file

@ -12,4 +12,5 @@ echo "--- Defend Workflows Cypress tests"
cd x-pack/plugins/security_solution
set +e
yarn cypress:dw:run; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Defend Workflows Endpoint Cypress tests"
cd x-pack/plugins/security_solution
set +e
yarn cypress:dw:endpoint:run; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,7 @@ export JOB=kibana-osquery-cypress
echo "--- Osquery Cypress tests"
yarn --cwd x-pack/plugins/osquery cypress:run
cd x-pack/plugins/osquery
set +e
yarn cypress:run; status=$?; yarn junit:merge && exit $status

View file

@ -14,4 +14,5 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'fals
echo "--- Osquery Cypress tests, burning changed specs (Chrome)"
yarn --cwd x-pack/plugins/osquery cypress:changed-specs-only
set +e
yarn cypress:changed-specs-only; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Response Ops Cypress Tests on Security Solution"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:run:respops:ess; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Response Ops Cases Cypress Tests on Security Solution"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:run:cases:ess; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Security Serverless Cypress Tests"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:run:serverless; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Explore - Security Solution Cypress Tests"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:explore:run:serverless; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Investigations Cypress Tests on Serverless"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:investigations:run:serverless; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Security Solution Cypress tests (Chrome)"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:run:ess; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Explore Cypress Tests on Security Solution"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:explore:run:ess; status=$?; yarn junit:merge && exit $status

View file

@ -12,4 +12,5 @@ echo "--- Investigations - Security Solution Cypress Tests"
cd x-pack/test/security_solution_cypress
set +e
yarn cypress:investigations:run:ess; status=$?; yarn junit:merge && exit $status