[Fleet] add upload report to cypress tests (#135282)

* add upload report to cypress tests

* added missing cypress reporter

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/scripts/lifecycle/post_command.sh

@@ -14,6 +14,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'
   buildkite-agent artifact upload 'test/**/screenshots/diff/*.png'

.buildkite/scripts/steps/functional/fleet_cypress.sh

@@ -2,10 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
+source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-fleet-cypress
 

vars/kibanaPipeline.groovy

@@ -183,6 +183,7 @@ def withGcsArtifactUpload(workerName, closure) {
     'target/kibana-*',
     'target/kibana-coverage/jest/**/*',
     'target/kibana-security-solution/**/*.png',
+    'target/kibana-fleet/**/*.png',
     'target/test-metrics/*',
     'target/test-suites-ci-plan.json',
     'test/**/screenshots/diff/*.png',

x-pack/plugins/fleet/cypress/README.md

@@ -22,6 +22,21 @@ A headless browser is a browser simulation program that does not have a user int
 
 This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/fleet_cypress`
 
+## Test Artifacts
+
+When Cypress tests are run headless on the command line, artifacts
+are generated under the `target` directory in the root of Kibana as follows:
+
+- HTML Reports
+  - location: `target/kibana-fleet/cypress/results/output.html`
+- `junit` Reports
+  - location: `target/kibana-fleet/cypress/results`
+- Screenshots (of failed tests)
+  - location: `target/kibana-fleet/cypress/screenshots`
+- Videos
+  - disabled by default, can be enabled by setting env var `CYPRESS_video=true`
+  - location: `target/kibana-fleet/cypress/videos`
+
 ### Test Execution: Examples
 
 #### FTR + Headless (Chrome)

x-pack/plugins/fleet/cypress/cypress.json

@@ -15,5 +15,10 @@
   "videosFolder": "../../../target/kibana-fleet/cypress/videos",
   "viewportHeight": 900,
   "viewportWidth": 1440,
-  "screenshotOnRunFailure": true
+  "screenshotOnRunFailure": true,
+  "env": {
+    "protocol": "http",
+    "hostname": "localhost",
+    "configport": "5601"
+  }
 }

x-pack/plugins/fleet/cypress/reporter_config.json

@@ -0,0 +1,10 @@
+{
+  "reporterEnabled": "mochawesome, mocha-junit-reporter",
+  "reporterOptions": {
+    "html": false,
+    "json": true,
+    "mochaFile": "../../../target/kibana-fleet/cypress/results/TEST-fleet-cypress-[hash].xml",
+    "overwrite": false,
+    "reportDir": "../../../target/kibana-fleet/cypress/results"
+  }
+}

x-pack/plugins/fleet/package.json

@@ -5,9 +5,12 @@
   "private": true,
   "license": "Elastic-License",
   "scripts": {
-    "cypress:open": "../../../node_modules/.bin/cypress open --config-file ./cypress/cypress.json",
+    "cypress": "../../../node_modules/.bin/cypress",
+    "cypress:open": "yarn cypress open --config-file ./cypress/cypress.json",
     "cypress:open-as-ci": "node ../../../scripts/functional_tests --config ../../test/fleet_cypress/visual_config.ts",
-    "cypress:run": "../../../node_modules/.bin/cypress run --config-file ./cypress/cypress.json",
-    "cypress:run-as-ci": "node ../../../scripts/functional_tests --config ../../test/fleet_cypress/cli_config.ts"
+    "cypress:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/integration/**/*.spec.ts'; status=$?; yarn junit:merge && exit $status",
+    "cypress:run-as-ci": "node ../../../scripts/functional_tests --config ../../test/fleet_cypress/cli_config.ts",
+    "cypress:run:reporter": "yarn cypress run --config-file ./cypress/cypress.json --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json",
+    "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-fleet/cypress/results/mochawesome*.json > ../../../target/kibana-fleet/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-fleet/cypress/results/output.json --reportDir ../../../target/kibana-fleet/cypress/results && mkdir -p ../../../target/junit && cp ../../../target/kibana-fleet/cypress/results/*.xml ../../../target/junit/"
   }
 }