Migrate existing Cypress tests to Cypress + Cucumber (#57299)

* chore: move gitignore to the cypress directory

* chore: ignore more test files

* fix: do not check the Loading Message

It seems not relevant to the main purpose of these tests

* chore: use cypres + webpack + cucumber scaffolding

See https://github.com/TheBrainFamily/cypress-cucumber-webpack-typescript-example

* chore: add eslint and prettier for code linting

* feat: convert existing Cypress test into BDD style

* feat: add support for using proper Node version in MacOSX

* chore: use tslint

* chore: use old layout

We are keeping cypress as rootDir to follow project's structure. On the other hand,
having a second cypress directory at the 2nd level is the default structure, as shown
in the examples:
- https://github.com/TheBrainFamily/cypress-cucumber-webpack-typescript-example
- a240054d7f/examples/preprocessors__typescript-webpack

* chore: remove prelint script meanwhile we fix TS lint

* chore: move test results to a specific directory

* chore: rename variable following old code

* chore: remove non-needed lints, as we are going to use kibana build

* chore: import snapshot function from cypress

* chore: add readFile utils back from a bad removal

* chore: change format of JSON spec file

It was automatically changed by tests

* chore: move CI directory to the proper layout in order for Jenkins to work

* chore: store test-results from proper dir on Jenkins

* chore: store artifacts properly on Jenkins

* Fix type issues

* chore: rename test application to e2e (end-to-end)

We are keeping the build system within the test application, isolating dependencies

* docs: reorganise docs for APM UI e2e tests

* fix: Use proper cypress support file

* chore: use existing NPM script for running cypress on CI

* chore: update paths in CI scripts

* docs: document how the CI runs the tests

* chore: use Node 10 for tests

* chore: Use kibana's Node version for tests

* chore: run yarn install

* docs: update docs

* fix: path was wrong

* docs: fix paths and flags used to load data

* docs: elasticsearch fix flag

* docs: Bootstrap kibana before running it

* docs: remove outdated info

* chore: move background steps to the scenario

This would avoid not reading the background when the number of scenarios grows

Co-authored-by: Dario Gieselaar <d.gieselaar@gmail.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.ci/end2end.groovy

@@ -13,7 +13,7 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/kibana'
     HOME = "${env.WORKSPACE}"
     APM_ITS = 'apm-integration-testing'
-    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/cypress'
+    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
   }
   options {
@@ -107,7 +107,7 @@ pipeline {
         dir("${BASE_DIR}"){
           sh '''
             jobs -l
-            docker build --tag cypress ${CYPRESS_DIR}/ci
+            docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) ${CYPRESS_DIR}/ci
             docker run --rm -t --user "$(id -u):$(id -g)" \
                     -v `pwd`:/app --network="host" \
                     --name cypress cypress'''
@@ -116,8 +116,8 @@ pipeline {
       post {
         always {
           dir("${BASE_DIR}"){
-            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/screenshots/**,${CYPRESS_DIR}/videos/**,${CYPRESS_DIR}/*e2e-tests.xml")
-            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/*e2e-tests.xml")
+            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/**/screenshots/**,${CYPRESS_DIR}/**/videos/**,${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
+            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
           }
           dir("${APM_ITS}"){
             sh 'docker-compose logs > apm-its.log || true'

.eslintignore

@@ -39,7 +39,7 @@ src/legacy/core_plugins/vis_type_vislib/public/vislib/__tests__/lib/fixtures/moc
 /x-pack/legacy/plugins/infra/common/graphql/types.ts
 /x-pack/legacy/plugins/infra/public/graphql/types.ts
 /x-pack/legacy/plugins/infra/server/graphql/types.ts
-/x-pack/legacy/plugins/apm/cypress/**/snapshots.js
+/x-pack/legacy/plugins/apm/e2e/cypress/**/snapshots.js
 /src/legacy/plugin_discovery/plugin_pack/__tests__/fixtures/plugins/broken
 **/graphql/types.ts
 **/*.js.snap

src/cli/cluster/cluster_manager.ts

@@ -264,7 +264,7 @@ export class ClusterManager {
       fromRoot('src/legacy/server/sass/__tmp__'),
       fromRoot('x-pack/legacy/plugins/reporting/.chromium'),
       fromRoot('x-pack/legacy/plugins/siem/cypress'),
-      fromRoot('x-pack/legacy/plugins/apm/cypress'),
+      fromRoot('x-pack/legacy/plugins/apm/e2e/cypress'),
       fromRoot('x-pack/legacy/plugins/apm/scripts'),
       fromRoot('x-pack/legacy/plugins/canvas/canvas_plugin_src'), // prevents server from restarting twice for Canvas plugin changes,
       'plugins/java_languageserver',

src/dev/ci_setup/setup_env.sh

@@ -168,4 +168,4 @@ if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then
   export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
 fi
 
-export CI_ENV_SETUP=true
+export CI_ENV_SETUP=true

src/dev/run_check_lockfile_symlinks.js

@@ -35,7 +35,7 @@ const IGNORE_FILE_GLOBS = [
   // fixtures aren't used in production, ignore them
   '**/*fixtures*/**/*',
   // cypress isn't used in production, ignore it
-  'x-pack/legacy/plugins/apm/cypress/*',
+  'x-pack/legacy/plugins/apm/e2e/*',
 ];
 
 run(async ({ log }) => {

src/dev/typescript/projects.ts

@@ -30,7 +30,7 @@ export const PROJECTS = [
   new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/siem/cypress/tsconfig.json'), {
     name: 'siem/cypress',
   }),
-  new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/apm/cypress/tsconfig.json'), {
+  new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/apm/e2e/tsconfig.json'), {
     name: 'apm/cypress',
     disableTypeCheck: true,
   }),

x-pack/legacy/plugins/apm/cypress/README.md

@@ -1,63 +0,0 @@
-### How to run
-
-_Note: Run the following commands from `kibana/x-pack/legacy/plugins/apm/cypress`._
-
-#### Interactive mode
-
-```
-yarn cypress open
-```
-
-#### Headless mode
-
-```
-yarn cypress run
-```
-
-### Connect to Elasticsearch on Cloud (internal devs only)
-
-Find the credentials for the cluster [here](https://github.com/elastic/apm-dev/blob/master/docs/credentials/apm-ui-clusters.md#e2e-cluster). The cloud instance contains the static data set
-
-### Kibana
-
-#### `--no-base-path`
-
-Kibana must be started with `yarn start --no-base-path`
-
-#### Content Security Policy (CSP) Settings
-
-Your local or cloud Kibana server must have the `csp.strict: false` setting
-configured in `kibana.dev.yml`, or `kibana.yml`, as shown in the example below:
-
-```yaml
-csp.strict: false
-```
-
-The above setting is required to prevent the _Please upgrade
-your browser_ / _This Kibana installation has strict security requirements
-enabled that your current browser does not meet._ warning that's displayed for
-unsupported user agents, like the one reported by Cypress when running tests.
-
-### Ingest static data into Elasticsearch via APM Server
-
-1. Download [static data file](https://storage.googleapis.com/apm-ui-e2e-static-data/events.json)
-
-2. Post to APM Server
-
-```
-node ingest-data/replay.js --server-url http://localhost:8200 --secret-token abcd --events ./events.json
-```
-
-### Generate static data
-
-Capture data from all agents with [apm-integration-testing](https://github.com/elastic/apm-integration-testing):
-
-```
-./scripts/compose.py start master --all --apm-server-record
-```
-
-To copy the captured data from the container to the host:
-
-```
-docker cp localtesting_8.0.0_apm-server-2:/app/events.json .
-```

x-pack/legacy/plugins/apm/cypress/cypress.json

@@ -1,18 +0,0 @@
-{
-  "baseUrl": "http://localhost:5601",
-  "video": false,
-  "trashAssetsBeforeRuns": false,
-  "fileServerFolder": "../",
-  "fixturesFolder": "./fixtures",
-  "integrationFolder": "./integration",
-  "pluginsFile": "./plugins/index.js",
-  "screenshotsFolder": "./screenshots",
-  "supportFile": "./support/index.ts",
-  "videosFolder": "./videos",
-  "useRelativeSnapshots": true,
-  "reporter": "junit",
-  "reporterOptions": {
-    "mochaFile": "[hash]-e2e-tests.xml",
-    "toConsole": false
-  }
-}

x-pack/legacy/plugins/apm/cypress/integration/apm.spec.ts

@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { loginAndWaitForPage } from './helpers';
-
-describe('When clicking opbeans-go service', () => {
-  before(() => {
-    // open service overview page
-    loginAndWaitForPage(`/app/apm#/services`);
-
-    // show loading text for services
-    cy.contains('Loading...');
-
-    // click opbeans-go service
-    cy.get(':contains(opbeans-go)')
-      .last()
-      .click({ force: true });
-  });
-
-  it('should redirect to correct path with correct params', () => {
-    cy.url().should('contain', `/app/apm#/services/opbeans-go/transactions`);
-    cy.url().should('contain', `transactionType=request`);
-  });
-
-  describe('transaction duration charts', () => {
-    it('should have correct y-axis ticks', () => {
-      const yAxisTick =
-        '[data-cy=transaction-duration-charts] .rv-xy-plot__axis--vertical .rv-xy-plot__axis__tick__text';
-
-      cy.get(yAxisTick)
-        .eq(2)
-        .invoke('text')
-        .snapshot();
-
-      cy.get(yAxisTick)
-        .eq(1)
-        .invoke('text')
-        .snapshot();
-
-      cy.get(yAxisTick)
-        .eq(0)
-        .invoke('text')
-        .snapshot();
-    });
-  });
-
-  describe('TPM charts', () => {});
-
-  describe('Transaction group list', () => {});
-});

x-pack/legacy/plugins/apm/cypress/screenshots/.gitignore

@@ -1 +0,0 @@
-*

x-pack/legacy/plugins/apm/cypress/snapshots.js

@@ -1,3 +0,0 @@
-// auto-generated by @cypress/snapshot
-{
-}

x-pack/legacy/plugins/apm/cypress/support/index.ts

@@ -1,10 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-// @ts-ignore
-import { register } from '@cypress/snapshot';
-
-register();

x-pack/legacy/plugins/apm/e2e/.gitignore

@@ -0,0 +1,4 @@
+cypress/ingest-data/events.json
+cypress/screenshots/*
+
+cypress/test-results

x-pack/legacy/plugins/apm/e2e/README.md

@@ -0,0 +1,68 @@
+# End-To-End (e2e) Test for APM UI
+
+## Ingest static data into Elasticsearch via APM Server
+
+1. Start Elasticsearch and APM Server, using [apm-integration-testing](https://github.com/elastic/apm-integration-testing):
+
+```shell
+$ git clone https://github.com/elastic/apm-integration-testing.git
+$ cd apm-integration-testing
+./scripts/compose.py start master --no-kibana --no-xpack-secure
+```
+
+2. Download [static data file](https://storage.googleapis.com/apm-ui-e2e-static-data/events.json)
+
+```shell
+$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
+$ curl https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output events.json
+```
+
+3. Post to APM Server
+
+```shell
+$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
+$ node replay.js --server-url http://localhost:8200 --secret-token abcd --events ./events.json
+```
+>This process will take a few minutes to ingest all data
+
+4. Start Kibana
+
+```shell
+$ yarn kbn bootstrap
+$ yarn start --no-base-path --csp.strict=false
+```
+
+> Content Security Policy (CSP) Settings: Your Kibana instance must have the `csp.strict: false`.
+
+## How to run the tests
+
+_Note: Run the following commands from `kibana/x-pack/legacy/plugins/apm/e2e/cypress`._
+
+### Interactive mode
+
+```
+yarn cypress open
+```
+
+### Headless mode
+
+```
+yarn cypress run
+```
+
+## Reproducing CI builds
+
+>This process is very slow compared to the local development described above. Consider that the CI must install and configure the build tools and create a Docker image for the project to run tests in a consistent manner.
+
+The Jenkins CI uses a shell script to prepare Kibana:
+
+```shell
+# Prepare and run Kibana locally
+$ x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
+# Build Docker image for Kibana
+$ docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) x-pack/legacy/plugins/apm/e2e/ci 
+# Run Docker image
+$ docker run --rm -t --user "$(id -u):$(id -g)" \
+    -v `pwd`:/app --network="host" \
+    --name cypress cypress
+```

x-pack/legacy/plugins/apm/e2e/ci/Dockerfile

@@ -1,4 +1,5 @@
-FROM node:12
+ARG NODE_VERSION
+FROM node:$NODE_VERSION
 
 RUN apt-get -qq update \
   && apt-get -y -qq install xvfb \

x-pack/legacy/plugins/apm/e2e/ci/entrypoint.sh

@@ -21,9 +21,9 @@ npm config set cache ${HOME}
 #      --exclude=packages/ \
 #      --exclude=built_assets  --exclude=target \
 #      --exclude=data /app ${HOME}/
-#cd ${HOME}/app/x-pack/legacy/plugins/apm/cypress
+#cd ${HOME}/app/x-pack/legacy/plugins/apm/e2e/cypress
 
-cd /app/x-pack/legacy/plugins/apm/cypress
+cd /app/x-pack/legacy/plugins/apm/e2e
 ## Install dependencies for cypress
 CI=true npm install
 yarn install
@@ -33,4 +33,4 @@ npm install wait-on
 ./node_modules/.bin/wait-on ${CYPRESS_BASE_URL}/status && echo 'Kibana is up and running'
 
 # Run cypress
-./node_modules/.bin/cypress run
+npm run cypress:run

x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-CYPRESS_DIR="x-pack/legacy/plugins/apm/cypress"
+CYPRESS_DIR="x-pack/legacy/plugins/apm/e2e"
 
 echo "1/3 Install dependencies ..."
 # shellcheck disable=SC1091

x-pack/legacy/plugins/apm/e2e/cypress.json

@@ -0,0 +1,19 @@
+{
+  "baseUrl": "http://localhost:5601",
+  "video": false,
+  "trashAssetsBeforeRuns": false,
+  "fileServerFolder": "../",
+  "fixturesFolder": "./cypress/fixtures",
+  "integrationFolder": "./cypress/integration",
+  "pluginsFile": "./cypress/plugins/index.js",
+  "screenshotsFolder": "./cypress/screenshots",
+  "supportFile": "./cypress/support/index.ts",
+  "videosFolder": "./cypress/videos",
+  "useRelativeSnapshots": true,
+  "reporter": "junit",
+  "reporterOptions": {
+    "mochaFile": "./cypress/test-results/[hash]-e2e-tests.xml",
+    "toConsole": false
+  },
+  "testFiles": "**/*.{feature,features}"
+}

x-pack/legacy/plugins/apm/e2e/cypress/integration/apm.feature

@@ -0,0 +1,7 @@
+Feature: APM
+
+  Scenario: Transaction duration charts
+    Given a user browses the APM UI application
+    When the user inspects the opbeans-go service
+    Then should redirect to correct path with correct params
+      And should have correct y-axis ticks

x-pack/legacy/plugins/apm/e2e/cypress/integration/snapshots.js

@@ -8,5 +8,12 @@ module.exports = {
       }
     }
   },
-  "__version": "3.4.1"
+  "__version": "3.8.3",
+  "APM": {
+    "Transaction duration charts": {
+      "1": "3.7 min",
+      "2": "1.8 min",
+      "3": "0.0 min"
+    }
+  }
 }

x-pack/legacy/plugins/apm/e2e/cypress/plugins/index.js

@@ -22,22 +22,8 @@ const wp = require('@cypress/webpack-preprocessor');
 const fs = require('fs');
 
 module.exports = on => {
-  // add typescript support
   const options = {
-    webpackOptions: {
-      resolve: {
-        extensions: ['.ts', '.tsx', '.js']
-      },
-      module: {
-        rules: [
-          {
-            test: /\.tsx?$/,
-            loader: 'ts-loader',
-            options: { transpileOnly: true }
-          }
-        ]
-      }
-    }
+    webpackOptions: require('../webpack.config.js')
   };
   on('file:preprocessor', wp(options));
 

x-pack/legacy/plugins/apm/e2e/cypress/support/commands.js

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add("login", (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This is will overwrite an existing command --
+// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })

x-pack/legacy/plugins/apm/e2e/cypress/support/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+import './commands';
+
+// @ts-ignore
+import { register } from '@cypress/snapshot';
+
+register();

x-pack/legacy/plugins/apm/e2e/cypress/support/step_definitions/apm.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Given, When, Then } from 'cypress-cucumber-preprocessor/steps';
+import { loginAndWaitForPage } from '../../integration/helpers';
+
+Given(`a user browses the APM UI application`, () => {
+  // open service overview page
+  loginAndWaitForPage(`/app/apm#/services`);
+});
+
+When(`the user inspects the opbeans-go service`, () => {
+  // click opbeans-go service
+  cy.get(':contains(opbeans-go)')
+    .last()
+    .click({ force: true });
+});
+
+Then(`should redirect to correct path with correct params`, () => {
+  cy.url().should('contain', `/app/apm#/services/opbeans-go/transactions`);
+  cy.url().should('contain', `transactionType=request`);
+});
+
+Then(`should have correct y-axis ticks`, () => {
+  const yAxisTick =
+    '[data-cy=transaction-duration-charts] .rv-xy-plot__axis--vertical .rv-xy-plot__axis__tick__text';
+
+  cy.get(yAxisTick)
+    .eq(2)
+    .invoke('text')
+    .snapshot();
+
+  cy.get(yAxisTick)
+    .eq(1)
+    .invoke('text')
+    .snapshot();
+
+  cy.get(yAxisTick)
+    .eq(0)
+    .invoke('text')
+    .snapshot();
+});

x-pack/legacy/plugins/apm/e2e/cypress/webpack.config.js

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+module.exports = {
+  resolve: {
+    extensions: ['.ts', '.js']
+  },
+  node: { fs: 'empty', child_process: 'empty', readline: 'empty' },
+  module: {
+    rules: [
+      {
+        test: /\.ts$/,
+        exclude: [/node_modules/],
+        use: [
+          {
+            loader: 'ts-loader'
+          }
+        ]
+      },
+      {
+        test: /\.feature$/,
+        use: [
+          {
+            loader: 'cypress-cucumber-preprocessor/loader'
+          }
+        ]
+      },
+      {
+        test: /\.features$/,
+        use: [
+          {
+            loader: 'cypress-cucumber-preprocessor/lib/featuresLoader'
+          }
+        ]
+      }
+    ]
+  }
+};

x-pack/legacy/plugins/apm/e2e/package.json

@@ -5,17 +5,20 @@
   "license": "MIT",
   "scripts": {
     "cypress:open": "cypress open",
-    "cypress:run": "cypress run"
+    "cypress:run": "cypress run --spec **/*.feature"
   },
   "dependencies": {
     "@cypress/snapshot": "^2.1.3",
     "@cypress/webpack-preprocessor": "^4.1.0",
+    "@types/cypress-cucumber-preprocessor": "^1.14.0",
     "@types/js-yaml": "^3.12.1",
+    "@types/node": "^10.12.11",
     "cypress": "^3.5.0",
+    "cypress-cucumber-preprocessor": "^2.0.1",
     "js-yaml": "^3.13.1",
     "p-limit": "^2.2.1",
     "ts-loader": "^6.1.0",
-    "typescript": "3.7.2",
+    "typescript": "3.7.5",
     "webpack": "^4.41.5"
   }
 }

x-pack/legacy/plugins/apm/e2e/tsconfig.json

@@ -1,8 +1,13 @@
 {
   "extends": "../../../../tsconfig.json",
   "exclude": [],
-  "include": ["./**/*"],
+  "include": [
+    "./**/*"
+  ],
   "compilerOptions": {
-    "types": ["cypress", "node"]
+    "types": [
+      "cypress",
+      "node"
+    ]
   }
 }

x-pack/tsconfig.json

@@ -12,7 +12,7 @@
   "exclude": [
     "test/**/*",
     "legacy/plugins/siem/cypress/**/*",
-    "legacy/plugins/apm/cypress/**/*",
+    "legacy/plugins/apm/e2e/cypress/**/*",
     "**/typespec_tests.ts"
   ],
   "compilerOptions": {