[APM] tests restructure (#110703)

* refactoring test

* improving test structure

* improvements

* improving readmes

* readme

* test

* fixing ts issue

* addressing pr comments

* fixing readme files

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

test/scripts/jenkins_apm_cypress.sh

@@ -6,7 +6,7 @@ echo " -> Running APM cypress tests"
 cd "$XPACK_DIR"
 
 checks-reporter-with-killswitch "APM Cypress Tests" \
- node plugins/apm/scripts/ftr_e2e/cypress_run
+ node plugins/apm/scripts/test/e2e.js
 
 echo ""
 echo ""

x-pack/plugins/apm/ftr_e2e/README.md

@@ -4,23 +4,4 @@ APM uses [FTR](../../../../packages/kbn-test/README.md) (functional test runner)
 
 ## Running tests
 
-**Run all tests**
-
-```sh
-//kibana directory
-node x-pack/plugins/apm/scripts/ftr_e2e/cypress_run.js
-```
-
-**Run specific test**
-
-```sh
-//kibana directory
-node x-pack/plugins/apm/scripts/ftr_e2e/cypress_run.js --spec ./cypress/integration/read_only_user/home.spec.ts
-```
-
-## Opening tests
-
-```sh
-//kibana directory
-node x-pack/plugins/apm/scripts/ftr_e2e/cypress_open.js
-```
+Go to [tests documentation](../scripts/test#e2e-tests-cypress/README.md)

x-pack/plugins/apm/ftr_e2e/cypress_start.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+/* eslint-disable no-console */
+
 import Url from 'url';
 import cypress from 'cypress';
 import { FtrProviderContext } from './ftr_provider_context';
@@ -59,6 +61,7 @@ async function cypressStart(
     },
   });
 
+  console.log('Loading esArchiver...');
   await esArchiverLoad('apm_8.0.0');
 
   const res = await cypressExecution({
@@ -71,6 +74,7 @@ async function cypressStart(
     },
   });
 
+  console.log('Removing esArchiver...');
   await esArchiverUnload('apm_8.0.0');
 
   return res;

x-pack/plugins/apm/readme.md

@@ -31,88 +31,7 @@ _Docker Compose is required_
 
 ## Testing
 
-### Cypress tests
-
-See [ftr_e2e](./ftr_e2e)
-
-### Jest tests
-
-Note: Run the following commands from `kibana/x-pack/plugins/apm`.
-
-#### Run
-
-```
-npx jest --watch
-```
-
-#### Update snapshots
-
-```
-npx jest --updateSnapshot
-```
-
-#### Coverage
-
-HTML coverage report can be found in target/coverage/jest after tests have run.
-
-```
-open target/coverage/jest/index.html
-```
-
-### Functional tests
-
-**Start server**
-
-```
-node scripts/functional_tests_server --config x-pack/test/functional/config.js
-```
-
-**Run tests**
-
-```
-node scripts/functional_test_runner --config x-pack/test/functional/config.js --grep='APM specs'
-```
-
-APM tests are located in `x-pack/test/functional/apps/apm`.
-For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme)
-
-### API integration tests
-
-API tests are separated in two suites:
-
-- a basic license test suite
-- a trial license test suite (the equivalent of gold+)
-
-This requires separate test servers and test runners.
-
-**Basic**
-
-```
-# Start server
-node scripts/functional_tests_server --config x-pack/test/apm_api_integration/basic/config.ts
-
-# Run tests
-node scripts/functional_test_runner --config x-pack/test/apm_api_integration/basic/config.ts
-```
-
-The API tests for "basic" are located in `x-pack/test/apm_api_integration/basic/tests`.
-
-**Trial**
-
-```
-# Start server
-node scripts/functional_tests_server --config x-pack/test/apm_api_integration/trial/config.ts
-
-# Run tests
-node scripts/functional_test_runner --config x-pack/test/apm_api_integration/trial/config.ts
-```
-
-The API tests for "trial" are located in `x-pack/test/apm_api_integration/trial/tests`.
-
-**API Test tips**
-
-- For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme)
-- To update snapshots append `--updateSnapshots` to the functional_test_runner command
+Go to [tests documentation](./scripts/test/README.md)
 
 ## Linting
 

x-pack/plugins/apm/scripts/ftr_e2e/cypress_open.js

@@ -1,16 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-const childProcess = require('child_process');
-const path = require('path');
-
-const e2eDir = path.join(__dirname, '../../ftr_e2e');
-
-childProcess.execSync(
-  `node ../../../../scripts/functional_tests --config ./cypress_open.ts`,
-  { cwd: e2eDir, stdio: 'inherit' }
-);

x-pack/plugins/apm/scripts/ftr_e2e/cypress_run.js

@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-const { argv } = require('yargs');
-const childProcess = require('child_process');
-const path = require('path');
-
-const { grep } = argv;
-
-const e2eDir = path.join(__dirname, '../../ftr_e2e');
-
-childProcess.execSync(
-  `node ../../../../scripts/functional_tests --config ./cypress_run.ts --grep ${grep}`,
-  { cwd: e2eDir, stdio: 'inherit' }
-);

x-pack/plugins/apm/scripts/test/README.md

@@ -0,0 +1,63 @@
+## Unit Tests (Jest)
+
+```
+node scripts/tests/jest [--watch] [--updateSnapshot]
+```
+
+#### Coverage
+
+HTML coverage report can be found in target/coverage/jest after tests have run.
+
+```
+open target/coverage/jest/index.html
+```
+
+---
+
+## API Tests
+
+API tests are separated in two suites:
+
+- a basic license test suite [default]
+- a trial license test suite (the equivalent of gold+)
+
+```
+node scripts/tests/api [--trial] [--help]
+```
+
+The API tests are located in `x-pack/test/apm_api_integration/`.
+
+**API Test tips**
+
+- For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme)
+- To update snapshots append `--updateSnapshots` to the functional_test_runner command
+
+---
+
+## E2E Tests (Cypress)
+
+```
+node scripts/tests/e2e [--trial] [--help]
+```
+
+The E2E tests are located [here](../../ftr_e2e)
+
+---
+
+## Functional tests (Security and Correlations tests)
+TODO: We could try moving this tests to the new e2e tests located at `x-pack/plugins/apm/ftr_e2e`.
+
+**Start server**
+
+```
+node scripts/functional_tests_server --config x-pack/test/functional/config.js
+```
+
+**Run tests**
+
+```
+node scripts/functional_test_runner --config x-pack/test/functional/config.js --grep='APM specs'
+```
+
+APM tests are located in `x-pack/test/functional/apps/apm`.
+For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme)

x-pack/plugins/apm/scripts/test/api.js

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+
+const yargs = require('yargs');
+const path = require('path');
+const childProcess = require('child_process');
+
+const { argv } = yargs(process.argv.slice(2))
+  .option('basic', {
+    default: true,
+    type: 'boolean',
+    description: 'Run tests with basic license',
+  })
+  .option('trial', {
+    default: false,
+    type: 'boolean',
+    description: 'Run tests with trial license',
+  })
+  .option('server', {
+    default: false,
+    type: 'boolean',
+    description: 'Start Elasticsearch and kibana',
+  })
+  .option('runner', {
+    default: false,
+    type: 'boolean',
+    description:
+      'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
+  })
+  .help();
+
+const { trial, server, runner } = argv;
+
+const license = trial ? 'trial' : 'basic';
+console.log(`License: ${license}`);
+
+let ftrScript = 'functional_tests';
+if (server) {
+  ftrScript = 'functional_tests_server';
+} else if (runner) {
+  ftrScript = 'functional_test_runner';
+}
+childProcess.execSync(
+  `node ../../../../scripts/${ftrScript} --config ../../../../test/apm_api_integration/${license}/config.ts`,
+  { cwd: path.join(__dirname), stdio: 'inherit' }
+);

x-pack/plugins/apm/scripts/test/e2e.js

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+
+const path = require('path');
+const yargs = require('yargs');
+const childProcess = require('child_process');
+
+const { argv } = yargs(process.argv.slice(2))
+  .option('server', {
+    default: false,
+    type: 'boolean',
+    description: 'Start Elasticsearch and kibana',
+  })
+  .option('runner', {
+    default: false,
+    type: 'boolean',
+    description:
+      'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
+  })
+  .option('open', {
+    default: false,
+    type: 'boolean',
+    description: 'Opens the Cypress Test Runner',
+  })
+  .help();
+
+const { server, runner, open } = argv;
+
+const e2eDir = path.join(__dirname, '../../ftr_e2e');
+
+let ftrScript = 'functional_tests';
+if (server) {
+  ftrScript = 'functional_tests_server';
+} else if (runner || open) {
+  ftrScript = 'functional_test_runner';
+}
+
+const config = open ? './cypress_open.ts' : './cypress_run.ts';
+
+childProcess.execSync(
+  `node ../../../../scripts/${ftrScript} --config ${config}`,
+  { cwd: e2eDir, stdio: 'inherit' }
+);

x-pack/plugins/apm/scripts/test/jest.js

@@ -13,7 +13,7 @@ const { run } = require('jest');
 
 process.env.NODE_ENV = process.env.NODE_ENV || 'test';
 
-const config = require('../jest.config.js');
+const config = require('../../jest.config.js');
 
 const argv = [...process.argv.slice(2), '--config', JSON.stringify(config)];