From ba92d08a58d67bc832078f52c8401d9e66598220 Mon Sep 17 00:00:00 2001 From: Sergi Romeu Date: Thu, 16 Jan 2025 12:57:50 +0100 Subject: [PATCH] [APM] Migrate APM Cypress tests to `on_merge` from `on_merge_unsupported_ftrs` (#203991) ## Summary Closes https://github.com/elastic/kibana/issues/203837 [Internal] Closes https://github.com/elastic/observability-dev/issues/4126?reload=1?reload=1 This PR moves APM Cypress tests to be run on the main pipeline instead of the unsupported one. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_oblt_stateful_configs.yml | 3 +- .buildkite/pipelines/on_merge.yml | 16 ++++ .../pipelines/on_merge_unsupported_ftrs.yml | 19 ----- .../pipelines/pull_request/apm_cypress.yml | 2 +- .../scripts/steps/functional/apm_cypress.sh | 30 ++------ .github/CODEOWNERS | 1 + .../plugins/apm/dev_docs/testing.md | 18 ++--- .../plugins/apm/ftr_e2e/README.md | 2 +- .../plugins/apm/ftr_e2e/cypress.config.ts | 20 ++--- .../plugins/apm/ftr_e2e/cypress/.gitignore | 1 + .../apm/ftr_e2e/cypress/support/e2e.ts | 1 - .../cypress/support/output_command_timings.ts | 65 ---------------- .../apm/ftr_e2e/ftr_provider_context.d.ts | 10 --- .../plugins/apm/ftr_e2e/package.json | 14 ++++ .../plugins/apm/ftr_e2e/reporter_config.json | 10 +++ .../apm/ftr_e2e/setup_cypress_node_events.ts | 14 ---- .../plugins/apm/ftr_e2e/tsconfig.json | 3 - .../plugins/apm/scripts/package.json | 6 -- .../plugins/apm/scripts/test/e2e.js | 76 ++----------------- .../create_apm_users/helpers/call_kibana.ts | 5 +- .../helpers/create_custom_role.ts | 3 + .../apm_cypress/cli_config.ts} | 26 +------ .../apm_cypress/runner.ts} | 62 +++++---------- 23 files changed, 99 insertions(+), 308 deletions(-) delete mode 100644 x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts delete mode 100644 x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_provider_context.d.ts create mode 100644 x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json create mode 100644 x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json delete mode 100644 x-pack/solutions/observability/plugins/apm/scripts/package.json rename x-pack/{solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts => test/apm_cypress/cli_config.ts} (60%) rename x-pack/{solutions/observability/plugins/apm/ftr_e2e/cypress_test_runner.ts => test/apm_cypress/runner.ts} (61%) diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml index c2dd4a025b2f..5972264018aa 100644 --- a/.buildkite/ftr_oblt_stateful_configs.yml +++ b/.buildkite/ftr_oblt_stateful_configs.yml @@ -1,10 +1,9 @@ disabled: # Cypress configs, for now these are still run manually + - x-pack/test/apm_cypress/cli_config.ts - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts - - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts - - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 70edf113aba6..c52f3ac6687a 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -479,6 +479,22 @@ steps: - exit_status: '-1' limit: 1 + - command: .buildkite/scripts/steps/functional/apm_cypress.sh + label: 'APM Cypress Tests' + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + depends_on: build + timeout_in_minutes: 120 + parallelism: 3 + retry: + automatic: + - exit_status: '-1' + limit: 1 + - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh' label: Trigger unsupported ftr tests timeout_in_minutes: 10 diff --git a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml index 1fe71d7711a7..78b3d6b44200 100644 --- a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml +++ b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml @@ -28,25 +28,6 @@ steps: - exit_status: '-1' limit: 3 - - command: .buildkite/scripts/steps/functional/apm_cypress.sh - label: 'APM Cypress Tests' - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-prod - provider: gcp - machineType: n2-standard-4 - preemptible: true - depends_on: build - env: - PING_SLACK_TEAM: "@obs-ux-infra_services-team" - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/profiling_cypress.sh label: 'Profiling Cypress Tests' agents: diff --git a/.buildkite/pipelines/pull_request/apm_cypress.yml b/.buildkite/pipelines/pull_request/apm_cypress.yml index 97935cc3489d..ec45ccf0f2ce 100644 --- a/.buildkite/pipelines/pull_request/apm_cypress.yml +++ b/.buildkite/pipelines/pull_request/apm_cypress.yml @@ -13,7 +13,7 @@ steps: - check_types - check_oas_snapshot timeout_in_minutes: 120 - parallelism: 1 # TODO: Set parallelism when apm_cypress handles it + parallelism: 3 retry: automatic: - exit_status: '-1' diff --git a/.buildkite/scripts/steps/functional/apm_cypress.sh b/.buildkite/scripts/steps/functional/apm_cypress.sh index 1b388eede871..900cdd7e0e05 100755 --- a/.buildkite/scripts/steps/functional/apm_cypress.sh +++ b/.buildkite/scripts/steps/functional/apm_cypress.sh @@ -2,35 +2,15 @@ set -euo pipefail -source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/functional/common.sh -APM_CYPRESS_RECORD_KEY="$(vault_get apm-cypress-dashboard-record-key CYPRESS_RECORD_KEY)" - -.buildkite/scripts/bootstrap.sh -.buildkite/scripts/download_build_artifacts.sh -.buildkite/scripts/copy_es_snapshot_cache.sh +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-apm-cypress -IS_FLAKY_TEST_RUNNER=${CLI_COUNT:-0} -GH_APM_TEAM_LABEL="Team:APM" - -if (! is_pr); then - echo "--- Add GH labels to buildkite metadata" - ts-node .buildkite/scripts/steps/add_gh_labels_to_bk_metadata.ts BUILDKITE_MESSAGE true - GH_ON_MERGE_LABELS="$(buildkite-agent meta-data get gh_labels --default '')" -fi - -# Enabling cypress dashboard recording when PR is labeled with `apm:cypress-record` and we are not using the flaky test runner OR on merge with Team:APM label applied -if ([[ "$IS_FLAKY_TEST_RUNNER" -ne 1 ]] && is_pr_with_label "apm:cypress-record") || ([[ $GH_ON_MERGE_LABELS == *"$GH_APM_TEAM_LABEL"* ]]); then - CYPRESS_ARGS="--record --key "$APM_CYPRESS_RECORD_KEY" --parallel --ci-build-id "${BUILDKITE_BUILD_ID}"" -else - CYPRESS_ARGS="" -fi echo "--- APM Cypress Tests" -cd "$XPACK_DIR" +cd "$XPACK_DIR/solutions/observability/plugins/apm/ftr_e2e" -node solutions/observability/plugins/apm/scripts/test/e2e.js \ - --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ - $CYPRESS_ARGS +set +e +yarn cypress:run; status=$?; yarn junit:merge || :; exit $status diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2cdd3dd1e62e..c56fd96814b2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1390,6 +1390,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql /test/api_integration/apis/ui_metric/*.ts @elastic/obs-ux-infra_services-team /x-pack/test/functional/apps/apm/ @elastic/obs-ux-infra_services-team /x-pack/test/apm_api_integration/ @elastic/obs-ux-infra_services-team +/x-pack/test/apm_cypress/ @elastic/obs-ux-infra_services-team /src/apm.js @elastic/kibana-core @vigneshshanmugam /src/platform/packages/shared/kbn-utility-types/src/dot.ts @dgieselaar /src/platform/packages/shared/kbn-utility-types/src/dot_test.ts @dgieselaar diff --git a/x-pack/solutions/observability/plugins/apm/dev_docs/testing.md b/x-pack/solutions/observability/plugins/apm/dev_docs/testing.md index dccfe16afc4c..c1c83768fd74 100644 --- a/x-pack/solutions/observability/plugins/apm/dev_docs/testing.md +++ b/x-pack/solutions/observability/plugins/apm/dev_docs/testing.md @@ -136,35 +136,27 @@ node x-pack/solutions/observability/plugins/apm/scripts/test/dat --runner --stat The E2E tests are located in [`x-pack/solutions/observability/plugins/apm/ftr_e2e`](../ftr_e2e). -When PR is labeled with `apm:cypress-record`, test runs are recorded to the [Cypress Dashboard](https://dashboard.cypress.io). - -Tests run on buildkite PR pipeline are parallelized (4 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`. +Tests run on buildkite PR pipeline are parallelized (8 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`. ```yml ... depends_on: build - parallelism: 4 + parallelism: 8 ... ``` [Test tips and best practices](../ftr_e2e/README.md) -#### Start test server +#### Start Cypress dashboard ``` -node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --server +node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --headed ``` #### Run tests ``` -node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --open -``` - -### Run tests multiple times to check for flakiness - -``` -node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --times [--spec ] +node x-pack/solutions/observability/plugins/apm/scripts/test/e2e ``` ### A11y checks diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md b/x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md index 721760bc46b9..cfa8f188d456 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md @@ -1,6 +1,6 @@ # APM E2E -APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/integration`. +APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/e2e`. ## Tips and best practices diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts index 8dad8724264e..79fd02bd3553 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts @@ -9,28 +9,28 @@ import { defineCypressConfig } from '@kbn/cypress-config'; import { setupNodeEvents } from './setup_cypress_node_events'; export default defineCypressConfig({ - projectId: 'omwh6f', + reporter: '../../../../../../node_modules/cypress-multi-reporters', + reporterOptions: { + configFile: './reporter_config.json', + }, fileServerFolder: './cypress', fixturesFolder: './cypress/fixtures', screenshotsFolder: './cypress/screenshots', videosFolder: './cypress/videos', - requestTimeout: 10000, - responseTimeout: 40000, - defaultCommandTimeout: 30000, + defaultCommandTimeout: 60000, execTimeout: 120000, pageLoadTimeout: 120000, viewportHeight: 1800, viewportWidth: 1440, - video: true, - screenshotOnRunFailure: true, - retries: { - runMode: 1, - }, + video: false, + screenshotOnRunFailure: false, e2e: { setupNodeEvents, baseUrl: 'http://localhost:5601', supportFile: './cypress/support/e2e.ts', - specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', + specPattern: './cypress/e2e/**/*.cy.ts', experimentalMemoryManagement: true, + numTestsKeptInMemory: 3, + experimentalRunAllSpecs: true, }, }); diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore index c2f807a100b1..d501d1d6d326 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore @@ -1,2 +1,3 @@ /videos/* /screenshots/* +/downloads/* diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts index 93daa0bc7ed2..5f5d1eb3b361 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts @@ -10,4 +10,3 @@ Cypress.on('uncaught:exception', (err, runnable) => { }); import './commands'; -// import './output_command_timings'; diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts b/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts deleted file mode 100644 index fd305b0c7e98..000000000000 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -const commands: Array<{ - name: string; - args: string; - started: number; - endedAt?: number; - elapsed: number; -}> = []; - -Cypress.on('test:after:run', (attributes, test) => { - if (attributes.state === 'pending') { - return; - } - - /* eslint-disable no-console */ - console.log('Test "%s" has finished in %dms', attributes.title, attributes.duration); - - let totalElapsed = 0; - - const commandsOutput = commands.map((e) => { - totalElapsed = totalElapsed + e.elapsed; - const startedDate = new Date(e.started); - return { - ...e, - started: `${startedDate.toLocaleTimeString()}:${startedDate.getMilliseconds()}`, - totalElapsed, - }; - }); - - commands.length = 0; - console.table(commandsOutput); - - if (test.state === 'failed') { - throw new Error(JSON.stringify(commandsOutput)); - } -}); - -Cypress.on('command:start', (c) => { - commands.push({ - name: c.attributes.name, - args: c.attributes.args - .slice(0, 5) - .map((arg: unknown) => JSON.stringify(arg)) - .join(','), - started: new Date().getTime(), - elapsed: 0, - }); -}); - -Cypress.on('command:end', (c) => { - const lastCommand = commands[commands.length - 1]; - - if (lastCommand.name !== c.attributes.name) { - throw new Error('Last command is wrong'); - } - - lastCommand.endedAt = new Date().getTime(); - lastCommand.elapsed = lastCommand.endedAt - lastCommand.started; -}); diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_provider_context.d.ts b/x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_provider_context.d.ts deleted file mode 100644 index 30a5f1fe518d..000000000000 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_provider_context.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { GenericFtrProviderContext } from '@kbn/test'; - -export type FtrProviderContext = GenericFtrProviderContext<{}, {}>; diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json b/x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json new file mode 100644 index 000000000000..1bad8ad8eeb8 --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json @@ -0,0 +1,14 @@ +{ + "author": "Elastic", + "name": "@kbn/apm-ftr-e2e", + "version": "1.0.0", + "private": true, + "license": "Elastic License 2.0", + "scripts": { + "cypress": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../security/plugins/security_solution/scripts/start_cypress_parallel --config-file ../observability/plugins/apm/ftr_e2e/cypress.config.ts --ftr-config-file ../../../../../test/apm_cypress/cli_config", + "cypress:open": "yarn cypress open", + "cypress:run": "yarn cypress run", + "junit:merge": "../../../../../../node_modules/.bin/mochawesome-merge ../../../../../../target/kibana-apm/cypress/results/mochawesome*.json > ../../../../../../target/kibana-apm/cypress/results/output.json && ../../../../../../node_modules/.bin/marge ../../../../../../target/kibana-apm/cypress/results/output.json --reportDir ../../../../../../target/kibana-apm/cypress/results && yarn junit:transform && mkdir -p ../../../../../../target/junit && cp ../../../../../../target/kibana-apm/cypress/results/*.xml ../../../../../../target/junit/", + "junit:transform": "node ../../../../security/plugins/security_solution/scripts/junit_transformer --pathPattern '../../../../../../target/kibana-apm/cypress/results/*.xml' --rootDirectory ../../../../../../ --reportName 'APM Cypress' --writeInPlace" + } +} diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json b/x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json new file mode 100644 index 000000000000..0852a2407e39 --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json @@ -0,0 +1,10 @@ +{ + "reporterEnabled": "mochawesome, mocha-junit-reporter", + "reporterOptions": { + "html": false, + "json": true, + "mochaFile": "../../../../../../target/kibana-apm/cypress/results/TEST-apm-cypress-[hash].xml", + "overwrite": false, + "reportDir": "../../../../../../target/kibana-apm/cypress/results" + } +} diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts b/x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts index e2fbf64f8f37..8a144fbae4c3 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts @@ -13,8 +13,6 @@ import { import { createEsClientForTesting } from '@kbn/test'; // eslint-disable-next-line @kbn/imports/no_unresolvable_imports import { initPlugin } from '@frsource/cypress-plugin-visual-regression-diff/plugins'; -import del from 'del'; -import { some } from 'lodash'; import { Readable } from 'stream'; export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) { @@ -75,18 +73,6 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin }, }); - on('after:spec', (spec, results) => { - // Delete videos that have no failures or retries - if (results && results.video) { - const failures = some(results.tests, (test) => { - return some(test.attempts, { state: 'failed' }); - }); - if (!failures) { - del(results.video); - } - } - }); - on('before:browser:launch', (browser, launchOptions) => { if (browser.name === 'electron' && browser.isHeadless) { launchOptions.preferences.width = 1440; diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json b/x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json index 87abf5bd4b4d..4d3665ec9c3d 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json +++ b/x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json @@ -11,11 +11,8 @@ "@kbn/test", "@kbn/apm-synthtrace", "@kbn/apm-synthtrace-client", - "@kbn/dev-utils", "@kbn/axe-config", "@kbn/cypress-config", "@kbn/apm-plugin", - "@kbn/ftr-common-functional-services", - "@kbn/ftr-common-functional-ui-services" ] } diff --git a/x-pack/solutions/observability/plugins/apm/scripts/package.json b/x-pack/solutions/observability/plugins/apm/scripts/package.json deleted file mode 100644 index d3303acb5c72..000000000000 --- a/x-pack/solutions/observability/plugins/apm/scripts/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "apm-scripts", - "version": "1.0.0", - "main": "index.js", - "license": "MIT" -} diff --git a/x-pack/solutions/observability/plugins/apm/scripts/test/e2e.js b/x-pack/solutions/observability/plugins/apm/scripts/test/e2e.js index 02cba1343fd4..12708ea44921 100644 --- a/x-pack/solutions/observability/plugins/apm/scripts/test/e2e.js +++ b/x-pack/solutions/observability/plugins/apm/scripts/test/e2e.js @@ -6,97 +6,35 @@ */ /* eslint-disable no-console */ -const { times } = require('lodash'); const path = require('path'); const yargs = require('yargs'); const childProcess = require('child_process'); -const { REPO_ROOT } = require('@kbn/repo-info'); const { argv } = yargs(process.argv.slice(2)) .parserConfiguration({ 'unknown-options-as-args': true }) - .option('kibana-install-dir', { - default: '', - type: 'string', - description: 'Path to the Kibana install directory', - }) - .option('server', { + .option('headed', { default: false, type: 'boolean', - description: 'Start Elasticsearch and Kibana', - }) - .option('runner', { - default: false, - type: 'boolean', - description: - 'Run all tests (an instance of Elasticsearch and kibana are needs to be available)', - }) - .option('times', { - type: 'number', - description: 'Repeat the test n number of times', - }) - .option('bail', { - default: false, - type: 'boolean', - description: 'stop tests after the first failure', + description: 'Runs Cypress in headed mode', }) .help(); const e2eDir = path.join(__dirname, '../../ftr_e2e'); -let ftrScript = 'functional_tests.js'; -if (argv.server) { - ftrScript = 'functional_tests_server.js'; -} else if (argv.runner) { - ftrScript = 'functional_test_runner.js'; -} - -const cypressCliArgs = yargs(argv._).parserConfiguration({ - 'boolean-negation': false, -}).argv; - -if (cypressCliArgs.grep) { - throw new Error('--grep is not supported. Please use --spec instead'); -} - -const spawnArgs = [ - `${REPO_ROOT}/scripts/${ftrScript}`, - `--config=./ftr_config.ts`, - `--kibana-install-dir=${argv.kibanaInstallDir}`, - ...(argv.bail ? [`--bail`] : []), -]; - function runTests() { - console.log(`Running e2e tests: "node ${spawnArgs.join(' ')}"`); + const mode = argv.headed ? 'open' : 'run'; + console.log(`Running e2e tests: "yarn cypress:${mode}"`); - return childProcess.spawnSync('node', spawnArgs, { + return childProcess.spawnSync('yarn', [`cypress:${mode}`], { cwd: e2eDir, - env: { - ...process.env, - CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs), - NODE_OPTIONS: '--openssl-legacy-provider', - }, encoding: 'utf8', stdio: 'inherit', }); } -const runCounter = { succeeded: 0, failed: 0, remaining: argv.times }; let exitStatus = 0; -times(argv.times ?? 1, () => { - const child = runTests(); - if (child.status === 0) { - runCounter.succeeded++; - } else { - exitStatus = child.status; - runCounter.failed++; - } - - runCounter.remaining--; - - if (argv.times > 1) { - console.log(runCounter); - } -}); +const child = runTests(); +exitStatus = child.status; process.exitCode = exitStatus; console.log(`Quitting with exit code ${exitStatus}`); diff --git a/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/call_kibana.ts b/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/call_kibana.ts index 9a24c55f4456..637177ca89de 100644 --- a/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/call_kibana.ts +++ b/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/call_kibana.ts @@ -6,7 +6,6 @@ */ import type { AxiosRequestConfig, AxiosError } from 'axios'; import axios from 'axios'; -import { once } from 'lodash'; import type { Elasticsearch, Kibana } from '../create_apm_users'; const DEFAULT_HEADERS = { @@ -34,7 +33,7 @@ export async function callKibana({ return data; } -const getBaseUrl = once(async (kibanaHostname: string) => { +const getBaseUrl = async (kibanaHostname: string) => { try { await axios.request({ url: kibanaHostname, @@ -52,7 +51,7 @@ const getBaseUrl = once(async (kibanaHostname: string) => { throw e; } return kibanaHostname; -}); +}; export function isAxiosError(e: AxiosError | Error): e is AxiosError { return 'isAxiosError' in e; diff --git a/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/create_custom_role.ts b/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/create_custom_role.ts index 0f1675e253eb..6f5738fce202 100644 --- a/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/create_custom_role.ts +++ b/x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/create_custom_role.ts @@ -39,6 +39,9 @@ export async function createCustomRole({ data: { ...omit(role, 'applications'), }, + headers: { + 'elastic-api-version': '2023-10-31', + }, }, }); } diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts b/x-pack/test/apm_cypress/cli_config.ts similarity index 60% rename from x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts rename to x-pack/test/apm_cypress/cli_config.ts index 5be4a8630f37..0c6b032c5224 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts +++ b/x-pack/test/apm_cypress/cli_config.ts @@ -7,27 +7,19 @@ import type { FtrConfigProviderContext } from '@kbn/test'; import { CA_CERT_PATH } from '@kbn/dev-utils'; -import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; -import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services'; -import { cypressTestRunner } from './cypress_test_runner'; -import type { FtrProviderContext } from './ftr_provider_context'; +import { cypressTestRunner } from './runner'; async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) { const kibanaCommonTestsConfig = await readConfigFile( require.resolve('@kbn/test-suites-src/common/config') ); const xpackFunctionalTestsConfig = await readConfigFile( - require.resolve('@kbn/test-suites-xpack/functional/config.base') + require.resolve('../functional/config.base.js') ); return { ...kibanaCommonTestsConfig.getAll(), - services: { - ...commonFunctionalServices, - ...commonFunctionalUIServices, - }, - esTestCluster: { ...xpackFunctionalTestsConfig.get('esTestCluster'), serverArgs: [ @@ -49,20 +41,8 @@ async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) { `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, ], }, - testRunner: async (ftrProviderContext: FtrProviderContext) => { - const result = await cypressTestRunner(ftrProviderContext); - - // set exit code explicitly if at least one Cypress test fails - if ( - result && - ((result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' || - (result as CypressCommandLine.CypressRunResult)?.totalFailed) - ) { - process.exitCode = 1; - } - }, + testRunner: cypressTestRunner, }; } -// eslint-disable-next-line import/no-default-export export default ftrConfig; diff --git a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress_test_runner.ts b/x-pack/test/apm_cypress/runner.ts similarity index 61% rename from x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress_test_runner.ts rename to x-pack/test/apm_cypress/runner.ts index 2d6804917f0a..431d7e07bf95 100644 --- a/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress_test_runner.ts +++ b/x-pack/test/apm_cypress/runner.ts @@ -6,11 +6,9 @@ */ import { ApmSynthtraceKibanaClient, createLogger, LogLevel } from '@kbn/apm-synthtrace'; -import cypress from 'cypress'; -import path from 'path'; import Url from 'url'; import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users'; -import type { FtrProviderContext } from './ftr_provider_context'; +import type { FtrProviderContext } from '../common/ftr_provider_context'; export async function cypressTestRunner({ getService }: FtrProviderContext) { const config = getService('config'); @@ -54,44 +52,22 @@ export async function cypressTestRunner({ getService }: FtrProviderContext) { port: config.get('servers.kibana.port'), }); - const cypressProjectPath = path.join(__dirname); - const { open, ...cypressCliArgs } = getCypressCliArgs(); - const cypressExecution = open ? cypress.open : cypress.run; - const res = await cypressExecution({ - ...cypressCliArgs, - project: cypressProjectPath, - browser: 'electron', - config: { - e2e: { - baseUrl: kibanaUrlWithoutAuth, - }, - }, - env: { - KIBANA_URL: kibanaUrlWithoutAuth, - APM_PACKAGE_VERSION: packageVersion, - ES_NODE: esNode, - ES_REQUEST_TIMEOUT: esRequestTimeout, - TEST_CLOUD: process.env.TEST_CLOUD, - }, - }); - - return res; -} - -function getCypressCliArgs(): Record { - if (!process.env.CYPRESS_CLI_ARGS) { - return {}; - } - - const { $0, _, ...cypressCliArgs } = JSON.parse(process.env.CYPRESS_CLI_ARGS) as Record< - string, - unknown - >; - - const spec = - typeof cypressCliArgs.spec === 'string' && !cypressCliArgs.spec.includes('**') - ? `**/${cypressCliArgs.spec}*` - : cypressCliArgs.spec; - - return { ...cypressCliArgs, spec }; + return { + KIBANA_URL: kibanaUrlWithoutAuth, + APM_PACKAGE_VERSION: packageVersion, + ES_NODE: esNode, + ES_REQUEST_TIMEOUT: esRequestTimeout, + TEST_CLOUD: process.env.TEST_CLOUD, + baseUrl: Url.format({ + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + port: config.get('servers.kibana.port'), + }), + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + configport: config.get('servers.kibana.port'), + ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')), + ELASTICSEARCH_USERNAME: config.get('servers.kibana.username'), + ELASTICSEARCH_PASSWORD: config.get('servers.kibana.password'), + }; }