[APM] Migrate APM Cypress tests to on_merge from on_merge_unsupported_ftrs (#203991)

## Summary

Closes https://github.com/elastic/kibana/issues/203837
[Internal] Closes
https://github.com/elastic/observability-dev/issues/4126?reload=1?reload=1

This PR moves APM Cypress tests to be run on the main pipeline instead
of the unsupported one.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/ftr_oblt_stateful_configs.yml

@@ -1,10 +1,9 @@
 disabled:
   # Cypress configs, for now these are still run manually
+  - x-pack/test/apm_cypress/cli_config.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
   - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts

.buildkite/pipelines/on_merge.yml

@@ -479,6 +479,22 @@ steps:
         - exit_status: '-1'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 3
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
   - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
     label: Trigger unsupported ftr tests
     timeout_in_minutes: 10

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -28,25 +28,6 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
-    label: 'APM Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on: build
-    env:
-      PING_SLACK_TEAM: "@obs-ux-infra_services-team"
-    timeout_in_minutes: 120
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
     label: 'Profiling Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -13,7 +13,7 @@ steps:
       - check_types
       - check_oas_snapshot
     timeout_in_minutes: 120
-    parallelism: 1 # TODO: Set parallelism when apm_cypress handles it
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/scripts/steps/functional/apm_cypress.sh

@@ -2,35 +2,15 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common.sh
 
-APM_CYPRESS_RECORD_KEY="$(vault_get apm-cypress-dashboard-record-key CYPRESS_RECORD_KEY)"
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
-.buildkite/scripts/copy_es_snapshot_cache.sh
 
 export JOB=kibana-apm-cypress
-IS_FLAKY_TEST_RUNNER=${CLI_COUNT:-0}
-GH_APM_TEAM_LABEL="Team:APM"
-
-if (! is_pr); then
-  echo "--- Add GH labels to buildkite metadata"
-  ts-node .buildkite/scripts/steps/add_gh_labels_to_bk_metadata.ts BUILDKITE_MESSAGE true
-  GH_ON_MERGE_LABELS="$(buildkite-agent meta-data get gh_labels --default '')"
-fi
-
-# Enabling cypress dashboard recording when PR is labeled with `apm:cypress-record` and we are not using the flaky test runner OR on merge with Team:APM label applied
-if ([[ "$IS_FLAKY_TEST_RUNNER" -ne 1 ]] && is_pr_with_label "apm:cypress-record") || ([[ $GH_ON_MERGE_LABELS == *"$GH_APM_TEAM_LABEL"* ]]); then
-  CYPRESS_ARGS="--record --key "$APM_CYPRESS_RECORD_KEY" --parallel --ci-build-id "${BUILDKITE_BUILD_ID}""
-else
-  CYPRESS_ARGS=""
-fi
 
 echo "--- APM Cypress Tests"
 
-cd "$XPACK_DIR"
+cd "$XPACK_DIR/solutions/observability/plugins/apm/ftr_e2e"
 
-node solutions/observability/plugins/apm/scripts/test/e2e.js \
+set +e
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+yarn cypress:run; status=$?; yarn junit:merge || :; exit $status
-  $CYPRESS_ARGS

.github/CODEOWNERS

@@ -1390,6 +1390,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /test/api_integration/apis/ui_metric/*.ts @elastic/obs-ux-infra_services-team
 /x-pack/test/functional/apps/apm/ @elastic/obs-ux-infra_services-team
 /x-pack/test/apm_api_integration/ @elastic/obs-ux-infra_services-team
+/x-pack/test/apm_cypress/ @elastic/obs-ux-infra_services-team
 /src/apm.js @elastic/kibana-core @vigneshshanmugam
 /src/platform/packages/shared/kbn-utility-types/src/dot.ts @dgieselaar
 /src/platform/packages/shared/kbn-utility-types/src/dot_test.ts @dgieselaar

x-pack/solutions/observability/plugins/apm/dev_docs/testing.md

@@ -136,35 +136,27 @@ node x-pack/solutions/observability/plugins/apm/scripts/test/dat --runner --stat
 
 The E2E tests are located in [`x-pack/solutions/observability/plugins/apm/ftr_e2e`](../ftr_e2e).
 
-When PR is labeled with `apm:cypress-record`, test runs are recorded to the [Cypress Dashboard](https://dashboard.cypress.io).
+Tests run on buildkite PR pipeline are parallelized (8 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`.
-
-Tests run on buildkite PR pipeline are parallelized (4 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`.
 
 ```yml
     ...
     depends_on: build
-    parallelism: 4
+    parallelism: 8
     ...
 ```
 
 [Test tips and best practices](../ftr_e2e/README.md)
 
-#### Start test server
+#### Start Cypress dashboard
 
 ```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --server
+node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --headed
 ```
 
 #### Run tests
 
 ```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --open
+node x-pack/solutions/observability/plugins/apm/scripts/test/e2e
-```
-
-### Run tests multiple times to check for flakiness
-
-```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --times <NUMBER> [--spec <FILE_NAME>]
 ```
 
 ### A11y checks

x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md

@@ -1,6 +1,6 @@
 # APM E2E
 
-APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/integration`.
+APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/e2e`.
 
 ## Tips and best practices
 

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts

@@ -9,28 +9,28 @@ import { defineCypressConfig } from '@kbn/cypress-config';
 import { setupNodeEvents } from './setup_cypress_node_events';
 
 export default defineCypressConfig({
-  projectId: 'omwh6f',
+  reporter: '../../../../../../node_modules/cypress-multi-reporters',
+  reporterOptions: {
+    configFile: './reporter_config.json',
+  },
   fileServerFolder: './cypress',
   fixturesFolder: './cypress/fixtures',
   screenshotsFolder: './cypress/screenshots',
   videosFolder: './cypress/videos',
-  requestTimeout: 10000,
+  defaultCommandTimeout: 60000,
-  responseTimeout: 40000,
-  defaultCommandTimeout: 30000,
   execTimeout: 120000,
   pageLoadTimeout: 120000,
   viewportHeight: 1800,
   viewportWidth: 1440,
-  video: true,
+  video: false,
-  screenshotOnRunFailure: true,
+  screenshotOnRunFailure: false,
-  retries: {
-    runMode: 1,
-  },
   e2e: {
     setupNodeEvents,
     baseUrl: 'http://localhost:5601',
     supportFile: './cypress/support/e2e.ts',
-    specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}',
+    specPattern: './cypress/e2e/**/*.cy.ts',
     experimentalMemoryManagement: true,
+    numTestsKeptInMemory: 3,
+    experimentalRunAllSpecs: true,
   },
 });

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore

@@ -1,2 +1,3 @@
 /videos/*
 /screenshots/*
+/downloads/*

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts

@@ -10,4 +10,3 @@ Cypress.on('uncaught:exception', (err, runnable) => {
 });
 
 import './commands';
-// import './output_command_timings';

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts

@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-const commands: Array<{
-  name: string;
-  args: string;
-  started: number;
-  endedAt?: number;
-  elapsed: number;
-}> = [];
-
-Cypress.on('test:after:run', (attributes, test) => {
-  if (attributes.state === 'pending') {
-    return;
-  }
-
-  /* eslint-disable no-console */
-  console.log('Test "%s" has finished in %dms', attributes.title, attributes.duration);
-
-  let totalElapsed = 0;
-
-  const commandsOutput = commands.map((e) => {
-    totalElapsed = totalElapsed + e.elapsed;
-    const startedDate = new Date(e.started);
-    return {
-      ...e,
-      started: `${startedDate.toLocaleTimeString()}:${startedDate.getMilliseconds()}`,
-      totalElapsed,
-    };
-  });
-
-  commands.length = 0;
-  console.table(commandsOutput);
-
-  if (test.state === 'failed') {
-    throw new Error(JSON.stringify(commandsOutput));
-  }
-});
-
-Cypress.on('command:start', (c) => {
-  commands.push({
-    name: c.attributes.name,
-    args: c.attributes.args
-      .slice(0, 5)
-      .map((arg: unknown) => JSON.stringify(arg))
-      .join(','),
-    started: new Date().getTime(),
-    elapsed: 0,
-  });
-});
-
-Cypress.on('command:end', (c) => {
-  const lastCommand = commands[commands.length - 1];
-
-  if (lastCommand.name !== c.attributes.name) {
-    throw new Error('Last command is wrong');
-  }
-
-  lastCommand.endedAt = new Date().getTime();
-  lastCommand.elapsed = lastCommand.endedAt - lastCommand.started;
-});

x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_provider_context.d.ts

@@ -1,10 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { GenericFtrProviderContext } from '@kbn/test';
-
-export type FtrProviderContext = GenericFtrProviderContext<{}, {}>;

x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json

@@ -0,0 +1,14 @@
+{
+  "author": "Elastic",
+  "name": "@kbn/apm-ftr-e2e",
+  "version": "1.0.0",
+  "private": true,
+  "license": "Elastic License 2.0",
+  "scripts": {
+    "cypress": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../security/plugins/security_solution/scripts/start_cypress_parallel --config-file ../observability/plugins/apm/ftr_e2e/cypress.config.ts --ftr-config-file ../../../../../test/apm_cypress/cli_config",
+    "cypress:open": "yarn cypress open",
+    "cypress:run": "yarn cypress run",
+    "junit:merge": "../../../../../../node_modules/.bin/mochawesome-merge ../../../../../../target/kibana-apm/cypress/results/mochawesome*.json > ../../../../../../target/kibana-apm/cypress/results/output.json && ../../../../../../node_modules/.bin/marge ../../../../../../target/kibana-apm/cypress/results/output.json --reportDir ../../../../../../target/kibana-apm/cypress/results && yarn junit:transform && mkdir -p ../../../../../../target/junit && cp ../../../../../../target/kibana-apm/cypress/results/*.xml ../../../../../../target/junit/",
+    "junit:transform": "node ../../../../security/plugins/security_solution/scripts/junit_transformer --pathPattern '../../../../../../target/kibana-apm/cypress/results/*.xml' --rootDirectory ../../../../../../ --reportName 'APM Cypress' --writeInPlace"
+  }
+}

x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json

@@ -0,0 +1,10 @@
+{
+  "reporterEnabled": "mochawesome, mocha-junit-reporter",
+  "reporterOptions": {
+    "html": false,
+    "json": true,
+    "mochaFile": "../../../../../../target/kibana-apm/cypress/results/TEST-apm-cypress-[hash].xml",
+    "overwrite": false,
+    "reportDir": "../../../../../../target/kibana-apm/cypress/results"
+  }
+}

x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts

@@ -13,8 +13,6 @@ import {
 import { createEsClientForTesting } from '@kbn/test';
 // eslint-disable-next-line @kbn/imports/no_unresolvable_imports
 import { initPlugin } from '@frsource/cypress-plugin-visual-regression-diff/plugins';
-import del from 'del';
-import { some } from 'lodash';
 import { Readable } from 'stream';
 
 export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) {
@@ -75,18 +73,6 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
     },
   });
 
-  on('after:spec', (spec, results) => {
-    // Delete videos that have no failures or retries
-    if (results && results.video) {
-      const failures = some(results.tests, (test) => {
-        return some(test.attempts, { state: 'failed' });
-      });
-      if (!failures) {
-        del(results.video);
-      }
-    }
-  });
-
   on('before:browser:launch', (browser, launchOptions) => {
     if (browser.name === 'electron' && browser.isHeadless) {
       launchOptions.preferences.width = 1440;

x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json

@@ -11,11 +11,8 @@
     "@kbn/test",
     "@kbn/apm-synthtrace",
     "@kbn/apm-synthtrace-client",
-    "@kbn/dev-utils",
     "@kbn/axe-config",
     "@kbn/cypress-config",
     "@kbn/apm-plugin",
-    "@kbn/ftr-common-functional-services",
-    "@kbn/ftr-common-functional-ui-services"
   ]
 }

x-pack/solutions/observability/plugins/apm/scripts/package.json

@@ -1,6 +0,0 @@
-{
-  "name": "apm-scripts",
-  "version": "1.0.0",
-  "main": "index.js",
-  "license": "MIT"
-}

x-pack/solutions/observability/plugins/apm/scripts/test/e2e.js

@@ -6,97 +6,35 @@
  */
 
 /* eslint-disable no-console */
-const { times } = require('lodash');
 const path = require('path');
 const yargs = require('yargs');
 const childProcess = require('child_process');
-const { REPO_ROOT } = require('@kbn/repo-info');
 
 const { argv } = yargs(process.argv.slice(2))
   .parserConfiguration({ 'unknown-options-as-args': true })
-  .option('kibana-install-dir', {
+  .option('headed', {
-    default: '',
-    type: 'string',
-    description: 'Path to the Kibana install directory',
-  })
-  .option('server', {
     default: false,
     type: 'boolean',
-    description: 'Start Elasticsearch and Kibana',
+    description: 'Runs Cypress in headed mode',
-  })
-  .option('runner', {
-    default: false,
-    type: 'boolean',
-    description:
-      'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
-  })
-  .option('times', {
-    type: 'number',
-    description: 'Repeat the test n number of times',
-  })
-  .option('bail', {
-    default: false,
-    type: 'boolean',
-    description: 'stop tests after the first failure',
   })
   .help();
 
 const e2eDir = path.join(__dirname, '../../ftr_e2e');
 
-let ftrScript = 'functional_tests.js';
-if (argv.server) {
-  ftrScript = 'functional_tests_server.js';
-} else if (argv.runner) {
-  ftrScript = 'functional_test_runner.js';
-}
-
-const cypressCliArgs = yargs(argv._).parserConfiguration({
-  'boolean-negation': false,
-}).argv;
-
-if (cypressCliArgs.grep) {
-  throw new Error('--grep is not supported. Please use --spec instead');
-}
-
-const spawnArgs = [
-  `${REPO_ROOT}/scripts/${ftrScript}`,
-  `--config=./ftr_config.ts`,
-  `--kibana-install-dir=${argv.kibanaInstallDir}`,
-  ...(argv.bail ? [`--bail`] : []),
-];
-
 function runTests() {
-  console.log(`Running e2e tests: "node ${spawnArgs.join(' ')}"`);
+  const mode = argv.headed ? 'open' : 'run';
+  console.log(`Running e2e tests: "yarn cypress:${mode}"`);
 
-  return childProcess.spawnSync('node', spawnArgs, {
+  return childProcess.spawnSync('yarn', [`cypress:${mode}`], {
     cwd: e2eDir,
-    env: {
-      ...process.env,
-      CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs),
-      NODE_OPTIONS: '--openssl-legacy-provider',
-    },
     encoding: 'utf8',
     stdio: 'inherit',
   });
 }
 
-const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
 let exitStatus = 0;
-times(argv.times ?? 1, () => {
+const child = runTests();
-  const child = runTests();
+exitStatus = child.status;
-  if (child.status === 0) {
-    runCounter.succeeded++;
-  } else {
-    exitStatus = child.status;
-    runCounter.failed++;
-  }
-
-  runCounter.remaining--;
-
-  if (argv.times > 1) {
-    console.log(runCounter);
-  }
-});
 
 process.exitCode = exitStatus;
 console.log(`Quitting with exit code ${exitStatus}`);

x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/call_kibana.ts

@@ -6,7 +6,6 @@
  */
 import type { AxiosRequestConfig, AxiosError } from 'axios';
 import axios from 'axios';
-import { once } from 'lodash';
 import type { Elasticsearch, Kibana } from '../create_apm_users';
 
 const DEFAULT_HEADERS = {
@@ -34,7 +33,7 @@ export async function callKibana<T>({
   return data;
 }
 
-const getBaseUrl = once(async (kibanaHostname: string) => {
+const getBaseUrl = async (kibanaHostname: string) => {
   try {
     await axios.request({
       url: kibanaHostname,
@@ -52,7 +51,7 @@ const getBaseUrl = once(async (kibanaHostname: string) => {
     throw e;
   }
   return kibanaHostname;
-});
+};
 
 export function isAxiosError(e: AxiosError | Error): e is AxiosError {
   return 'isAxiosError' in e;

x-pack/solutions/observability/plugins/apm/server/test_helpers/create_apm_users/helpers/create_custom_role.ts

@@ -39,6 +39,9 @@ export async function createCustomRole({
       data: {
         ...omit(role, 'applications'),
       },
+      headers: {
+        'elastic-api-version': '2023-10-31',
+      },
     },
   });
 }

x-pack/test/apm_cypress/cli_config.ts

@@ -7,27 +7,19 @@
 
 import type { FtrConfigProviderContext } from '@kbn/test';
 import { CA_CERT_PATH } from '@kbn/dev-utils';
-import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
+import { cypressTestRunner } from './runner';
-import { commonFunctionalUIServices } from '@kbn/ftr-common-functional-ui-services';
-import { cypressTestRunner } from './cypress_test_runner';
-import type { FtrProviderContext } from './ftr_provider_context';
 
 async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) {
   const kibanaCommonTestsConfig = await readConfigFile(
     require.resolve('@kbn/test-suites-src/common/config')
   );
   const xpackFunctionalTestsConfig = await readConfigFile(
-    require.resolve('@kbn/test-suites-xpack/functional/config.base')
+    require.resolve('../functional/config.base.js')
   );
 
   return {
     ...kibanaCommonTestsConfig.getAll(),
 
-    services: {
-      ...commonFunctionalServices,
-      ...commonFunctionalUIServices,
-    },
-
     esTestCluster: {
       ...xpackFunctionalTestsConfig.get('esTestCluster'),
       serverArgs: [
@@ -49,20 +41,8 @@ async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) {
         `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
       ],
     },
-    testRunner: async (ftrProviderContext: FtrProviderContext) => {
+    testRunner: cypressTestRunner,
-      const result = await cypressTestRunner(ftrProviderContext);
-
-      // set exit code explicitly if at least one Cypress test fails
-      if (
-        result &&
-        ((result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
-          (result as CypressCommandLine.CypressRunResult)?.totalFailed)
-      ) {
-        process.exitCode = 1;
-      }
-    },
   };
 }
 
-// eslint-disable-next-line import/no-default-export
 export default ftrConfig;

x-pack/test/apm_cypress/runner.ts

@@ -6,11 +6,9 @@
  */
 
 import { ApmSynthtraceKibanaClient, createLogger, LogLevel } from '@kbn/apm-synthtrace';
-import cypress from 'cypress';
-import path from 'path';
 import Url from 'url';
 import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
-import type { FtrProviderContext } from './ftr_provider_context';
+import type { FtrProviderContext } from '../common/ftr_provider_context';
 
 export async function cypressTestRunner({ getService }: FtrProviderContext) {
   const config = getService('config');
@@ -54,44 +52,22 @@ export async function cypressTestRunner({ getService }: FtrProviderContext) {
     port: config.get('servers.kibana.port'),
   });
 
-  const cypressProjectPath = path.join(__dirname);
+  return {
-  const { open, ...cypressCliArgs } = getCypressCliArgs();
-  const cypressExecution = open ? cypress.open : cypress.run;
-  const res = await cypressExecution({
-    ...cypressCliArgs,
-    project: cypressProjectPath,
-    browser: 'electron',
-    config: {
-      e2e: {
-        baseUrl: kibanaUrlWithoutAuth,
-      },
-    },
-    env: {
     KIBANA_URL: kibanaUrlWithoutAuth,
     APM_PACKAGE_VERSION: packageVersion,
     ES_NODE: esNode,
     ES_REQUEST_TIMEOUT: esRequestTimeout,
     TEST_CLOUD: process.env.TEST_CLOUD,
-    },
+    baseUrl: Url.format({
-  });
+      protocol: config.get('servers.kibana.protocol'),
-
+      hostname: config.get('servers.kibana.hostname'),
-  return res;
+      port: config.get('servers.kibana.port'),
-}
+    }),
-
+    protocol: config.get('servers.kibana.protocol'),
-function getCypressCliArgs(): Record<string, unknown> {
+    hostname: config.get('servers.kibana.hostname'),
-  if (!process.env.CYPRESS_CLI_ARGS) {
+    configport: config.get('servers.kibana.port'),
-    return {};
+    ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-  }
+    ELASTICSEARCH_USERNAME: config.get('servers.kibana.username'),
-
+    ELASTICSEARCH_PASSWORD: config.get('servers.kibana.password'),
-  const { $0, _, ...cypressCliArgs } = JSON.parse(process.env.CYPRESS_CLI_ARGS) as Record<
+  };
-    string,
-    unknown
-  >;
-
-  const spec =
-    typeof cypressCliArgs.spec === 'string' && !cypressCliArgs.spec.includes('**')
-      ? `**/${cypressCliArgs.spec}*`
-      : cypressCliArgs.spec;
-
-  return { ...cypressCliArgs, spec };
 }