[APM] Cypress: Enable data streams and refactor runner (#139322)

x-pack/plugins/apm/ftr_e2e/cypress/.gitignore

@@ -0,0 +1,2 @@
+/videos/*
+/screenshots/*

x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/settings/custom_links.spec.ts

@@ -37,6 +37,10 @@ describe('Custom links', () => {
 
   it('clears filter values when field is selected', () => {
     cy.visitKibana(basePath);
+
+    // wait for empty prompt
+    cy.get('[data-test-subj="customLinksEmptyPrompt"]').should('be.visible');
+
     cy.contains('Create custom link').click();
     cy.get('[data-test-subj="filter-0"]').select('service.name');
     cy.get(

x-pack/plugins/apm/ftr_e2e/cypress/plugins/index.ts

@@ -28,6 +28,7 @@ import { createEsClientForTesting } from '@kbn/test';
 /**
  * @type {Cypress.PluginConfig}
  */
+
 const plugin: Cypress.PluginConfig = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
@@ -42,12 +43,19 @@ const plugin: Cypress.PluginConfig = (on, config) => {
     client,
     createLogger(LogLevel.info),
     {
-      forceLegacyIndices: true,
+      forceLegacyIndices: false,
       refreshAfterIndex: true,
     }
   );
 
   on('task', {
+    // send logs to node process
+    log(message) {
+      // eslint-disable-next-line no-console
+      console.log(message);
+      return null;
+    },
+
     'synthtrace:index': async (events: Array<Record<string, any>>) => {
       await synthtraceEsClient.index(new EntityArrayIterable(events));
       return null;

x-pack/plugins/apm/ftr_e2e/cypress/support/index.ts

@@ -5,8 +5,5 @@
  * 2.0.
  */
 
-Cypress.on('uncaught:exception', (err, runnable) => {
-  return false;
-});
-
 import './commands';
+// import './output_command_timings';

x-pack/plugins/apm/ftr_e2e/cypress/support/output_command_timings.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const commands: Array<{
+  name: string;
+  args: string;
+  started: number;
+  endedAt?: number;
+  elapsed: number;
+}> = [];
+
+Cypress.on('test:after:run', (attributes, test) => {
+  if (attributes.state === 'pending') {
+    return;
+  }
+
+  /* eslint-disable no-console */
+  console.log(
+    'Test "%s" has finished in %dms',
+    attributes.title,
+    attributes.duration
+  );
+
+  let totalElapsed = 0;
+
+  const commandsOutput = commands.map((e) => {
+    totalElapsed = totalElapsed + e.elapsed;
+    const startedDate = new Date(e.started);
+    return {
+      ...e,
+      started: `${startedDate.toLocaleTimeString()}:${startedDate.getMilliseconds()}`,
+      totalElapsed,
+    };
+  });
+
+  commands.length = 0;
+  console.table(commandsOutput);
+
+  if (test.state === 'failed') {
+    throw new Error(JSON.stringify(commandsOutput));
+  }
+});
+
+Cypress.on('command:start', (c) => {
+  commands.push({
+    name: c.attributes.name,
+    args: c.attributes.args
+      .slice(0, 5)
+      .map((arg: unknown) => JSON.stringify(arg))
+      .join(','),
+    started: new Date().getTime(),
+    elapsed: 0,
+  });
+});
+
+Cypress.on('command:end', (c) => {
+  const lastCommand = commands[commands.length - 1];
+
+  if (lastCommand.name !== c.attributes.name) {
+    throw new Error('Last command is wrong');
+  }
+
+  lastCommand.endedAt = new Date().getTime();
+  lastCommand.elapsed = lastCommand.endedAt - lastCommand.started;
+});

x-pack/plugins/apm/ftr_e2e/cypress_start.ts

@@ -1,76 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-/* eslint-disable no-console */
-
-import { argv } from 'yargs';
-import Url from 'url';
-import cypress from 'cypress';
-import { FtrProviderContext } from './ftr_provider_context';
-import { createApmUsers } from '../scripts/create_apm_users/create_apm_users';
-import { esArchiverLoad, esArchiverUnload } from './cypress/tasks/es_archiver';
-
-export async function cypressStart(
-  getService: FtrProviderContext['getService'],
-  cypressExecution: typeof cypress.run | typeof cypress.open
-) {
-  const config = getService('config');
-
-  const kibanaUrl = Url.format({
-    protocol: config.get('servers.kibana.protocol'),
-    hostname: config.get('servers.kibana.hostname'),
-    port: config.get('servers.kibana.port'),
-  });
-
-  // Creates APM users
-  await createApmUsers({
-    elasticsearch: {
-      username: config.get('servers.elasticsearch.username'),
-      password: config.get('servers.elasticsearch.password'),
-    },
-    kibana: {
-      hostname: kibanaUrl,
-    },
-  });
-
-  const esNode = Url.format({
-    protocol: config.get('servers.elasticsearch.protocol'),
-    port: config.get('servers.elasticsearch.port'),
-    hostname: config.get('servers.elasticsearch.hostname'),
-    auth: `${config.get('servers.elasticsearch.username')}:${config.get(
-      'servers.elasticsearch.password'
-    )}`,
-  });
-
-  const esRequestTimeout = config.get('timeouts.esRequestTimeout');
-  const archiveName = 'apm_mappings_only_8.0.0';
-  const metricsArchiveName = 'metrics_8.0.0';
-
-  console.log(`Creating APM mappings`);
-  await esArchiverLoad(archiveName);
-  console.log(`Creating Metrics mappings`);
-  await esArchiverLoad(metricsArchiveName);
-
-  const spec = argv.grep as string | undefined;
-  const res = await cypressExecution({
-    ...(spec ? { spec } : {}),
-    config: { baseUrl: kibanaUrl },
-    env: {
-      KIBANA_URL: kibanaUrl,
-      ES_NODE: esNode,
-      ES_REQUEST_TIMEOUT: esRequestTimeout,
-      TEST_CLOUD: process.env.TEST_CLOUD,
-    },
-  });
-
-  console.log('Removing APM mappings');
-  await esArchiverUnload(archiveName);
-  console.log('Removing Metrics mappings');
-  await esArchiverUnload(metricsArchiveName);
-
-  return res;
-}

x-pack/plugins/apm/ftr_e2e/cypress_test_runner.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import Url from 'url';
+import cypress from 'cypress';
+import { esTestConfig } from '@kbn/test';
+import { apm, createLogger, LogLevel } from '@kbn/apm-synthtrace';
+import path from 'path';
+import { FtrProviderContext } from './ftr_provider_context';
+import { createApmUsers } from '../scripts/create_apm_users/create_apm_users';
+
+export async function cypressTestRunner({ getService }: FtrProviderContext) {
+  const config = getService('config');
+  const kibanaVersion = esTestConfig.getVersion();
+
+  const kibanaUrl = Url.format({
+    protocol: config.get('servers.kibana.protocol'),
+    hostname: config.get('servers.kibana.hostname'),
+    port: config.get('servers.kibana.port'),
+  });
+
+  const username = config.get('servers.elasticsearch.username');
+  const password = config.get('servers.elasticsearch.password');
+
+  // Creates APM users
+  await createApmUsers({
+    elasticsearch: { username, password },
+    kibana: { hostname: kibanaUrl },
+  });
+
+  const esNode = Url.format({
+    protocol: config.get('servers.elasticsearch.protocol'),
+    port: config.get('servers.elasticsearch.port'),
+    hostname: config.get('servers.elasticsearch.hostname'),
+    auth: `${username}:${password}`,
+  });
+
+  const esRequestTimeout = config.get('timeouts.esRequestTimeout');
+  const kibanaClient = new apm.ApmSynthtraceKibanaClient(
+    createLogger(LogLevel.info)
+  );
+
+  await kibanaClient.installApmPackage(
+    kibanaUrl,
+    kibanaVersion,
+    username,
+    password
+  );
+
+  const cypressProjectPath = path.join(__dirname);
+  const { open, ...cypressCliArgs } = getCypressCliArgs();
+  const cypressExecution = open ? cypress.open : cypress.run;
+  const res = await cypressExecution({
+    ...cypressCliArgs,
+    project: cypressProjectPath,
+    config: {
+      baseUrl: kibanaUrl,
+      requestTimeout: 10000,
+      responseTimeout: 60000,
+      defaultCommandTimeout: 15000,
+    },
+    env: {
+      KIBANA_URL: kibanaUrl,
+      ES_NODE: esNode,
+      ES_REQUEST_TIMEOUT: esRequestTimeout,
+      TEST_CLOUD: process.env.TEST_CLOUD,
+    },
+  });
+
+  return res;
+}
+
+function getCypressCliArgs() {
+  if (!process.env.CYPRESS_CLI_ARGS) {
+    return {};
+  }
+
+  const { $0, _, ...cypressCliArgs } = JSON.parse(
+    process.env.CYPRESS_CLI_ARGS
+  ) as Record<string, unknown>;
+
+  return cypressCliArgs;
+}

x-pack/plugins/apm/ftr_e2e/ftr_config.ts

@@ -7,8 +7,10 @@
 
 import { FtrConfigProviderContext } from '@kbn/test';
 import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { cypressTestRunner } from './cypress_test_runner';
+import { FtrProviderContext } from './ftr_provider_context';
 
-async function config({ readConfigFile }: FtrConfigProviderContext) {
+async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) {
   const kibanaCommonTestsConfig = await readConfigFile(
     require.resolve('../../../../test/common/config.js')
   );
@@ -40,8 +42,16 @@ async function config({ readConfigFile }: FtrConfigProviderContext) {
         `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
       ],
     },
+    testRunner: async (ftrProviderContext: FtrProviderContext) => {
+      const result = await cypressTestRunner(ftrProviderContext);
+
+      // set exit code explicitly if at least one Cypress test fails
+      if (result && (result.status === 'failed' || result.totalFailed > 0)) {
+        process.exitCode = 1;
+      }
+    },
   };
 }
 
 // eslint-disable-next-line import/no-default-export
-export default config;
+export default ftrConfig;

x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts

@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { FtrConfigProviderContext } from '@kbn/test';
-import cypress from 'cypress';
-import { FtrProviderContext } from './ftr_provider_context';
-import { cypressStart } from './cypress_start';
-
-async function ftrConfigOpen({ readConfigFile }: FtrConfigProviderContext) {
-  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
-  return {
-    ...kibanaConfig.getAll(),
-    testRunner,
-  };
-}
-
-export async function testRunner({ getService }: FtrProviderContext) {
-  await cypressStart(getService, cypress.open);
-}
-
-// eslint-disable-next-line import/no-default-export
-export default ftrConfigOpen;

x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts

@@ -1,31 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { FtrConfigProviderContext } from '@kbn/test';
-import cypress from 'cypress';
-import { cypressStart } from './cypress_start';
-import { FtrProviderContext } from './ftr_provider_context';
-
-async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) {
-  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
-
-  return {
-    ...kibanaConfig.getAll(),
-    testRunner,
-  };
-}
-
-async function testRunner({ getService }: FtrProviderContext) {
-  const result = await cypressStart(getService, cypress.run);
-
-  if (result && (result.status === 'failed' || result.totalFailed > 0)) {
-    process.exit(1);
-  }
-}
-
-// eslint-disable-next-line import/no-default-export
-export default ftrConfigRun;

x-pack/plugins/apm/scripts/test/e2e.js

@@ -12,6 +12,7 @@ const yargs = require('yargs');
 const childProcess = require('child_process');
 
 const { argv } = yargs(process.argv.slice(2))
+  .parserConfiguration({ 'unknown-options-as-args': true })
   .option('kibana-install-dir', {
     default: '',
     type: 'string',
@@ -28,67 +29,69 @@ const { argv } = yargs(process.argv.slice(2))
     description:
       'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
   })
-  .option('grep', {
-    alias: 'spec',
-    default: false,
-    type: 'string',
-    description:
-      'Specify the spec files to run (use doublequotes for glob matching)',
-  })
-  .option('open', {
-    default: false,
-    type: 'boolean',
-    description: 'Opens the Cypress Test Runner',
+  .option('times', {
+    type: 'number',
+    description: 'Repeat the test n number of times',
   })
   .option('bail', {
     default: false,
     type: 'boolean',
     description: 'stop tests after the first failure',
   })
-  .option('times', {
-    type: 'number',
-    description: 'Repeat the test n number of times',
-  })
   .help();
 
-const { server, runner, open, grep, bail, kibanaInstallDir } = argv;
-
 const e2eDir = path.join(__dirname, '../../ftr_e2e');
 
-let ftrScript = 'functional_tests';
-if (server) {
-  ftrScript = 'functional_tests_server';
-} else if (runner || open) {
-  ftrScript = 'functional_test_runner';
+let ftrScript = 'functional_tests.js';
+if (argv.server) {
+  ftrScript = 'functional_tests_server.js';
+} else if (argv.runner) {
+  ftrScript = 'functional_test_runner.js';
 }
 
-const config = open ? './ftr_config_open.ts' : './ftr_config_run.ts';
-const grepArg = grep ? `--grep "${grep}"` : '';
-const bailArg = bail ? `--bail` : '';
-const cmd = `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`;
+const cypressCliArgs = yargs(argv._).parserConfiguration({
+  'boolean-negation': false,
+}).argv;
+
+if (cypressCliArgs.grep) {
+  throw new Error('--grep is not supported. Please use --spec instead');
+}
+
+const spawnArgs = [
+  `../../../../scripts/${ftrScript}`,
+  `--config=./ftr_config.ts`,
+  `--kibana-install-dir=${argv.kibanaInstallDir}`,
+  ...(argv.bail ? [`--bail`] : []),
+];
 
 function runTests() {
-  console.log(`Running "${cmd}"`);
-  childProcess.execSync(cmd, { cwd: e2eDir, stdio: 'inherit' });
+  console.log(`Running e2e tests: "node ${spawnArgs.join(' ')}"`);
+
+  return childProcess.spawnSync('node', spawnArgs, {
+    cwd: e2eDir,
+    env: { ...process.env, CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs) },
+    encoding: 'utf8',
+    stdio: 'inherit',
+  });
 }
 
-if (argv.times) {
-  const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
-  let exitStatus = 0;
-  times(argv.times, () => {
-    try {
-      runTests();
-      runCounter.succeeded++;
-    } catch (e) {
-      exitStatus = 1;
-      runCounter.failed++;
-    }
-    runCounter.remaining--;
-    if (argv.times > 1) {
-      console.log(runCounter);
-    }
-  });
-  process.exit(exitStatus);
-} else {
-  runTests();
-}
+const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
+let exitStatus = 0;
+times(argv.times ?? 1, () => {
+  const child = runTests();
+  if (child.status === 0) {
+    runCounter.succeeded++;
+  } else {
+    exitStatus = child.status;
+    runCounter.failed++;
+  }
+
+  runCounter.remaining--;
+
+  if (argv.times > 1) {
+    console.log(runCounter);
+  }
+});
+
+process.exitCode = exitStatus;
+console.log(`Quitting with exit code ${exitStatus}`);