[Profiling] Adding cypress e2e (#153145)

.buildkite/ftr_configs.yml

@@ -47,6 +47,9 @@ disabled:
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/plugins/profiling/e2e/ftr_config.ts
 
   # Elastic Synthetics configs
   - x-pack/plugins/synthetics/e2e/config.ts

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -35,6 +35,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Tests'

.buildkite/pipelines/pull_request/profiling_cypress.yml

@@ -0,0 +1,14 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profiling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -115,6 +115,13 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/profiling/])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/profiling_cypress.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([/^x-pack\/plugins\/fleet/, /^x-pack\/test\/fleet_cypress/])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')

.buildkite/scripts/steps/functional/profiling_cypress.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export JOB=kibana-profiling-cypress
+
+echo "--- Profiling Cypress Tests"
+
+cd "$XPACK_DIR"
+
+node plugins/profiling/scripts/test/e2e.js \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

x-pack/plugins/profiling/e2e/README.md

@@ -0,0 +1,51 @@
+# Profiling E2E tests
+
+Profiling uses [FTR](../../../../packages/kbn-test/README.md) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/plugins/profiling/e2e/cypress/e2e`.
+
+## E2E Tests (Cypress)
+
+The E2E tests are located in [`x-pack/plugins/profiling/e2e`](../e2e).
+
+Tests run on buildkite PR pipeline are parallelized (4 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/profiling_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/profiling_cypress.yml) with the property `parallelism`.
+
+```yml
+    ...
+    depends_on: build
+    parallelism: 4
+    ...
+```
+
+## Running it locally
+
+### Start test server
+
+```
+node x-pack/plugins/profiling/scripts/test/e2e --server
+```
+
+### Run tests
+Runs all tests in the terminal
+
+```
+node x-pack/plugins/profiling/scripts/test/e2e --runner
+```
+
+### Open cypress dashboard
+Opens cypress dashboard, there it's possible to select what test you want to run.
+
+```
+node x-pack/plugins/profiling/scripts/test/e2e --open
+```
+### Arguments
+
+| Option       | Description                                     |
+| ------------ | ----------------------------------------------- |
+| --server     | Only start ES and Kibana                        |
+| --runner     | Only run tests                                  |
+| --spec       | Specify the specs to run                        |
+| --times      | Repeat the test n number of times               |
+| --bail       | stop tests after the first failure              |
+
+```
+node x-pack/plugins/profiling/scripts/test/e2e.js --runner --spec cypress/e2e/profiling.cy.ts --times 2
+```

x-pack/plugins/profiling/e2e/cypress.config.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineCypressConfig } from '@kbn/cypress-config';
+
+export default defineCypressConfig({
+  fileServerFolder: './cypress',
+  fixturesFolder: './cypress/fixtures',
+  screenshotsFolder: './cypress/screenshots',
+  videosFolder: './cypress/videos',
+  requestTimeout: 10000,
+  responseTimeout: 40000,
+  defaultCommandTimeout: 30000,
+  execTimeout: 120000,
+  pageLoadTimeout: 120000,
+  viewportHeight: 1800,
+  viewportWidth: 1440,
+  video: false,
+  videoUploadOnPasses: false,
+  screenshotOnRunFailure: false,
+  retries: {
+    runMode: 1,
+  },
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    supportFile: './cypress/support/e2e.ts',
+    specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}',
+  },
+});

x-pack/plugins/profiling/e2e/cypress/e2e/profiling.cy.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const start = '2021-10-10T00:00:00.000Z';
+const end = '2021-10-10T00:15:00.000Z';
+
+describe('Profiling', () => {
+  before(() => {
+    cy.loginAsElastic();
+  });
+  it('Shows profiling empty prompt', () => {
+    cy.visitKibana('/app/profiling');
+    cy.contains('Universal Profiling (now in Beta)');
+  });
+});

x-pack/plugins/profiling/e2e/cypress/support/commands.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+Cypress.Commands.add(
+  'loginAs',
+  ({ username, password }: { username: string; password: string }) => {
+    const kibanaUrl = Cypress.env('KIBANA_URL');
+    cy.log(`Logging in as ${username} on ${kibanaUrl}`);
+    cy.visit('/');
+    cy.request({
+      log: true,
+      method: 'POST',
+      url: `${kibanaUrl}/internal/security/login`,
+      body: {
+        providerType: 'basic',
+        providerName: 'basic',
+        currentURL: `${kibanaUrl}/login`,
+        params: { username, password },
+      },
+      headers: {
+        'kbn-xsrf': 'e2e_test',
+      },
+    });
+    cy.visit('/');
+  }
+);
+
+Cypress.Commands.add('loginAsElastic', () => {
+  return cy.loginAs({
+    username: 'elastic',
+    password: 'changeme',
+  });
+});
+
+Cypress.Commands.add('getByTestSubj', (selector: string) => {
+  return cy.get(`[data-test-subj="${selector}"]`);
+});
+
+Cypress.Commands.add('visitKibana', (url: string) => {
+  cy.visit(url);
+  cy.getByTestSubj('kbnLoadingMessage').should('exist');
+  cy.getByTestSubj('kbnLoadingMessage').should('not.exist', {
+    timeout: 50000,
+  });
+});

x-pack/plugins/profiling/e2e/cypress/support/e2e.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+Cypress.on('uncaught:exception', (err, runnable) => {
+  return false;
+});
+
+import './commands';
+// import './output_command_timings';

x-pack/plugins/profiling/e2e/cypress/support/types.d.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+declare namespace Cypress {
+  interface Chainable {
+    loginAs(params: {
+      username: string;
+      password: string;
+    }): Cypress.Chainable<Cypress.Response<any>>;
+    loginAsElastic(): Cypress.Chainable<Cypress.Response<any>>;
+    getByTestSubj(selector: string): Chainable<JQuery<Element>>;
+    visitKibana(url: string): void;
+  }
+}

x-pack/plugins/profiling/e2e/cypress_test_runner.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import cypress from 'cypress';
+import path from 'path';
+import Url from 'url';
+import { FtrProviderContext } from './ftr_provider_context';
+
+export async function cypressTestRunner({
+  ftrProviderContext: { getService },
+  cypressExecution,
+}: {
+  ftrProviderContext: FtrProviderContext;
+  cypressExecution: typeof cypress.run | typeof cypress.open;
+}) {
+  const config = getService('config');
+
+  const username = config.get('servers.elasticsearch.username');
+  const password = config.get('servers.elasticsearch.password');
+
+  const esNode = Url.format({
+    protocol: config.get('servers.elasticsearch.protocol'),
+    port: config.get('servers.elasticsearch.port'),
+    hostname: config.get('servers.elasticsearch.hostname'),
+    auth: `${username}:${password}`,
+  });
+
+  const esRequestTimeout = config.get('timeouts.esRequestTimeout');
+
+  const kibanaUrlWithoutAuth = Url.format({
+    protocol: config.get('servers.kibana.protocol'),
+    hostname: config.get('servers.kibana.hostname'),
+    port: config.get('servers.kibana.port'),
+  });
+
+  const cypressProjectPath = path.join(__dirname);
+  const { open, ...cypressCliArgs } = getCypressCliArgs();
+  // const cypressExecution = cypress.open; // open ? cypress.open : cypress.run;
+  const res = await cypressExecution({
+    ...cypressCliArgs,
+    project: cypressProjectPath,
+    config: {
+      e2e: {
+        baseUrl: kibanaUrlWithoutAuth,
+      },
+    },
+    env: {
+      KIBANA_URL: kibanaUrlWithoutAuth,
+      ES_NODE: esNode,
+      ES_REQUEST_TIMEOUT: esRequestTimeout,
+      TEST_CLOUD: process.env.TEST_CLOUD,
+    },
+  });
+
+  return res;
+}
+
+function getCypressCliArgs(): Record<string, unknown> {
+  if (!process.env.CYPRESS_CLI_ARGS) {
+    return {};
+  }
+
+  const { $0, _, ...cypressCliArgs } = JSON.parse(process.env.CYPRESS_CLI_ARGS) as Record<
+    string,
+    unknown
+  >;
+
+  const spec =
+    typeof cypressCliArgs.spec === 'string' && !cypressCliArgs.spec.includes('**')
+      ? `**/${cypressCliArgs.spec}*`
+      : cypressCliArgs.spec;
+
+  return { ...cypressCliArgs, spec };
+}

x-pack/plugins/profiling/e2e/ftr_config.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+
+async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaCommonTestsConfig = await readConfigFile(
+    require.resolve('../../../../test/common/config.js')
+  );
+  const xpackFunctionalTestsConfig = await readConfigFile(
+    require.resolve('../../../test/functional/config.base.js')
+  );
+
+  return {
+    ...kibanaCommonTestsConfig.getAll(),
+
+    esTestCluster: {
+      ...xpackFunctionalTestsConfig.get('esTestCluster'),
+      serverArgs: [
+        ...xpackFunctionalTestsConfig.get('esTestCluster.serverArgs'),
+        // define custom es server here
+        // API Keys is enabled at the top level
+        'xpack.security.enabled=true',
+      ],
+    },
+
+    kbnTestServer: {
+      ...xpackFunctionalTestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xpackFunctionalTestsConfig.get('kbnTestServer.serverArgs'),
+        '--home.disableWelcomeScreen=true',
+        '--csp.strict=false',
+        '--csp.warnLegacyBrowsers=false',
+        // define custom kibana server args here
+        `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+        '--xpack.profiling.enabled=true',
+      ],
+    },
+  };
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ftrConfig;

x-pack/plugins/profiling/e2e/ftr_config_open.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import cypress from 'cypress';
+import { FtrProviderContext } from './ftr_provider_context';
+import { cypressTestRunner } from './cypress_test_runner';
+
+async function ftrConfigOpen({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
+  return {
+    ...kibanaConfig.getAll(),
+    testRunner,
+  };
+}
+
+export async function testRunner(ftrProviderContext: FtrProviderContext) {
+  await cypressTestRunner({ ftrProviderContext, cypressExecution: cypress.open });
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ftrConfigOpen;

x-pack/plugins/profiling/e2e/ftr_config_runner.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import cypress from 'cypress';
+import { cypressTestRunner } from './cypress_test_runner';
+import { FtrProviderContext } from './ftr_provider_context';
+
+async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
+
+  return {
+    ...kibanaConfig.getAll(),
+    testRunner,
+  };
+}
+
+async function testRunner(ftrProviderContext: FtrProviderContext) {
+  const result = await cypressTestRunner({ ftrProviderContext, cypressExecution: cypress.run });
+
+  if (result && (result.status === 'failed' || result.totalFailed > 0)) {
+    process.exit(1);
+  }
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ftrConfigRun;

x-pack/plugins/profiling/e2e/ftr_provider_context.d.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test';
+
+export type FtrProviderContext = GenericFtrProviderContext<{}, {}>;

x-pack/plugins/profiling/e2e/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "include": [
+    "**/*"
+  ],
+  "exclude": [
+    "tmp",
+    "target/**/*"
+  ],
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "cypress",
+      "node",
+      "cypress-real-events"
+    ],
+    "isolatedModules": false
+  },
+  "kbn_references": [
+    { "path": "../../../test/tsconfig.json" },
+    { "path": "../../../../test/tsconfig.json" },
+    "@kbn/test",
+    "@kbn/dev-utils",
+    "@kbn/cypress-config",
+  ]
+}

x-pack/plugins/profiling/scripts/test/e2e.js

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+const { times } = require('lodash');
+const path = require('path');
+const yargs = require('yargs');
+const childProcess = require('child_process');
+
+const { argv } = yargs(process.argv.slice(2))
+  .parserConfiguration({ 'unknown-options-as-args': true })
+  .option('kibana-install-dir', {
+    default: '',
+    type: 'string',
+    description: 'Path to the Kibana install directory',
+  })
+  .option('server', {
+    default: false,
+    type: 'boolean',
+    description: 'Start Elasticsearch and Kibana',
+  })
+  .option('runner', {
+    default: false,
+    type: 'boolean',
+    description:
+      'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
+  })
+  .option('open', {
+    default: false,
+    type: 'boolean',
+    description:
+      'Open cypress dashboard (an instance of Elasticsearch and kibana are needs to be available)',
+  })
+  .option('times', {
+    type: 'number',
+    description: 'Repeat the test n number of times',
+  })
+  .option('bail', {
+    default: false,
+    type: 'boolean',
+    description: 'stop tests after the first failure',
+  })
+  .help();
+
+const e2eDir = path.join(__dirname, '../../e2e');
+
+let ftrScript = 'functional_tests.js';
+if (argv.server) {
+  ftrScript = 'functional_tests_server.js';
+} else if (argv.runner || argv.open) {
+  ftrScript = 'functional_test_runner.js';
+}
+
+const cypressCliArgs = yargs(argv._).parserConfiguration({
+  'boolean-negation': false,
+}).argv;
+
+if (cypressCliArgs.grep) {
+  throw new Error('--grep is not supported. Please use --spec instead');
+}
+
+const ftrConfig = argv.open ? './ftr_config_open.ts' : './ftr_config_runner.ts';
+const spawnArgs = [
+  `../../../../scripts/${ftrScript}`,
+  `--config=${ftrConfig}`,
+  `--kibana-install-dir=${argv.kibanaInstallDir}`,
+  ...(argv.bail ? [`--bail`] : []),
+];
+
+function runTests() {
+  console.log(`Running e2e tests: "node ${spawnArgs.join(' ')}"`);
+
+  return childProcess.spawnSync('node', spawnArgs, {
+    cwd: e2eDir,
+    env: { ...process.env, CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs) },
+    encoding: 'utf8',
+    stdio: 'inherit',
+  });
+}
+
+const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
+let exitStatus = 0;
+times(argv.times ?? 1, () => {
+  const child = runTests();
+  if (child.status === 0) {
+    runCounter.succeeded++;
+  } else {
+    exitStatus = child.status;
+    runCounter.failed++;
+  }
+
+  runCounter.remaining--;
+
+  if (argv.times > 1) {
+    console.log(runCounter);
+  }
+});
+
+process.exitCode = exitStatus;
+console.log(`Quitting with exit code ${exitStatus}`);