[Security solution]Dynamic split of cypress tests (#125986)

- adds `parallelism: 4` for security_solution cypress buildkite pipeline
- added parsing /integrations folder with cypress tests, to retrieve paths to individual test files using `glob` utility
- list of test files split equally between agents(there are approx 70+ tests files, split ~20 per job with **parallelism=4**)
- small refactoring of existing cypress runners for `security_solution`

Old metrics(before @MadameSheema https://github.com/elastic/kibana/pull/127558 performance improvements):
before split: average time of completion ~ 1h 40m for tests, 1h 55m for Kibana build
after split in 4 chunks: chunk completion between 20m - 30m, Kibana build 1h 20m

**Current metrics:**
before split: average time of completion ~ 1h for tests, 1h 10m for Kibana build
after split in 4 chunks: each chunk completion between 10m - 20m, 1h Kibana build 1h

.buildkite/ftr_configs.yml

@@ -26,6 +26,7 @@ disabled:
   - x-pack/test/security_solution_cypress/cases_cli_config.ts
   - x-pack/test/security_solution_cypress/ccs_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
+  - x-pack/test/security_solution_cypress/cli_config_parallel.ts
   - x-pack/test/security_solution_cypress/config.firefox.ts
   - x-pack/test/security_solution_cypress/config.ts
   - x-pack/test/security_solution_cypress/response_ops_cli_config.ts

.buildkite/pipelines/pull_request/security_solution.yml

@@ -5,6 +5,7 @@ steps:
       queue: ci-group-6
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'

.buildkite/scripts/steps/functional/security_solution.sh

@@ -5,11 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export CLI_NUMBER=$((BUILDKITE_PARALLEL_JOB+1))
+export CLI_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
 
 echo "--- Security Solution tests (Chrome)"
 
-checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome)" \
+checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome) $CLI_NUMBER" \
  node scripts/functional_tests \
    --debug --bail \
    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-   --config x-pack/test/security_solution_cypress/cli_config.ts
+   --config x-pack/test/security_solution_cypress/cli_config_parallel.ts

x-pack/plugins/security_solution/cypress/README.md

@@ -64,6 +64,18 @@ A headless browser is a browser simulation program that does not have a user int
 
 This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/security_solution_cypress`
 
+Tests run on buildkite PR pipeline is parallelized(current value = 4 parallel jobs). It can be configured in [.buildkite/pipelines/pull_request/security_solution.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/security_solution.yml) with property `parallelism` 
+
+```yml
+    ...
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    ...
+```
+
 #### Custom Targets
 
 This configuration runs cypress tests against an arbitrary host.

x-pack/plugins/security_solution/cypress/integration/users/user_details.spec.ts

@@ -24,13 +24,14 @@ describe('user details flyout', () => {
   before(() => {
     cleanKibana();
     login();
+  });
+
+  it('shows user detail flyout from alert table', () => {
     visitWithoutDateRange(ALERTS_URL);
     createCustomRuleEnabled({ ...getNewRule(), customQuery: 'user.name:*' });
     refreshPage();
     waitForAlertsToPopulate();
-  });
 
-  it('shows user detail flyout from alert table', () => {
     scrollAlertTableColumnIntoView(USER_COLUMN);
     expandAlertTableCellValue(USER_COLUMN);
     openUserDetailsFlyout();

x-pack/plugins/security_solution/package.json

@@ -13,12 +13,13 @@
     "cypress:open-as-ci": "node ../../../scripts/functional_tests --config ../../test/security_solution_cypress/visual_config.ts",
     "cypress:open:upgrade": "yarn cypress:open --config integrationFolder=./cypress/upgrade_integration",
     "cypress:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/integration/**/*.spec.ts'; status=$?; yarn junit:merge && exit $status",
+    "cypress:run:spec": "yarn cypress:run:reporter --browser chrome --spec ${SPEC_LIST:-'./cypress/integration/**/*.spec.ts'}; status=$?; yarn junit:merge && exit $status",
     "cypress:run:cases": "yarn cypress:run:reporter --browser chrome --spec './cypress/integration/cases/*.spec.ts'; status=$?; yarn junit:merge && exit $status",
     "cypress:run:firefox": "yarn cypress:run:reporter --browser firefox --spec './cypress/integration/**/*.spec.ts'; status=$?; yarn junit:merge && exit $status",
     "cypress:run:reporter": "yarn cypress run --config-file ./cypress/cypress.json --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json",
     "cypress:run:respops": "yarn cypress:run:reporter --browser chrome --spec ./cypress/integration/detection_alerts/*.spec.ts,./cypress/integration/detection_rules/*.spec.ts,./cypress/integration/exceptions/*.spec.ts; status=$?; yarn junit:merge && exit $status",
     "cypress:run:ccs": "yarn cypress:run:reporter --browser chrome --config integrationFolder=./cypress/ccs_integration; status=$?; yarn junit:merge && exit $status",
-    "cypress:run-as-ci": "node --max-old-space-size=2048 ../../../scripts/functional_tests --config ../../test/security_solution_cypress/cli_config.ts",
+    "cypress:run-as-ci": "node --max-old-space-size=2048 ../../../scripts/functional_tests --config ../../test/security_solution_cypress/cli_config_parallel.ts",
     "cypress:run-as-ci:firefox": "node --max-old-space-size=2048 ../../../scripts/functional_tests --config ../../test/security_solution_cypress/config.firefox.ts",
     "cypress:run:upgrade": "yarn cypress:run:reporter --browser chrome --config integrationFolder=./cypress/upgrade_integration",
     "cypress:run:upgrade:old": "yarn cypress:run:reporter --browser chrome --config integrationFolder=./cypress/upgrade_integration --spec ./cypress/upgrade_integration/threat_hunting/**/*.spec.ts,./cypress/upgrade_integration/detections/**/custom_query_rule.spec.ts; status=$?; yarn junit:merge && exit $status",

x-pack/test/security_solution_cypress/cli_config_parallel.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { FtrProviderContext } from './ftr_provider_context';
+
+import { SecuritySolutionCypressCliTestRunnerCI } from './runner';
+
+const cliNumber = parseInt(process.env.CLI_NUMBER ?? '1', 10);
+const cliCount = parseInt(process.env.CLI_COUNT ?? '1', 10);
+
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const securitySolutionCypressConfig = await readConfigFile(require.resolve('./config.ts'));
+  return {
+    ...securitySolutionCypressConfig.getAll(),
+
+    testRunner: (context: FtrProviderContext) =>
+      SecuritySolutionCypressCliTestRunnerCI(context, cliCount, cliNumber),
+  };
+}

x-pack/test/security_solution_cypress/runner.ts

@@ -5,7 +5,10 @@
  * 2.0.
  */
 
+import { chunk } from 'lodash';
 import { resolve } from 'path';
+import glob from 'glob';
+
 import Url from 'url';
 
 import { withProcRunner } from '@kbn/dev-proc-runner';
@@ -13,7 +16,22 @@ import { withProcRunner } from '@kbn/dev-proc-runner';
 import semver from 'semver';
 import { FtrProviderContext } from './ftr_provider_context';
 
-export async function SecuritySolutionCypressCliTestRunner({ getService }: FtrProviderContext) {
+const retrieveIntegrations = (chunksTotal: number, chunkIndex: number) => {
+  const pattern = resolve(
+    __dirname,
+    '../../plugins/security_solution/cypress/integration/**/*.spec.ts'
+  );
+  const integrationsPaths = glob.sync(pattern);
+  const chunkSize = Math.ceil(integrationsPaths.length / chunksTotal);
+
+  return chunk(integrationsPaths, chunkSize)[chunkIndex - 1];
+};
+
+export async function SecuritySolutionConfigurableCypressTestRunner(
+  { getService }: FtrProviderContext,
+  command: string,
+  envVars?: Record<string, string>
+) {
   const log = getService('log');
   const config = getService('config');
   const esArchiver = getService('esArchiver');
@@ -23,7 +41,7 @@ export async function SecuritySolutionCypressCliTestRunner({ getService }: FtrPr
   await withProcRunner(log, async (procs) => {
     await procs.run('cypress', {
       cmd: 'yarn',
-      args: ['cypress:run'],
+      args: [command],
       cwd: resolve(__dirname, '../../plugins/security_solution'),
       env: {
         FORCE_COLOR: '1',
@@ -32,91 +50,42 @@ export async function SecuritySolutionCypressCliTestRunner({ getService }: FtrPr
         CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
         CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
         ...process.env,
+        ...envVars,
       },
       wait: true,
     });
   });
 }
 
-export async function SecuritySolutionCypressCliResponseOpsTestRunner({
-  getService,
-}: FtrProviderContext) {
-  const log = getService('log');
-  const config = getService('config');
-  const esArchiver = getService('esArchiver');
-
-  await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat');
-
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:run:respops'],
-      cwd: resolve(__dirname, '../../plugins/security_solution'),
-      env: {
-        FORCE_COLOR: '1',
-        CYPRESS_BASE_URL: Url.format(config.get('servers.kibana')),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        ...process.env,
-      },
-      wait: true,
-    });
+export async function SecuritySolutionCypressCliTestRunnerCI(
+  context: FtrProviderContext,
+  totalCiJobs: number,
+  ciJobNumber: number
+) {
+  const integrations = retrieveIntegrations(totalCiJobs, ciJobNumber);
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:run:spec', {
+    SPEC_LIST: integrations.join(','),
   });
 }
 
-export async function SecuritySolutionCypressCliCasesTestRunner({
-  getService,
-}: FtrProviderContext) {
-  const log = getService('log');
-  const config = getService('config');
-  const esArchiver = getService('esArchiver');
-
-  await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat');
-
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:run:cases'],
-      cwd: resolve(__dirname, '../../plugins/security_solution'),
-      env: {
-        FORCE_COLOR: '1',
-        CYPRESS_BASE_URL: Url.format(config.get('servers.kibana')),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        ...process.env,
-      },
-      wait: true,
-    });
-  });
+export async function SecuritySolutionCypressCliResponseOpsTestRunner(context: FtrProviderContext) {
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:run:respops');
 }
 
-export async function SecuritySolutionCypressCliFirefoxTestRunner({
-  getService,
-}: FtrProviderContext) {
-  const log = getService('log');
-  const config = getService('config');
-  const esArchiver = getService('esArchiver');
+export async function SecuritySolutionCypressCliCasesTestRunner(context: FtrProviderContext) {
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:run:cases');
+}
 
-  await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat');
+export async function SecuritySolutionCypressCliTestRunner(context: FtrProviderContext) {
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:run');
+}
 
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:run:firefox'],
-      cwd: resolve(__dirname, '../../plugins/security_solution'),
-      env: {
-        FORCE_COLOR: '1',
-        CYPRESS_BASE_URL: Url.format(config.get('servers.kibana')),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        ...process.env,
-      },
-      wait: true,
-    });
-  });
+export async function SecuritySolutionCypressCliFirefoxTestRunner(context: FtrProviderContext) {
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:run:firefox');
+}
+
+export async function SecuritySolutionCypressVisualTestRunner(context: FtrProviderContext) {
+  return SecuritySolutionConfigurableCypressTestRunner(context, 'cypress:open');
 }
 
 export async function SecuritySolutionCypressCcsTestRunner({ getService }: FtrProviderContext) {
@@ -143,31 +112,6 @@ export async function SecuritySolutionCypressCcsTestRunner({ getService }: FtrPr
   });
 }
 
-export async function SecuritySolutionCypressVisualTestRunner({ getService }: FtrProviderContext) {
-  const log = getService('log');
-  const config = getService('config');
-  const esArchiver = getService('esArchiver');
-
-  await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat');
-
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:open'],
-      cwd: resolve(__dirname, '../../plugins/security_solution'),
-      env: {
-        FORCE_COLOR: '1',
-        CYPRESS_BASE_URL: Url.format(config.get('servers.kibana')),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        ...process.env,
-      },
-      wait: true,
-    });
-  });
-}
-
 export async function SecuritySolutionCypressUpgradeCliTestRunner({
   getService,
 }: FtrProviderContext) {