[Security Solution] [Serverless] Integrates Cypress in visual mode with QA environment (#171107)

x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts

@@ -24,8 +24,8 @@ import pRetry from 'p-retry';
 
 import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
 import { INITIAL_REST_VERSION } from '@kbn/data-views-plugin/server/constants';
+import { exec } from 'child_process';
 import { renderSummaryTable } from './print_run';
-import type { SecuritySolutionDescribeBlockFtrConfig } from './utils';
 import { parseTestFileConfig, retrieveIntegrations } from './utils';
 
 interface ProductType {
@@ -53,6 +53,12 @@ interface Credentials {
   password: string;
 }
 
+const DEFAULT_CONFIGURATION: Readonly<ProductType[]> = [
+  { product_line: 'security', product_tier: 'complete' },
+  { product_line: 'cloud', product_tier: 'complete' },
+  { product_line: 'endpoint', product_tier: 'complete' },
+] as const;
+
 const DEFAULT_REGION = 'aws-eu-west-1';
 const PROJECT_NAME_PREFIX = 'kibana-cypress-security-solution-ephemeral';
 const BASE_ENV_URL = 'https://global.qa.cld.elstc.co';
@@ -82,19 +88,14 @@ const getApiKeyFromElasticCloudJsonFile = (): string | undefined => {
 async function createSecurityProject(
   projectName: string,
   apiKey: string,
-  ftrConfig: SecuritySolutionDescribeBlockFtrConfig
+  productTypes: ProductType[]
 ): Promise<Project | undefined> {
   const body: CreateProjectRequestBody = {
     name: projectName,
     region_id: DEFAULT_REGION,
+    product_types: productTypes,
   };
 
-  const productTypes: ProductType[] = [];
-  ftrConfig?.productTypes?.forEach((t) => {
-    productTypes.push(t as ProductType);
-  });
-  if (productTypes.length > 0) body.product_types = productTypes;
-
   try {
     const response = await axios.post(`${BASE_ENV_URL}/api/v1/serverless/projects/security`, body, {
       headers: {
@@ -325,9 +326,32 @@ function waitForKibanaLogin(kbUrl: string, credentials: Credentials): Promise<vo
   return pRetry(fetchLoginStatusAttempt, retryOptions);
 }
 
+const getProductTypes = (
+  tier: string,
+  endpointAddon: boolean,
+  cloudAddon: boolean
+): ProductType[] => {
+  let productTypes: ProductType[] = [...DEFAULT_CONFIGURATION];
+
+  if (tier) {
+    productTypes = productTypes.map((product) => ({
+      ...product,
+      product_tier: tier,
+    }));
+  }
+  if (!cloudAddon) {
+    productTypes = productTypes.filter((product) => product.product_line !== 'cloud');
+  }
+  if (!endpointAddon) {
+    productTypes = productTypes.filter((product) => product.product_line !== 'endpoint');
+  }
+
+  return productTypes;
+};
+
 export const cli = () => {
   run(
-    async () => {
+    async (context) => {
       log = new ToolingLog({
         level: 'info',
         writeTo: process.stdout,
@@ -371,7 +395,22 @@ export const cli = () => {
             }
             return acc;
           }, {} as Record<string, string | number>)
-        );
+        )
+        .option('tier', {
+          alias: 't',
+          type: 'string',
+          default: 'complete',
+        })
+        .option('endpointAddon', {
+          alias: 'ea',
+          type: 'boolean',
+          default: true,
+        })
+        .option('cloudAddon', {
+          alias: 'ca',
+          type: 'boolean',
+          default: true,
+        });
 
       log.info(`
 ----------------------------------------------
@@ -388,6 +427,10 @@ ${JSON.stringify(argv, null, 2)}
       const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string;
       const cypressConfigFile = await import(cypressConfigFilePath);
 
+      const tier: string = argv.tier;
+      const endpointAddon: boolean = argv.endpointAddon;
+      const cloudAddon: boolean = argv.cloudAddon;
+
       log.info(`
 ----------------------------------------------
 Cypress config for file: ${cypressConfigFilePath}:
@@ -456,7 +499,10 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
           await withProcRunner(log, async (procs) => {
             const id = crypto.randomBytes(8).toString('hex');
             const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`;
-            const specFileFTRConfig = parseTestFileConfig(filePath);
+
+            const productTypes = isOpen
+              ? getProductTypes(tier, endpointAddon, cloudAddon)
+              : (parseTestFileConfig(filePath).productTypes as ProductType[]);
 
             if (!API_KEY) {
               log.info('API KEY to create project could not be retrieved.');
@@ -466,7 +512,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
 
             log.info(`${id}: Creating project ${PROJECT_NAME}...`);
             // Creating project for the test to run
-            const project = await createSecurityProject(PROJECT_NAME, API_KEY, specFileFTRConfig);
+            const project = await createSecurityProject(PROJECT_NAME, API_KEY, productTypes);
 
             if (!project) {
               log.info('Failed to create project.');
@@ -474,6 +520,11 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
               return process.exit(1);
             }
 
+            context.addCleanupTask(() => {
+              const command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`;
+              exec(command);
+            });
+
             // Reset credentials for elastic user
             const credentials = await resetCredentials(project.id, id, API_KEY);
 
@@ -553,15 +604,13 @@ ${JSON.stringify(cypressConfigFile, null, 2)}
                     env: cyCustomEnv,
                   },
                 });
+                // Delete serverless project
+                log.info(`${id} : Deleting project ${PROJECT_NAME}...`);
+                await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY);
               } catch (error) {
                 result = error;
               }
             }
-
-            // Delete serverless project
-            log.info(`${id} : Deleting project ${PROJECT_NAME}...`);
-            await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY);
-
             return result;
           });
           return result;

x-pack/test/security_solution_cypress/cypress/README.md

@@ -42,11 +42,10 @@ Please, before opening a PR with the new test, please make sure that the test fa
 
 Note that we use tags in order to select which tests we want to execute:
 
-- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate). You need to explicitly add this tag to any test you want to run in CI for open PRs. These tests will run against a local, "simulated" serverless environment.
+- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate) and QA environemnt (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in CI for serverless. 
-- `@serverlessQA` includes a test in the Serverless test suite for QA (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in the CD pipeline against real serverless projects deployed in the Serverless QA environment.
-  - **NOTE:** We are adding this tag temporarily until we check the behavior of our tests in the second quality gate.
 - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
 - `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.
+- `@brokenInServerlessQA` excludes a test form the Serverless QA enviornment (second quality gate). Indicates that a test should run on it, but currently is broken.
 - `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why".
 
 Please, before opening a PR with a new test, make sure that the test fails. If you never see your test fail you don’t know if your test is actually testing the right thing, or testing anything at all.
@@ -72,6 +71,10 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio
 | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode |
 | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode |
 | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode |
+| cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. |
+| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.|
+| cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. |
+| cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. |
 | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory |
 
 Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging.
@@ -190,7 +193,7 @@ Task [cypress/support/es_archiver.ts](https://github.com/elastic/kibana/blob/mai
 Note that we use tags in order to select which tests we want to execute, if you want a test to be executed on serverless you need to add @serverless tag to it.
 
 
-### Running the serverless tests locally
+### Running serverless tests locally pointing to FTR serverless (First Quality Gate) 
 
 Run the tests with the following yarn scripts from `x-pack/test/security_solution_cypress`:
 
@@ -203,7 +206,7 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio
 
 Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging.
 
-### PLIs
+#### PLIs
 When running serverless Cypress tests, the following PLIs are set by default:
 
 ```
@@ -234,6 +237,97 @@ Per the way we set the environment during the execution process on CI, the above
 
 For test developing or test debugging purposes, you need to modify the configuration but without committing and pushing the changes in `x-pack/test/security_solution_cypress/serverless_config.ts`.
 
+
+### Running serverless tests locally pointing to a MKI project created in QA environment (Second Quality Gate)
+
+Run the tests with the following yarn scripts from `x-pack/test/security_solution_cypress`:
+
+| Script Name | Description |
+| ----------- | ----------- |
+| cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. |
+| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.|
+| cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. |
+| cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. |
+
+Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging.
+
+
+#### Setup required
+
+Setup a valid Elastic Cloud API key for QA environment:
+
+1. Navigate to QA environment.
+2. Click on the `User menu button` located on the top right of the header.
+3. Click on `Organization`.
+4. Click on the `API keys` tab.
+5. Click on `Create API key` button.
+6. Add a name, set an expiration date, assign an organization owner role.
+7. Click on `Create API key`
+8. Save the value of the key
+
+Store the saved key on `~/.elastic/cloud.json` using the following format:
+
+```json
+{
+  "api_key": {
+    "qa": "<API_KEY>"
+  }
+}
+```
+
+#### Known limitations 
+- Currently RBAC cannot be tested.
+
+#### PLIs
+
+When running serverless Cypress tests on QA environment, the following PLIs are set by default:
+```
+      { product_line: 'security', product_tier: 'complete' },
+      { product_line: 'endpoint', product_tier: 'complete' },
+      { product_line: 'cloud', product_tier: 'complete' },
+```
+
+With the above configuration we'll be able to cover most of the scenarios, but there are some cases were we might want to use a different configuration. In that case, we just need to pass to the header of the test, which is the configuration we want for it.
+
+```typescript
+describe(
+  'Entity Analytics Dashboard in Serverless',
+  {
+    tags: '@serverless',
+    env: {
+      ftrConfig: {
+        productTypes: [
+          { product_line: 'security', product_tier: 'essentials' },
+          { product_line: 'endpoint', product_tier: 'essentials' },
+        ],
+      },
+    },
+  },
+```
+
+For test developing or test debugging purposes on QA, you have avaialable the following options:
+
+```
+yarn cypress:open:qa:serverless --tier <essentials|complete>
+```
+
+The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment with the passed tier essentials or complete. If no flag is passed, the project will be cretaed as complete.
+
+```
+yarn cypress:open:qa:serverless --no-endpoint-addon
+```
+
+The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the endpoint add-on.
+
+```
+yarn cypress:open:qa:serverless --no-cloud-addon
+```
+
+The above command will open the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment without the cloud add-on.
+
+Note that all the above flags can be combined.
+
+
 ## Development Best Practices
 
 Below you will a set of best practices that should be followed when writing Cypress tests.

x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts

@@ -41,6 +41,7 @@ export default defineCypressConfig({
     specPattern: './cypress/e2e/**/*.cy.ts',
     setupNodeEvents(on, config) {
       esArchiver(on, config);
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
       // eslint-disable-next-line @typescript-eslint/no-var-requires
       require('@cypress/grep/src/plugin')(config);
       return config;

x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts

@@ -16,7 +16,7 @@ import { OVERVIEW_URL } from '../../../urls/navigation';
 import { createTimeline, favoriteTimeline } from '../../../tasks/api_calls/timelines';
 import { getTimeline } from '../../../objects/timeline';
 
-describe('Overview Page', { tags: ['@ess', '@serverless', '@serverlessQA'] }, () => {
+describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => {
   before(() => {
     cy.task('esArchiverLoad', { archiveName: 'overview' });
   });

x-pack/test/security_solution_cypress/package.json

@@ -28,6 +28,7 @@
     "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5",
     "cypress:burn:serverless": "yarn cypress:serverless --env burn=2",
     "cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts",
+    "cypress:open:qa:serverless": "yarn cypress:qa:serverless open",
     "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'",
     "cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'",
     "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'"