[Security Solution] Adding serverlessQA tag (#167494)

## Summary

Currently, we have our cypress tests properly integrated with the first
quality gate. 


In this quality gate, our tests are executed as part of the PR process
using the serverless FTR environment. This environment uses the latest
elasticsearch snapshot and the kibana source code, but it is not a real
serverless environment because is not a deployed project on the cloud.

In order to continue assuring that we don’t introduce new issues, we
want to execute automated tests in the second quality gate as well. The
second quality gate is a deployed serverless project in the QA
environment.

We want to start moving slowly in that direction and at the same time
make sure we don't introduce flakiness, and the tests perform well.

In this PR we are creating the `@serverlessQA` tag. Tests including that
label will be executed on the second quality gate. 
We are also adding
the label to one of the tests we know that is stable in serverless to
test the full integration.

Note that currently, we have some known limitations in this gate pending
to be solved and we are working on it:
- Execution of the tests is sequential
- No differentiation between teams during the execution is done over the
AET umbrella
- The environment is set in `complete` so tests that exercise the
`essential` behaviour should not be added.
- Please ping me in case you want to add more tests into that gate to
asses the risk before we have the gate tested and all the serverless
tests stabilized.

---------

Co-authored-by: Georgii Gorbachev <banderror@gmail.com>
Co-authored-by: dkirchan <diamantis.kirchantzoglou@elastic.co>

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
-
 set -euo pipefail
 
-echo "In the entrypoint for the quality gate"
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+echo "--- Serverless Security Second Quality Gate"
+cd x-pack/test/security_solution_cypress
+set +e
+
+VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
+ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+
+CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status

x-pack/test/security_solution_cypress/cypress/README.md

@@ -42,7 +42,9 @@ Please, before opening a PR with the new test, please make sure that the test fa
 
 Note that we use tags in order to select which tests we want to execute:
 
-- `@serverless` includes a test in the Serverless test suite. You need to explicitly add this tag to any test you want to run against a Serverless environment.
+- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate). You need to explicitly add this tag to any test you want to run in CI for open PRs. These tests will run against a local, "simulated" serverless environment.
+- `@serverlessQA` includes a test in the Serverless test suite for QA (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in the CD pipeline against real serverless projects deployed in the Serverless QA environment.
+  - **NOTE:** We are adding this tag temporarily until we check the behavior of our tests in the second quality gate.
 - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
 - `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.
 - `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why".

x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineCypressConfig } from '@kbn/cypress-config';
+import { esArchiver } from './support/es_archiver';
+
+// eslint-disable-next-line import/no-default-export
+export default defineCypressConfig({
+  reporter: '../../../node_modules/cypress-multi-reporters',
+  reporterOptions: {
+    configFile: './cypress/reporter_config.json',
+  },
+  defaultCommandTimeout: 150000,
+  env: {
+    grepFilterSpecs: true,
+    grepOmitFiltered: true,
+    grepTags: '@serverlessQA --@brokenInServerless --@skipInServerless',
+    // Grep plugin is working taking under consideration the directory where cypress lives.
+    // https://github.com/elastic/kibana/pull/167494#discussion_r1340567022 for more context.
+    grepIntegrationFolder: '../',
+  },
+  execTimeout: 150000,
+  pageLoadTimeout: 150000,
+  numTestsKeptInMemory: 0,
+  retries: {
+    runMode: 1,
+  },
+  screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  videosFolder: '../../../../target/kibana-security-solution/cypress/videos',
+  viewportHeight: 946,
+  viewportWidth: 1680,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    experimentalCspAllowList: ['default-src', 'script-src', 'script-src-elem'],
+    experimentalMemoryManagement: true,
+    specPattern: './cypress/e2e/**/*.cy.ts',
+    setupNodeEvents(on, config) {
+      esArchiver(on, config);
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      require('@cypress/grep/src/plugin')(config);
+      return config;
+    },
+  },
+});

x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts

@@ -17,7 +17,7 @@ import { cleanKibana } from '../../../tasks/common';
 import { createTimeline, favoriteTimeline } from '../../../tasks/api_calls/timelines';
 import { getTimeline } from '../../../objects/timeline';
 
-describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => {
+describe('Overview Page', { tags: ['@ess', '@serverless', '@serverlessQA'] }, () => {
   before(() => {
     cleanKibana();
     cy.task('esArchiverLoad', { archiveName: 'overview' });

x-pack/test/security_solution_cypress/package.json

@@ -23,6 +23,7 @@
       "cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'",
       "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'",
       "cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true",
+      "cypress:run:qa:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless_qa.config.ts --env CLOUD_SERVERLESS=true",
       "cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'",
       "cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'",
       "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5",