github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fixed parallel script for cypress tests in QA and buildkite (#169311)

Browse source 
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Gloria Hornero <gloria.hornero@elastic.co>
Co-authored-by: Maxim Palenov <maxim.palenov@elastic.co>
This commit is contained in:
dkirchan 2023-11-06 19:21:01 +02:00 • committed by GitHub
parent bb3bbc9e94
commit ed4ef2a3ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 1259 additions and 676 deletions
Download patch file Download diff file Expand all files Collapse all files

36
.buildkite/pipelines/security_solution/base.yml Normal file
View file

@ -0,0 +1,36 @@
steps:
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless
label: 'Serverless MKI QA Security Cypress Tests'
agents:
queue: n2-4-spot
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
timeout_in_minutes: 300
parallelism: 6
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
agents:
queue: n2-4-spot
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
timeout_in_minutes: 300
parallelism: 4
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
agents:
queue: n2-4-spot
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
timeout_in_minutes: 300
parallelism: 8
retry:
automatic:
- exit_status: '*'
limit: 1

24
.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh Executable file
View file

@ -0,0 +1,24 @@
#!/bin/bash
set -euo pipefail
if [ -z "$1" ]
then
echo "No target script from the package.json file, is supplied"
exit 1
fi
source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/functional/common_cypress.sh
.buildkite/scripts/bootstrap.sh
export JOB=kibana-security-solution-chrome
buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
cd x-pack/test/security_solution_cypress
set +e
QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

18
.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh
View file

@ -1,19 +1,5 @@
#!/bin/bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/functional/common_cypress.sh
.buildkite/scripts/bootstrap.sh
export JOB=kibana-security-solution-chrome
buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
echo "--- Serverless Security Second Quality Gate"
cd x-pack/test/security_solution_cypress
set +e
VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status
ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

38
.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { execSync } from 'child_process';
import fs from 'fs';
const getPipeline = (filename: string, removeSteps = true) => {
const str = fs.readFileSync(filename).toString();
return removeSteps ? str.replace(/^steps:/, '') : str;
};
const uploadPipeline = (pipelineContent: string | object) => {
const str =
typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
execSync('buildkite-agent pipeline upload', {
input: str,
stdio: ['pipe', 'inherit', 'inherit'],
});
};
(async () => {
try {
const pipeline = [];
pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false));
// remove duplicated steps
uploadPipeline([...new Set(pipeline)].join('\n'));
} catch (ex) {
console.error('PR pipeline generation error', ex.message);
process.exit(1);
}
})();

31
x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts
View file

@ -28,38 +28,9 @@ import {
import { createFailError } from '@kbn/dev-cli-errors';
import pRetry from 'p-retry';
import { renderSummaryTable } from './print_run';
import { isSkipped, parseTestFileConfig } from './utils';
import { parseTestFileConfig, retrieveIntegrations } from './utils';
import { getFTRConfig } from './get_ftr_config';
/**
* Retrieve test files using a glob pattern.
* If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB
*/
const retrieveIntegrations = (integrationsPaths: string[]) => {
const nonSkippedSpecs = integrationsPaths.filter((filePath) => !isSkipped(filePath));
if (process.env.RUN_ALL_TESTS === 'true') {
return nonSkippedSpecs;
} else {
// The number of instances of this job were created
const chunksTotal: number = process.env.BUILDKITE_PARALLEL_JOB_COUNT
? parseInt(process.env.BUILDKITE_PARALLEL_JOB_COUNT, 10)
: 1;
// An index which uniquely identifies this instance of the job
const chunkIndex: number = process.env.BUILDKITE_PARALLEL_JOB
? parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10)
: 0;
const nonSkippedSpecsForChunk: string[] = [];
for (let i = chunkIndex; i < nonSkippedSpecs.length; i += chunksTotal) {
nonSkippedSpecsForChunk.push(nonSkippedSpecs[i]);
}
return nonSkippedSpecsForChunk;
}
};
export const cli = () => {
run(
async () => {

480
x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts Normal file
View file

@ -0,0 +1,480 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { run } from '@kbn/dev-cli-runner';
import yargs from 'yargs';
import _ from 'lodash';
import globby from 'globby';
import pMap from 'p-map';
import { ToolingLog } from '@kbn/tooling-log';
import { withProcRunner } from '@kbn/dev-proc-runner';
import cypress from 'cypress';
import grep from '@cypress/grep/src/plugin';
import crypto from 'crypto';
import fs from 'fs';
import { createFailError } from '@kbn/dev-cli-errors';
import axios, { AxiosError } from 'axios';
import path from 'path';
import os from 'os';
import pRetry from 'p-retry';
import { renderSummaryTable } from './print_run';
import type { SecuritySolutionDescribeBlockFtrConfig } from './utils';
import { parseTestFileConfig, retrieveIntegrations } from './utils';
interface ProductType {
product_line: string;
product_tier: string;
}
interface CreateEnvironmentRequestBody {
name: string;
region_id: string;
product_types?: ProductType[];
}
interface Environment {
name: string;
id: string;
region: string;
es_url: string;
kb_url: string;
product: string;
}
interface Credentials {
username: string;
password: string;
}
const DEFAULT_REGION = 'aws-eu-west-1';
const PROJECT_NAME_PREFIX = 'kibana-cypress-security-solution-ephemeral';
const BASE_ENV_URL = 'https://global.qa.cld.elstc.co';
let log: ToolingLog;
const delay = async (timeout: number) => {
await new Promise((r) => setTimeout(r, timeout));
};
const getApiKeyFromElasticCloudJsonFile = (): string | undefined => {
const userHomeDir = os.homedir();
try {
const jsonString = fs.readFileSync(path.join(userHomeDir, '.elastic/cloud.json'), 'utf-8');
const jsonData = JSON.parse(jsonString);
return jsonData.api_key.qa;
} catch (e) {
log.info('API KEY could not be found in .elastic/cloud.json');
}
};
// Method to invoke the create environment API for serverless.
async function createEnvironment(
projectName: string,
apiKey: string,
ftrConfig: SecuritySolutionDescribeBlockFtrConfig
): Promise<Environment | undefined> {
const body: CreateEnvironmentRequestBody = {
name: projectName,
region_id: DEFAULT_REGION,
};
const productTypes: ProductType[] = [];
ftrConfig?.productTypes?.forEach((t) => {
productTypes.push(t as ProductType);
});
if (productTypes.length > 0) body.product_types = productTypes;
try {
const response = await axios.post(`${BASE_ENV_URL}/api/v1/serverless/projects/security`, body, {
headers: {
Authorization: `ApiKey ${apiKey}`,
},
});
return {
name: response.data.name,
id: response.data.id,
region: response.data.region_id,
es_url: `${response.data.endpoints.elasticsearch}:443`,
kb_url: `${response.data.endpoints.kibana}:443`,
product: response.data.type,
};
} catch (error) {
if (error instanceof AxiosError) {
log.error(`${error.response?.status}:${error.response?.data}`);
} else {
log.error(`${error.message}`);
}
}
}
// Method to invoke the delete environment API for serverless.
async function deleteEnvironment(
projectId: string,
projectName: string,
apiKey: string
): Promise<void> {
try {
await axios.delete(`${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}`, {
headers: {
Authorization: `ApiKey ${apiKey}`,
},
});
log.info(`Environment ${projectName} was successfully deleted!`);
} catch (error) {
if (error instanceof AxiosError) {
log.error(`${error.response?.status}:${error.response?.data}`);
} else {
log.error(`${error.message}`);
}
}
}
// Method to reset the credentials for the created environment.
async function resetCredentials(
environmentId: string,
runnerId: string,
apiKey: string
): Promise<Credentials | undefined> {
log.info(`${runnerId} : Reseting credentials`);
try {
const response = await axios.post(
`${BASE_ENV_URL}/api/v1/serverless/projects/security/${environmentId}/_reset-credentials`,
{},
{
headers: {
Authorization: `ApiKey ${apiKey}`,
},
}
);
return {
password: response.data.password,
username: response.data.username,
};
} catch (error) {
throw new Error(`${error.message}`);
}
}
// Wait until elasticsearch status goes green
function waitForEsStatusGreen(esUrl: string, auth: string, runnerId: string): Promise<void> {
const fetchHealthStatusAttempt = async (attemptNum: number) => {
log.info(`Retry number ${attemptNum} to check if es is green.`);
const response = await axios.get(`${esUrl}/_cluster/health?wait_for_status=green&timeout=50s`, {
headers: {
Authorization: `Basic ${auth}`,
},
});
log.info(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`);
};
const retryOptions = {
onFailedAttempt: (error: Error | AxiosError) => {
if (error instanceof AxiosError && error.code === 'ENOTFOUND') {
log.info(
`${runnerId}: The elasticsearch url is not yet reachable. A retry will be triggered soon...`
);
}
},
retries: 50,
factor: 2,
maxTimeout: 20000,
};
return pRetry(fetchHealthStatusAttempt, retryOptions);
}
// Wait until Kibana is available
function waitForKibanaAvailable(kbUrl: string, auth: string, runnerId: string): Promise<void> {
const fetchKibanaStatusAttempt = async (attemptNum: number) => {
log.info(`Retry number ${attemptNum} to check if kibana is available.`);
const response = await axios.get(`${kbUrl}/api/status`, {
headers: {
Authorization: `Basic ${auth}`,
},
});
if (response.data.status.overall.level !== 'available') {
throw new Error(`${runnerId}: Kibana is not available. Retrying in 20s...`);
} else {
log.info(`${runnerId}: Kibana status overall is ${response.data.status.overall.level}.`);
}
};
const retryOptions = {
onFailedAttempt: (error: Error | AxiosError) => {
if (error instanceof AxiosError && error.code === 'ENOTFOUND') {
log.info(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`);
} else {
log.info(`${runnerId}: ${error}`);
}
},
retries: 50,
factor: 2,
maxTimeout: 20000,
};
return pRetry(fetchKibanaStatusAttempt, retryOptions);
}
export const cli = () => {
run(
async () => {
log = new ToolingLog({
level: 'info',
writeTo: process.stdout,
});
// Checking if API key is either provided via env variable or in ~/.elastic.cloud.json
if (!process.env.CLOUD_QA_API_KEY && !getApiKeyFromElasticCloudJsonFile()) {
log.error('The api key for the environment needs to be provided with the env var API_KEY.');
log.error(
'If running locally, ~/.elastic/cloud.json is attempted to be read which contains the api key.'
);
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
const API_KEY = process.env.CLOUD_QA_API_KEY
? process.env.CLOUD_QA_API_KEY
: getApiKeyFromElasticCloudJsonFile();
const PARALLEL_COUNT = process.env.PARALLEL_COUNT ? Number(process.env.PARALLEL_COUNT) : 1;
if (!process.env.CLOUD_ENV) {
log.warning(
'The cloud environment to be provided with the env var CLOUD_ENV. Currently working only for QA so the script can proceed.'
);
// Abort when more environments will be integrated
// return process.exit(0);
}
const { argv } = yargs(process.argv.slice(2))
.coerce('configFile', (arg) => (_.isArray(arg) ? _.last(arg) : arg))
.coerce('spec', (arg) => (_.isArray(arg) ? _.last(arg) : arg))
.coerce('env', (arg: string) =>
arg.split(',').reduce((acc, curr) => {
const [key, value] = curr.split('=');
if (key === 'burn') {
acc[key] = parseInt(value, 10);
} else {
acc[key] = value;
}
return acc;
}, {} as Record<string, string | number>)
);
log.info(`
----------------------------------------------
Script arguments:
----------------------------------------------
${JSON.stringify(argv, null, 2)}
----------------------------------------------
`);
const isOpen = argv._.includes('open');
const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string;
const cypressConfigFile = await import(cypressConfigFilePath);
log.info(`
----------------------------------------------
Cypress config for file: ${cypressConfigFilePath}:
----------------------------------------------
${JSON.stringify(cypressConfigFile, null, 2)}
----------------------------------------------
`);
const specConfig = cypressConfigFile.e2e.specPattern;
const specArg = argv.spec;
const specPattern = specArg ?? specConfig;
log.info('Config spec pattern:', specConfig);
log.info('Arguments spec pattern:', specArg);
log.info('Resulting spec pattern:', specPattern);
// The grep function will filter Cypress specs by tags: it will include and exclude
// spec files according to the tags configuration.
const grepSpecPattern = grep({
...cypressConfigFile,
specPattern,
excludeSpecPattern: [],
}).specPattern;
log.info('Resolved spec files or pattern after grep:', grepSpecPattern);
const isGrepReturnedFilePaths = _.isArray(grepSpecPattern);
const isGrepReturnedSpecPattern = !isGrepReturnedFilePaths && grepSpecPattern === specPattern;
const grepFilterSpecs = cypressConfigFile.env?.grepFilterSpecs;
// IMPORTANT!
// When grep returns the same spec pattern as it gets in its arguments, we treat it as
// it couldn't find any concrete specs to execute (maybe because all of them are skipped).
// In this case, we do an early return - it's important to do that.
// If we don't return early, these specs will start executing, and Cypress will be skipping
// tests at runtime: those that should be excluded according to the tags passed in the config.
// This can take so much time that the job can fail by timeout in CI.
if (grepFilterSpecs && isGrepReturnedSpecPattern) {
log.info('No tests found - all tests could have been skipped via Cypress tags');
// eslint-disable-next-line no-process-exit
return process.exit(0);
}
const concreteFilePaths = isGrepReturnedFilePaths
? grepSpecPattern // use the returned concrete file paths
: globby.sync(specPattern); // convert the glob pattern to concrete file paths
const files = retrieveIntegrations(concreteFilePaths);
log.info('Resolved spec files after retrieveIntegrations:', files);
if (!files?.length) {
log.info('No tests found');
// eslint-disable-next-line no-process-exit
return process.exit(0);
}
const results = await pMap(
files,
async (filePath) => {
let result:
| CypressCommandLine.CypressRunResult
| CypressCommandLine.CypressFailedRunResult
| undefined;
await withProcRunner(log, async (procs) => {
const id = crypto.randomBytes(8).toString('hex');
const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`;
const specFileFTRConfig = parseTestFileConfig(filePath);
if (!API_KEY) {
log.info('API KEY to create environment could not be retrieved.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
log.info(`${id}: Creating environment ${PROJECT_NAME}...`);
// Creating environment for the test to run
const environment = await createEnvironment(PROJECT_NAME, API_KEY, specFileFTRConfig);
if (!environment) {
log.info('Failed to create environment.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
// Reset credentials for elastic user
const credentials = await resetCredentials(environment.id, id, API_KEY);
if (!credentials) {
log.info('Credentials could not be reset.');
// eslint-disable-next-line no-process-exit
return process.exit(1);
}
// Wait for 8 minutes in order for the environment to be ready
delay(480000);
// Base64 encode the credentials in order to invoke ES and KB APIs
const auth = btoa(`${credentials.username}:${credentials.password}`);
// Wait for elasticsearch status to go green.
await waitForEsStatusGreen(environment.es_url, auth, id);
// Wait until Kibana is available
await waitForKibanaAvailable(environment.kb_url, auth, id);
// Normalized the set of available env vars in cypress
const cyCustomEnv = {
CYPRESS_BASE_URL: environment.kb_url,
ELASTICSEARCH_URL: environment.es_url,
ELASTICSEARCH_USERNAME: credentials.username,
ELASTICSEARCH_PASSWORD: credentials.password,
KIBANA_URL: environment.kb_url,
KIBANA_USERNAME: credentials.username,
KIBANA_PASSWORD: credentials.password,
CLOUD_SERVERLESS: true,
};
if (process.env.DEBUG && !process.env.CI) {
log.info(`
----------------------------------------------
Cypress run ENV for file: ${filePath}:
----------------------------------------------
${JSON.stringify(cyCustomEnv, null, 2)}
----------------------------------------------
`);
}
if (isOpen) {
await cypress.open({
configFile: cypressConfigFilePath,
config: {
e2e: {
baseUrl: environment.kb_url,
},
env: cyCustomEnv,
},
});
} else {
try {
result = await cypress.run({
browser: 'electron',
spec: filePath,
configFile: cypressConfigFilePath,
reporter: argv.reporter as string,
reporterOptions: argv.reporterOptions,
headed: argv.headed as boolean,
config: {
e2e: {
baseUrl: environment.kb_url,
},
numTestsKeptInMemory: 0,
env: cyCustomEnv,
},
});
} catch (error) {
result = error;
}
}
// Delete serverless environment
log.info(`${id} : Deleting Environment ${PROJECT_NAME}...`);
await deleteEnvironment(environment.id, PROJECT_NAME, API_KEY);
return result;
});
return result;
},
{
concurrency: PARALLEL_COUNT,
}
);
if (results) {
renderSummaryTable(results as CypressCommandLine.CypressRunResult[]);
const hasFailedTests = _.some(
results,
(result) =>
(result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' ||
(result as CypressCommandLine.CypressRunResult)?.totalFailed
);
if (hasFailedTests) {
throw createFailError('Not all tests passed');
}
}
},
{
flags: {
allowUnexpected: true,
},
}
);
};

29
x-pack/plugins/security_solution/scripts/run_cypress/utils.ts
View file

@ -13,6 +13,35 @@ import type { ExpressionStatement, ObjectExpression, ObjectProperty } from '@bab
import { schema, type TypeOf } from '@kbn/config-schema';
import { getExperimentalAllowedValues } from '../../common/experimental_features';
/**
* Retrieve test files using a glob pattern.
* If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB
*/
export const retrieveIntegrations = (integrationsPaths: string[]) => {
const nonSkippedSpecs = integrationsPaths.filter((filePath) => !isSkipped(filePath));
if (process.env.RUN_ALL_TESTS === 'true') {
return nonSkippedSpecs;
} else {
// The number of instances of this job were created
const chunksTotal: number = process.env.BUILDKITE_PARALLEL_JOB_COUNT
? parseInt(process.env.BUILDKITE_PARALLEL_JOB_COUNT, 10)
: 1;
// An index which uniquely identifies this instance of the job
const chunkIndex: number = process.env.BUILDKITE_PARALLEL_JOB
? parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10)
: 0;
const nonSkippedSpecsForChunk: string[] = [];
for (let i = chunkIndex; i < nonSkippedSpecs.length; i += chunksTotal) {
nonSkippedSpecsForChunk.push(nonSkippedSpecs[i]);
}
return nonSkippedSpecsForChunk;
}
};
export const isSkipped = (filePath: string): boolean => {
const testFile = fs.readFileSync(filePath, { encoding: 'utf8' });

9
x-pack/plugins/security_solution/scripts/start_cypress_parallel_serverless.js Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
require('../../../../src/setup_node_env');
require('./run_cypress/parallel_serverless').cli();

2
x-pack/plugins/security_solution/tsconfig.json
View file

@ -177,6 +177,6 @@
"@kbn/react-kibana-mount",
"@kbn/unified-doc-viewer-plugin",
"@kbn/shared-ux-error-boundary",
"@kbn/zod-helpers"
"@kbn/zod-helpers",
]
}

13
x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts
View file

@ -14,18 +14,17 @@ export default defineCypressConfig({
reporterOptions: {
configFile: './cypress/reporter_config.json',
},
defaultCommandTimeout: 150000,
defaultCommandTimeout: 300000,
env: {
grepFilterSpecs: true,
grepOmitFiltered: true,
grepTags: '@serverlessQA --@brokenInServerless --@skipInServerless',
// Grep plugin is working taking under consideration the directory where cypress lives.
// https://github.com/elastic/kibana/pull/167494#discussion_r1340567022 for more context.
grepIntegrationFolder: '../',
grepTags: '@serverless --@brokenInServerless --@skipInServerless --@brokenInServerlessQA',
},
execTimeout: 150000,
pageLoadTimeout: 150000,
execTimeout: 300000,
pageLoadTimeout: 300000,
numTestsKeptInMemory: 0,
requestTimeout: 300000,
responseTimeout: 300000,
retries: {
runMode: 1,
},

164
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/missing_privileges_callout.cy.ts
View file

@ -44,101 +44,105 @@ const waitForPageTitleToBeShown = () => {
cy.get(PAGE_TITLE).should('be.visible');
};
describe('Detections > Callouts', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
// First, we have to open the app on behalf of a privileged user in order to initialize it.
// Otherwise the app will be disabled and show a "welcome"-like page.
login();
visit(ALERTS_URL);
waitForPageTitleToBeShown();
});
describe(
'Detections > Callouts',
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
before(() => {
// First, we have to open the app on behalf of a privileged user in order to initialize it.
// Otherwise the app will be disabled and show a "welcome"-like page.
login();
visit(ALERTS_URL);
waitForPageTitleToBeShown();
});
context('indicating read-only access to resources', () => {
context('On Detections home page', () => {
beforeEach(() => {
loadPageAsReadOnlyUser(ALERTS_URL);
});
context('indicating read-only access to resources', () => {
context('On Detections home page', () => {
beforeEach(() => {
loadPageAsReadOnlyUser(ALERTS_URL);
});
it('We show one primary callout', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
});
context('When a user clicks Dismiss on the callout', () => {
it('We hide it and persist the dismissal', () => {
it('We show one primary callout', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
dismissCallOut(MISSING_PRIVILEGES_CALLOUT);
reloadPage();
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
context('When a user clicks Dismiss on the callout', () => {
it('We hide it and persist the dismissal', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
dismissCallOut(MISSING_PRIVILEGES_CALLOUT);
reloadPage();
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
});
// FYI: Rules Management check moved to ../detection_rules/all_rules_read_only.spec.ts
context('On Rule Details page', () => {
beforeEach(() => {
createRule(getNewRule()).then((rule) =>
loadPageAsReadOnlyUser(ruleDetailsUrl(rule.body.id))
);
});
afterEach(() => {
deleteCustomRule();
});
it('We show one primary callout', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
});
context('When a user clicks Dismiss on the callouts', () => {
it('We hide them and persist the dismissal', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
dismissCallOut(MISSING_PRIVILEGES_CALLOUT);
reloadPage();
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
});
});
// FYI: Rules Management check moved to ../detection_rules/all_rules_read_only.spec.ts
context('indicating read-write access to resources', () => {
context('On Detections home page', () => {
beforeEach(() => {
loadPageAsPlatformEngineer(ALERTS_URL);
});
context('On Rule Details page', () => {
beforeEach(() => {
createRule(getNewRule()).then((rule) =>
loadPageAsReadOnlyUser(ruleDetailsUrl(rule.body.id))
);
it('We show no callout', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
afterEach(() => {
deleteCustomRule();
context('On Rules Management page', () => {
beforeEach(() => {
login(ROLES.platform_engineer);
loadPageAsPlatformEngineer(RULES_MANAGEMENT_URL);
});
it('We show no callout', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
it('We show one primary callout', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
});
context('On Rule Details page', () => {
beforeEach(() => {
createRule(getNewRule()).then((rule) =>
loadPageAsPlatformEngineer(ruleDetailsUrl(rule.body.id))
);
});
context('When a user clicks Dismiss on the callouts', () => {
it('We hide them and persist the dismissal', () => {
waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary');
dismissCallOut(MISSING_PRIVILEGES_CALLOUT);
reloadPage();
afterEach(() => {
deleteCustomRule();
});
it('We show no callouts', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
});
});
context('indicating read-write access to resources', () => {
context('On Detections home page', () => {
beforeEach(() => {
loadPageAsPlatformEngineer(ALERTS_URL);
});
it('We show no callout', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
context('On Rules Management page', () => {
beforeEach(() => {
login(ROLES.platform_engineer);
loadPageAsPlatformEngineer(RULES_MANAGEMENT_URL);
});
it('We show no callout', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
context('On Rule Details page', () => {
beforeEach(() => {
createRule(getNewRule()).then((rule) =>
loadPageAsPlatformEngineer(ruleDetailsUrl(rule.body.id))
);
});
afterEach(() => {
deleteCustomRule();
});
it('We show no callouts', () => {
getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist');
});
});
});
});
}
);

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts
View file

@ -68,7 +68,7 @@ const loginPageAsWriteAuthorizedUser = (url: string) => {
describe(
'Detection rules, Prebuilt Rules Installation and Update - Authorization/RBAC',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
beforeEach(() => {
preventPrebuiltRulesPackageInstallation();

3
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts
View file

@ -39,13 +39,12 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management';
describe(
'Detection rules, Prebuilt Rules Installation and Update - Error handling',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
beforeEach(() => {
preventPrebuiltRulesPackageInstallation();
cleanKibana();
login();
visitRulesManagementTable();
});

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts
View file

@ -31,7 +31,7 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management';
describe(
'Detection rules, Prebuilt Rules Installation and Update workflow',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
describe('Installation of prebuilt rules', () => {
const RULE_1 = createRuleAssetSavedObject({

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts
View file

@ -51,7 +51,7 @@ const rules = Array.from(Array(5)).map((_, i) => {
});
});
describe('Prebuilt rules', { tags: ['@ess', '@serverless'] }, () => {
describe('Prebuilt rules', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => {
before(() => {
cleanKibana();
});

180
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts
View file

@ -54,16 +54,20 @@ describe(
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
it('should NOT display install or update notifications when latest rules are installed', () => {
visitRulesManagementTable();
createAndInstallMockedPrebuiltRules([RULE_1]);
it(
'should NOT display install or update notifications when latest rules are installed',
{ tags: ['@brokenInServerlessQA'] },
() => {
visitRulesManagementTable();
createAndInstallMockedPrebuiltRules([RULE_1]);
/* Assert that there are no installation or update notifications */
/* Add Elastic Rules button should not contain a number badge */
/* and Rule Upgrade tab should not be displayed */
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', 'Add Elastic rules');
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
/* Assert that there are no installation or update notifications */
/* Add Elastic Rules button should not contain a number badge */
/* and Rule Upgrade tab should not be displayed */
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', 'Add Elastic rules');
cy.get(RULES_UPDATES_TAB).should('not.exist');
}
);
});
describe('Notifications', () => {
@ -71,60 +75,68 @@ describe(
installPrebuiltRuleAssets([RULE_1]);
});
describe('Rules installation notification when no rules have been installed', () => {
beforeEach(() => {
visitRulesManagementTable();
});
it('should notify user about prebuilt rules available for installation', () => {
cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible');
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`);
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
});
describe('Rule installation notification when at least one rule already installed', () => {
beforeEach(() => {
installAllPrebuiltRulesRequest().then(() => {
/* Create new rule assets with a different rule_id as the one that was */
/* installed before in order to trigger the installation notification */
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
});
const RULE_3 = createRuleAssetSavedObject({
name: 'Test rule 3',
rule_id: 'rule_3',
});
describe(
'Rules installation notification when no rules have been installed',
{ tags: ['@brokenInServerlessQA'] },
() => {
beforeEach(() => {
visitRulesManagementTable();
installPrebuiltRuleAssets([RULE_2, RULE_3]);
});
});
it('should notify user about prebuilt rules available for installation', () => {
const numberOfAvailableRules = 2;
cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible');
cy.get(ADD_ELASTIC_RULES_BTN).should(
'have.text',
`Add Elastic rules${numberOfAvailableRules}`
);
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
it('should notify user a rule is again available for installation if it is deleted', () => {
/* Install available rules, assert that the notification is gone */
/* then delete one rule and assert that the notification is back */
installAllPrebuiltRulesRequest().then(() => {
cy.reload();
deleteFirstRule();
it('should notify user about prebuilt rules available for installation', () => {
cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible');
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`);
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
});
});
}
);
describe('Rule update notification', () => {
describe(
'Rule installation notification when at least one rule already installed',
{ tags: ['@brokenInServerlessQA'] },
() => {
beforeEach(() => {
installAllPrebuiltRulesRequest().then(() => {
/* Create new rule assets with a different rule_id as the one that was */
/* installed before in order to trigger the installation notification */
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
});
const RULE_3 = createRuleAssetSavedObject({
name: 'Test rule 3',
rule_id: 'rule_3',
});
visitRulesManagementTable();
installPrebuiltRuleAssets([RULE_2, RULE_3]);
});
});
it('should notify user about prebuilt rules available for installation', () => {
const numberOfAvailableRules = 2;
cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible');
cy.get(ADD_ELASTIC_RULES_BTN).should(
'have.text',
`Add Elastic rules${numberOfAvailableRules}`
);
cy.get(RULES_UPDATES_TAB).should('not.exist');
});
it('should notify user a rule is again available for installation if it is deleted', () => {
/* Install available rules, assert that the notification is gone */
/* then delete one rule and assert that the notification is back */
installAllPrebuiltRulesRequest().then(() => {
cy.reload();
deleteFirstRule();
cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible');
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`);
});
});
}
);
describe('Rule update notification', { tags: ['@brokenInServerlessQA'] }, () => {
beforeEach(() => {
installAllPrebuiltRulesRequest().then(() => {
/* Create new rule asset with the same rule_id as the one that was installed */
@ -148,35 +160,39 @@ describe(
});
});
describe('Rule installation available and rule update available notifications', () => {
beforeEach(() => {
installAllPrebuiltRulesRequest().then(() => {
/* Create new rule assets with a different rule_id as the one that was */
/* installed before in order to trigger the installation notification */
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
describe(
'Rule installation available and rule update available notifications',
{ tags: ['@brokenInServerlessQA'] },
() => {
beforeEach(() => {
installAllPrebuiltRulesRequest().then(() => {
/* Create new rule assets with a different rule_id as the one that was */
/* installed before in order to trigger the installation notification */
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
});
/* Create new rule asset with the same rule_id as the one that was installed */
/* but with a higher version, in order to trigger the update notification */
const UPDATED_RULE = createRuleAssetSavedObject({
name: 'Test rule 1.1 (updated)',
rule_id: 'rule_1',
version: 2,
});
installPrebuiltRuleAssets([RULE_2, UPDATED_RULE]);
visitRulesManagementTable();
});
/* Create new rule asset with the same rule_id as the one that was installed */
/* but with a higher version, in order to trigger the update notification */
const UPDATED_RULE = createRuleAssetSavedObject({
name: 'Test rule 1.1 (updated)',
rule_id: 'rule_1',
version: 2,
});
installPrebuiltRuleAssets([RULE_2, UPDATED_RULE]);
visitRulesManagementTable();
});
});
it('should notify user about prebuilt rules available for installation and for upgrade', () => {
// 1 rule available for installation
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`);
// 1 rule available for update
cy.get(RULES_UPDATES_TAB).should('be.visible');
cy.get(RULES_UPDATES_TAB).should('have.text', `Rule Updates${1}`);
});
});
it('should notify user about prebuilt rules available for installation and for upgrade', () => {
// 1 rule available for installation
cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`);
// 1 rule available for update
cy.get(RULES_UPDATES_TAB).should('be.visible');
cy.get(RULES_UPDATES_TAB).should('have.text', `Rule Updates${1}`);
});
}
);
});
}
);

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts
View file

@ -31,7 +31,7 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management';
describe(
'Detection rules, Prebuilt Rules Installation and Update workflow',
{ tags: ['@ess', '@serverless'] },
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
describe('Upgrade of prebuilt rules', () => {
const RULE_1_ID = 'rule_1';

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/common_flows.cy.ts
View file

@ -48,13 +48,13 @@ import { visit } from '../../../tasks/navigation';
// them in the relevant /rule_creation/[RULE_TYPE].cy.ts test.
describe('Common rule creation flows', { tags: ['@ess', '@serverless'] }, () => {
beforeEach(() => {
login();
deleteAlertsAndRules();
createTimeline(getTimeline())
.then((response) => {
return response.body.data.persistTimeline.timeline.savedObjectId;
})
.as('timelineId');
login();
visit(CREATE_RULE_URL);
});

6
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/event_correlation_rule.cy.ts
View file

@ -43,7 +43,7 @@ import {
import { getDetails, waitForTheRuleToBeExecuted } from '../../../tasks/rule_details';
import { expectNumberOfRules, goToRuleDetailsOf } from '../../../tasks/alerts_detection_rules';
import { cleanKibana, deleteAlertsAndRules } from '../../../tasks/common';
import { deleteAlertsAndRules } from '../../../tasks/common';
import {
createAndEnableRule,
fillAboutRuleAndContinue,
@ -58,10 +58,6 @@ import { openRuleManagementPageViaBreadcrumbs } from '../../../tasks/rules_manag
import { CREATE_RULE_URL } from '../../../urls/navigation';
describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
});
beforeEach(() => {
login();
deleteAlertsAndRules();

126
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/machine_learning_rule.cy.ts
View file

@ -41,7 +41,6 @@ import {
import { getDetails } from '../../../tasks/rule_details';
import { expectNumberOfRules, goToRuleDetailsOf } from '../../../tasks/alerts_detection_rules';
import { cleanKibana } from '../../../tasks/common';
import {
createAndEnableRule,
fillAboutRuleAndContinue,
@ -54,74 +53,77 @@ import { visit } from '../../../tasks/navigation';
import { openRuleManagementPageViaBreadcrumbs } from '../../../tasks/rules_management';
import { CREATE_RULE_URL } from '../../../urls/navigation';
describe('Machine Learning rules', { tags: ['@ess', '@serverless'] }, () => {
const expectedUrls = (getMachineLearningRule().references ?? []).join('');
const expectedFalsePositives = (getMachineLearningRule().false_positives ?? []).join('');
const expectedTags = (getMachineLearningRule().tags ?? []).join('');
const expectedMitre = formatMitreAttackDescription(getMachineLearningRule().threat ?? []);
const expectedNumberOfRules = 1;
describe(
'Machine Learning rules',
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
const expectedUrls = (getMachineLearningRule().references ?? []).join('');
const expectedFalsePositives = (getMachineLearningRule().false_positives ?? []).join('');
const expectedTags = (getMachineLearningRule().tags ?? []).join('');
const expectedMitre = formatMitreAttackDescription(getMachineLearningRule().threat ?? []);
const expectedNumberOfRules = 1;
beforeEach(() => {
cleanKibana();
login();
visit(CREATE_RULE_URL);
});
beforeEach(() => {
login();
visit(CREATE_RULE_URL);
});
it('Creates and enables a new ml rule', () => {
const mlRule = getMachineLearningRule();
selectMachineLearningRuleType();
fillDefineMachineLearningRuleAndContinue(mlRule);
fillAboutRuleAndContinue(mlRule);
fillScheduleRuleAndContinue(mlRule);
createAndEnableRule();
openRuleManagementPageViaBreadcrumbs();
it('Creates and enables a new ml rule', () => {
const mlRule = getMachineLearningRule();
selectMachineLearningRuleType();
fillDefineMachineLearningRuleAndContinue(mlRule);
fillAboutRuleAndContinue(mlRule);
fillScheduleRuleAndContinue(mlRule);
createAndEnableRule();
openRuleManagementPageViaBreadcrumbs();
cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)');
cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)');
expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules);
expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules);
cy.get(RULE_NAME).should('have.text', mlRule.name);
cy.get(RISK_SCORE).should('have.text', mlRule.risk_score);
cy.get(SEVERITY).should('have.text', 'Critical');
cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true');
cy.get(RULE_NAME).should('have.text', mlRule.name);
cy.get(RISK_SCORE).should('have.text', mlRule.risk_score);
cy.get(SEVERITY).should('have.text', 'Critical');
cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true');
goToRuleDetailsOf(mlRule.name);
goToRuleDetailsOf(mlRule.name);
cy.get(RULE_NAME_HEADER).should('contain', `${mlRule.name}`);
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', mlRule.description);
cy.get(ABOUT_DETAILS).within(() => {
getDetails(SEVERITY_DETAILS).should('have.text', 'Critical');
getDetails(RISK_SCORE_DETAILS).should('have.text', mlRule.risk_score);
getDetails(REFERENCE_URLS_DETAILS).should((details) => {
expect(removeExternalLinkText(details.text())).equal(expectedUrls);
cy.get(RULE_NAME_HEADER).should('contain', `${mlRule.name}`);
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', mlRule.description);
cy.get(ABOUT_DETAILS).within(() => {
getDetails(SEVERITY_DETAILS).should('have.text', 'Critical');
getDetails(RISK_SCORE_DETAILS).should('have.text', mlRule.risk_score);
getDetails(REFERENCE_URLS_DETAILS).should((details) => {
expect(removeExternalLinkText(details.text())).equal(expectedUrls);
});
getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives);
getDetails(MITRE_ATTACK_DETAILS).should((mitre) => {
expect(removeExternalLinkText(mitre.text())).equal(expectedMitre);
});
getDetails(TAGS_DETAILS).should('have.text', expectedTags);
});
getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives);
getDetails(MITRE_ATTACK_DETAILS).should((mitre) => {
expect(removeExternalLinkText(mitre.text())).equal(expectedMitre);
cy.get(DEFINITION_DETAILS).within(() => {
getDetails(ANOMALY_SCORE_DETAILS).should('have.text', mlRule.anomaly_threshold);
getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning');
getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None');
const machineLearningJobsArray = isArray(mlRule.machine_learning_job_id)
? mlRule.machine_learning_job_id
: [mlRule.machine_learning_job_id];
// With the #1912 ML rule improvement changes we enable jobs on rule creation.
// Though, in cypress jobs enabling does not work reliably and job can be started or stopped.
// Thus, we disable next check till we fix the issue with enabling jobs in cypress.
// Relevant ticket: https://github.com/elastic/security-team/issues/5389
// cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'StoppedStopped');
cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningJobsArray.join(''));
});
cy.get(SCHEDULE_DETAILS).within(() => {
getDetails(RUNS_EVERY_DETAILS).should('have.text', `${mlRule.interval}`);
const humanizedDuration = getHumanizedDuration(
mlRule.from ?? 'now-6m',
mlRule.interval ?? '5m'
);
getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`);
});
getDetails(TAGS_DETAILS).should('have.text', expectedTags);
});
cy.get(DEFINITION_DETAILS).within(() => {
getDetails(ANOMALY_SCORE_DETAILS).should('have.text', mlRule.anomaly_threshold);
getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning');
getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None');
const machineLearningJobsArray = isArray(mlRule.machine_learning_job_id)
? mlRule.machine_learning_job_id
: [mlRule.machine_learning_job_id];
// With the #1912 ML rule improvement changes we enable jobs on rule creation.
// Though, in cypress jobs enabling does not work reliably and job can be started or stopped.
// Thus, we disable next check till we fix the issue with enabling jobs in cypress.
// Relevant ticket: https://github.com/elastic/security-team/issues/5389
// cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'StoppedStopped');
cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningJobsArray.join(''));
});
cy.get(SCHEDULE_DETAILS).within(() => {
getDetails(RUNS_EVERY_DETAILS).should('have.text', `${mlRule.interval}`);
const humanizedDuration = getHumanizedDuration(
mlRule.from ?? 'now-6m',
mlRule.interval ?? '5m'
);
getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`);
});
});
});
}
);

19
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts
View file

@ -45,7 +45,7 @@ import {
} from '../../../screens/rule_details';
import { createTimeline } from '../../../tasks/api_calls/timelines';
import { cleanKibana, deleteAlertsAndRules, deleteConnectors } from '../../../tasks/common';
import { deleteAlertsAndRules, deleteConnectors } from '../../../tasks/common';
import { login } from '../../../tasks/login';
import { visit } from '../../../tasks/navigation';
import { ruleDetailsUrl } from '../../../urls/rule_details';
@ -53,15 +53,11 @@ import { ruleDetailsUrl } from '../../../urls/rule_details';
// This test is meant to test all common aspects of the rule details page that should function
// the same regardless of rule type. For any rule type specific functionalities, please include
// them in the relevant /rule_details/[RULE_TYPE].cy.ts test.
describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
});
describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, function () {
beforeEach(() => {
login();
deleteAlertsAndRules();
deleteConnectors();
login();
createTimeline(getTimeline()).then((response) => {
createRule({
...getNewRule({
@ -93,12 +89,13 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => {
],
}),
}).then((rule) => {
visit(ruleDetailsUrl(rule.body.id));
cy.wrap(rule.body.id).as('ruleId');
});
});
});
it('Only modifies rule active status on enable/disable', () => {
it('Only modifies rule active status on enable/disable', function () {
visit(ruleDetailsUrl(this.ruleId));
cy.get(RULE_NAME_HEADER).should('contain', ruleFields.ruleName);
cy.intercept('POST', '/api/detection_engine/rules/_bulk_action?dry_run=false').as(
@ -117,6 +114,7 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => {
});
it('Displays rule details', function () {
visit(ruleDetailsUrl(this.ruleId));
cy.get(RULE_NAME_HEADER).should('contain', ruleFields.ruleName);
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', ruleFields.ruleDescription);
cy.get(ABOUT_DETAILS).within(() => {
@ -161,7 +159,8 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => {
});
});
it('Deletes one rule from detail page', () => {
it('Deletes one rule from detail page', function () {
visit(ruleDetailsUrl(this.ruleId));
cy.intercept('POST', '/api/detection_engine/rules/_bulk_delete').as('deleteRule');
deleteRuleFromDetailsPage();

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/custom_query_rule.cy.ts
View file

@ -54,7 +54,7 @@ import { saveEditedRule, visitEditRulePage } from '../../../tasks/edit_rule';
import { login } from '../../../tasks/login';
import { getDetails } from '../../../tasks/rule_details';
describe('Custom query rules', { tags: ['@ess', '@serverless'] }, () => {
describe('Custom query rules', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => {
const rule = getEditedRule();
const expectedEditedtags = rule.tags?.join('');
const expectedEditedIndexPatterns = rule.index;

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts
View file

@ -45,7 +45,7 @@ import {
waitForPageToBeLoaded,
} from '../../../../tasks/rule_details';
describe('Related integrations', { tags: ['@ess', '@serverless'] }, () => {
describe('Related integrations', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => {
const DATA_STREAM_NAME = 'logs-related-integrations-test';
const PREBUILT_RULE_NAME = 'Prebuilt rule with related integrations';
const RULE_RELATED_INTEGRATIONS: IntegrationDefinition[] = [

82
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts
View file

@ -185,7 +185,7 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () =>
cy.get(APPLY_TIMELINE_RULE_BULK_MENU_ITEM).should('be.disabled');
});
it('Only prebuilt rules selected', () => {
it('Only prebuilt rules selected', { tags: ['@brokenInServerlessQA'] }, () => {
createAndInstallMockedPrebuiltRules(PREBUILT_RULES);
// select Elastic(prebuilt) rules, check if we can't proceed further, as Elastic rules are not editable
@ -203,47 +203,55 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () =>
});
});
it('Prebuilt and custom rules selected: user proceeds with custom rules editing', () => {
getRulesManagementTableRows().then((existedRulesRows) => {
it(
'Prebuilt and custom rules selected: user proceeds with custom rules editing',
{ tags: ['@brokenInServerlessQA'] },
() => {
getRulesManagementTableRows().then((existedRulesRows) => {
createAndInstallMockedPrebuiltRules(PREBUILT_RULES);
// modal window should show how many rules can be edit, how many not
selectAllRules();
clickAddTagsMenuItem();
waitForMixedRulesBulkEditModal(existedRulesRows.length);
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
checkPrebuiltRulesCannotBeModified(availablePrebuiltRulesCount);
});
// user can proceed with custom rule editing
cy.get(MODAL_CONFIRMATION_BTN)
.should('have.text', `Edit ${existedRulesRows.length} custom rules`)
.click();
// action should finish
typeTags(['test-tag']);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: existedRulesRows.length });
});
}
);
it(
'Prebuilt and custom rules selected: user cancels action',
{ tags: ['@brokenInServerlessQA'] },
() => {
createAndInstallMockedPrebuiltRules(PREBUILT_RULES);
// modal window should show how many rules can be edit, how many not
selectAllRules();
clickAddTagsMenuItem();
getRulesManagementTableRows().then((rows) => {
// modal window should show how many rules can be edit, how many not
selectAllRules();
clickAddTagsMenuItem();
waitForMixedRulesBulkEditModal(rows.length);
waitForMixedRulesBulkEditModal(existedRulesRows.length);
checkPrebuiltRulesCannotBeModified(PREBUILT_RULES.length);
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
checkPrebuiltRulesCannotBeModified(availablePrebuiltRulesCount);
// user cancels action and modal disappears
cancelConfirmationModal();
});
// user can proceed with custom rule editing
cy.get(MODAL_CONFIRMATION_BTN)
.should('have.text', `Edit ${existedRulesRows.length} custom rules`)
.click();
// action should finish
typeTags(['test-tag']);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: existedRulesRows.length });
});
});
it('Prebuilt and custom rules selected: user cancels action', () => {
createAndInstallMockedPrebuiltRules(PREBUILT_RULES);
getRulesManagementTableRows().then((rows) => {
// modal window should show how many rules can be edit, how many not
selectAllRules();
clickAddTagsMenuItem();
waitForMixedRulesBulkEditModal(rows.length);
checkPrebuiltRulesCannotBeModified(PREBUILT_RULES.length);
// user cancels action and modal disappears
cancelConfirmationModal();
});
});
}
);
it('should not lose rules selection after edit action', () => {
const rulesToUpdate = [RULE_NAME, 'New EQL Rule', 'New Terms Rule'] as const;

344
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts
View file

@ -74,78 +74,80 @@ const ruleNameToAssert = 'Custom rule name with actions';
const expectedExistingSlackMessage = 'Existing slack action';
const expectedSlackMessage = 'Slack action test message';
describe('Detection rules, bulk edit of rule actions', { tags: ['@ess', '@serverless'] }, () => {
beforeEach(() => {
cleanKibana();
login();
deleteAlertsAndRules();
deleteConnectors();
describe(
'Detection rules, bulk edit of rule actions',
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
beforeEach(() => {
cleanKibana();
login();
deleteAlertsAndRules();
deleteConnectors();
createSlackConnector().then(({ body }) => {
const actions: RuleActionArray = [
{
id: body.id,
action_type_id: '.slack',
group: 'default',
params: {
message: expectedExistingSlackMessage,
createSlackConnector().then(({ body }) => {
const actions: RuleActionArray = [
{
id: body.id,
action_type_id: '.slack',
group: 'default',
params: {
message: expectedExistingSlackMessage,
},
frequency: {
summary: true,
throttle: null,
notifyWhen: 'onActiveAlert',
},
},
frequency: {
summary: true,
throttle: null,
notifyWhen: 'onActiveAlert',
},
},
];
];
createRule(
getNewRule({
rule_id: '1',
name: ruleNameToAssert,
max_signals: 500,
actions,
enabled: false,
})
);
});
createRule(getEqlRule({ rule_id: '2', name: 'New EQL Rule', enabled: false }));
createRule(
getNewRule({
rule_id: '1',
name: ruleNameToAssert,
max_signals: 500,
actions,
getMachineLearningRule({ rule_id: '3', name: 'New ML Rule Test', enabled: false })
);
createRule(
getNewThreatIndicatorRule({
rule_id: '4',
name: 'Threat Indicator Rule Test',
enabled: false,
})
);
createRule(getNewThresholdRule({ rule_id: '5', name: 'Threshold Rule', enabled: false }));
createRule(getNewTermsRule({ rule_id: '6', name: 'New Terms Rule', enabled: false }));
createRule(
getNewRule({ saved_id: 'mocked', rule_id: '7', name: 'New Rule Test', enabled: false })
);
createSlackConnector();
// Prevent prebuilt rules package installation and mock two prebuilt rules
preventPrebuiltRulesPackageInstallation();
const RULE_1 = createRuleAssetSavedObject({
name: 'Test rule 1',
rule_id: 'rule_1',
});
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
});
createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]);
});
createRule(getEqlRule({ rule_id: '2', name: 'New EQL Rule', enabled: false }));
createRule(getMachineLearningRule({ rule_id: '3', name: 'New ML Rule Test', enabled: false }));
createRule(
getNewThreatIndicatorRule({
rule_id: '4',
name: 'Threat Indicator Rule Test',
enabled: false,
})
);
createRule(getNewThresholdRule({ rule_id: '5', name: 'Threshold Rule', enabled: false }));
createRule(getNewTermsRule({ rule_id: '6', name: 'New Terms Rule', enabled: false }));
createRule(
getNewRule({ saved_id: 'mocked', rule_id: '7', name: 'New Rule Test', enabled: false })
);
createSlackConnector();
// Prevent prebuilt rules package installation and mock two prebuilt rules
preventPrebuiltRulesPackageInstallation();
const RULE_1 = createRuleAssetSavedObject({
name: 'Test rule 1',
rule_id: 'rule_1',
});
const RULE_2 = createRuleAssetSavedObject({
name: 'Test rule 2',
rule_id: 'rule_2',
});
createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]);
});
context('Restricted action privileges', () => {
it(
"User with no privileges can't add rule actions",
{ tags: ['@ess', '@skipInServerless'] },
() => {
context('Restricted action privileges', () => {
it("User with no privileges can't add rule actions", () => {
login(ROLES.hunter_no_actions);
visitRulesManagementTable(ROLES.hunter_no_actions);
@ -167,129 +169,129 @@ describe('Detection rules, bulk edit of rule actions', { tags: ['@ess', '@server
openBulkActionsMenu();
cy.get(ADD_RULE_ACTIONS_MENU_ITEM).should('be.disabled');
}
);
});
context('All actions privileges', () => {
beforeEach(() => {
login();
visitRulesManagementTable();
disableAutoRefresh();
expectManagementTableRules([
ruleNameToAssert,
'New EQL Rule',
'New ML Rule Test',
'Threat Indicator Rule Test',
'Threshold Rule',
'New Terms Rule',
'New Rule Test',
'Test rule 1',
'Test rule 2',
]);
});
});
it('Add a rule action to rules (existing connector)', () => {
const expectedActionFrequency: RuleActionCustomFrequency = {
throttle: 1,
throttleUnit: 'd',
};
context('All actions privileges', () => {
beforeEach(() => {
login();
visitRulesManagementTable();
disableAutoRefresh();
excessivelyInstallAllPrebuiltRules();
expectManagementTableRules([
ruleNameToAssert,
'New EQL Rule',
'New ML Rule Test',
'Threat Indicator Rule Test',
'Threshold Rule',
'New Terms Rule',
'New Rule Test',
'Test rule 1',
'Test rule 2',
]);
});
getRulesManagementTableRows().then((rows) => {
// select both custom and prebuilt rules
selectAllRules();
it('Add a rule action to rules (existing connector)', () => {
const expectedActionFrequency: RuleActionCustomFrequency = {
throttle: 1,
throttleUnit: 'd',
};
excessivelyInstallAllPrebuiltRules();
getRulesManagementTableRows().then((rows) => {
// select both custom and prebuilt rules
selectAllRules();
openBulkEditRuleActionsForm();
// ensure rule actions info callout displayed on the form
cy.get(RULES_BULK_EDIT_ACTIONS_INFO).should('be.visible');
addSlackRuleAction(expectedSlackMessage);
pickSummaryOfAlertsOption();
pickCustomFrequencyOption(expectedActionFrequency);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: rows.length });
// check if rule has been updated
goToEditRuleActionsSettingsOf(ruleNameToAssert);
assertSelectedSummaryOfAlertsOption();
assertSelectedCustomFrequencyOption(expectedActionFrequency, 1);
assertSlackRuleAction(expectedExistingSlackMessage, 0);
assertSlackRuleAction(expectedSlackMessage, 1);
// ensure there is no third action
cy.get(actionFormSelector(2)).should('not.exist');
});
});
it('Overwrite rule actions in rules', () => {
excessivelyInstallAllPrebuiltRules();
getRulesManagementTableRows().then((rows) => {
// select both custom and prebuilt rules
selectAllRules();
openBulkEditRuleActionsForm();
addSlackRuleAction(expectedSlackMessage);
pickSummaryOfAlertsOption();
pickPerRuleRunFrequencyOption();
// check overwrite box, ensure warning is displayed
checkOverwriteRuleActionsCheckbox();
cy.get(RULES_BULK_EDIT_ACTIONS_WARNING).contains(
`You're about to overwrite rule actions for ${rows.length} selected rules`
);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: rows.length });
// check if rule has been updated
goToEditRuleActionsSettingsOf(ruleNameToAssert);
assertSelectedSummaryOfAlertsOption();
assertSelectedPerRuleRunFrequencyOption();
assertSlackRuleAction(expectedSlackMessage);
// ensure existing action was overwritten
cy.get(actionFormSelector(1)).should('not.exist');
});
});
it('Add a rule action to rules (new connector)', () => {
const rulesToSelect = [
ruleNameToAssert,
'New EQL Rule',
'New ML Rule Test',
'Threat Indicator Rule Test',
'Threshold Rule',
'New Terms Rule',
'New Rule Test',
] as const;
const expectedActionFrequency: RuleActionCustomFrequency = {
throttle: 2,
throttleUnit: 'h',
};
const expectedEmail = 'test@example.com';
const expectedSubject = 'Subject';
selectRulesByName(rulesToSelect);
openBulkEditRuleActionsForm();
// ensure rule actions info callout displayed on the form
cy.get(RULES_BULK_EDIT_ACTIONS_INFO).should('be.visible');
addSlackRuleAction(expectedSlackMessage);
addEmailConnectorAndRuleAction(expectedEmail, expectedSubject);
pickSummaryOfAlertsOption();
pickCustomFrequencyOption(expectedActionFrequency);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: rows.length });
waitForBulkEditActionToFinish({ updatedCount: rulesToSelect.length });
// check if rule has been updated
goToEditRuleActionsSettingsOf(ruleNameToAssert);
assertSelectedSummaryOfAlertsOption();
assertSelectedCustomFrequencyOption(expectedActionFrequency, 1);
assertSlackRuleAction(expectedExistingSlackMessage, 0);
assertSlackRuleAction(expectedSlackMessage, 1);
// ensure there is no third action
cy.get(actionFormSelector(2)).should('not.exist');
assertEmailRuleAction(expectedEmail, expectedSubject);
});
});
it('Overwrite rule actions in rules', () => {
excessivelyInstallAllPrebuiltRules();
getRulesManagementTableRows().then((rows) => {
// select both custom and prebuilt rules
selectAllRules();
openBulkEditRuleActionsForm();
addSlackRuleAction(expectedSlackMessage);
pickSummaryOfAlertsOption();
pickPerRuleRunFrequencyOption();
// check overwrite box, ensure warning is displayed
checkOverwriteRuleActionsCheckbox();
cy.get(RULES_BULK_EDIT_ACTIONS_WARNING).contains(
`You're about to overwrite rule actions for ${rows.length} selected rules`
);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: rows.length });
// check if rule has been updated
goToEditRuleActionsSettingsOf(ruleNameToAssert);
assertSelectedSummaryOfAlertsOption();
assertSelectedPerRuleRunFrequencyOption();
assertSlackRuleAction(expectedSlackMessage);
// ensure existing action was overwritten
cy.get(actionFormSelector(1)).should('not.exist');
});
});
it('Add a rule action to rules (new connector)', () => {
const rulesToSelect = [
ruleNameToAssert,
'New EQL Rule',
'New ML Rule Test',
'Threat Indicator Rule Test',
'Threshold Rule',
'New Terms Rule',
'New Rule Test',
] as const;
const expectedActionFrequency: RuleActionCustomFrequency = {
throttle: 2,
throttleUnit: 'h',
};
const expectedEmail = 'test@example.com';
const expectedSubject = 'Subject';
selectRulesByName(rulesToSelect);
openBulkEditRuleActionsForm();
addEmailConnectorAndRuleAction(expectedEmail, expectedSubject);
pickSummaryOfAlertsOption();
pickCustomFrequencyOption(expectedActionFrequency);
submitBulkEditForm();
waitForBulkEditActionToFinish({ updatedCount: rulesToSelect.length });
// check if rule has been updated
goToEditRuleActionsSettingsOf(ruleNameToAssert);
assertSelectedSummaryOfAlertsOption();
assertSelectedCustomFrequencyOption(expectedActionFrequency, 1);
assertEmailRuleAction(expectedEmail, expectedSubject);
});
});
});
}
);

26
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts
View file

@ -100,19 +100,23 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => {
expectManagementTableRules(['Enabled rule to export']);
});
it('shows a modal saying that no rules can be exported if all the selected rules are prebuilt', function () {
createAndInstallMockedPrebuiltRules(prebuiltRules);
it(
'shows a modal saying that no rules can be exported if all the selected rules are prebuilt',
{ tags: ['@brokenInServerlessQA'] },
function () {
createAndInstallMockedPrebuiltRules(prebuiltRules);
filterByElasticRules();
selectAllRules();
bulkExportRules();
filterByElasticRules();
selectAllRules();
bulkExportRules();
cy.get(MODAL_CONFIRMATION_BODY).contains(
`${prebuiltRules.length} prebuilt Elastic rules (exporting prebuilt rules is not supported)`
);
});
cy.get(MODAL_CONFIRMATION_BODY).contains(
`${prebuiltRules.length} prebuilt Elastic rules (exporting prebuilt rules is not supported)`
);
}
);
it('exports only custom rules', function () {
it('exports only custom rules', { tags: ['@brokenInServerlessQA'] }, function () {
const expectedNumberCustomRulesToBeExported = 1;
createAndInstallMockedPrebuiltRules(prebuiltRules);
@ -141,7 +145,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => {
});
});
context('rules with exceptions', () => {
context('rules with exceptions', { tags: ['@brokenInServerlessQA'] }, () => {
beforeEach(() => {
deleteExceptionList(exceptionList.list_id, exceptionList.namespace_type);
// create rule with exceptions

2
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/snoozing/rule_snoozing.cy.ts
View file

@ -181,7 +181,7 @@ describe('rule snoozing', { tags: ['@ess', '@serverless'] }, () => {
deleteConnectors();
});
it('adds an action to a snoozed rule', () => {
it('adds an action to a snoozed rule', { tags: ['@brokenInServerlessQA'] }, () => {
createSnoozedRule(getNewRule({ name: 'Snoozed rule' })).then(({ body: rule }) => {
visitEditRulePage(rule.id);
goToActionsStepTab();

6
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_links.cy.ts
View file

@ -8,16 +8,12 @@
import { getNewRule } from '../../../../objects/rule';
import { RULES_MONITORING_TAB, RULE_NAME } from '../../../../screens/alerts_detection_rules';
import { createRule } from '../../../../tasks/api_calls/rules';
import { cleanKibana, deleteAlertsAndRules } from '../../../../tasks/common';
import { deleteAlertsAndRules } from '../../../../tasks/common';
import { login } from '../../../../tasks/login';
import { visit } from '../../../../tasks/navigation';
import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management';
describe('Rules table: links', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
});
beforeEach(() => {
login();
deleteAlertsAndRules();

122
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts
View file

@ -34,68 +34,72 @@ const RULE_2 = createRuleAssetSavedObject({
rule_id: 'rule_2',
});
describe('Rules table: selection', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
});
beforeEach(() => {
login();
/* Create and install two mock rules */
createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]);
visit(RULES_MANAGEMENT_URL);
waitForPrebuiltDetectionRulesToBeLoaded();
});
it('should correctly update the selection label when rules are individually selected and unselected', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
selectRulesByName(['Test rule 1', 'Test rule 2']);
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2');
unselectRulesByName(['Test rule 1', 'Test rule 2']);
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
});
it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
cy.get(SELECT_ALL_RULES_BTN).click();
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount);
describe(
'Rules table: selection',
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
before(() => {
cleanKibana();
});
// Un-select all rules via the Bulk Selection button from the Utility bar
cy.get(SELECT_ALL_RULES_BTN).click();
// Current selection should be 0 rules
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
// Bulk selection button should be back to displaying all rules
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount);
});
});
it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
cy.get(SELECT_ALL_RULES_BTN).click();
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount);
beforeEach(() => {
login();
/* Create and install two mock rules */
createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]);
visit(RULES_MANAGEMENT_URL);
waitForPrebuiltDetectionRulesToBeLoaded();
});
// Un-select all rules via the Un-select All checkbox from the table
cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click();
it('should correctly update the selection label when rules are individually selected and unselected', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
// Current selection should be 0 rules
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
// Bulk selection button should be back to displaying all rules
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount);
selectRulesByName(['Test rule 1', 'Test rule 2']);
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2');
unselectRulesByName(['Test rule 1', 'Test rule 2']);
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
});
});
});
it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
cy.get(SELECT_ALL_RULES_BTN).click();
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount);
});
// Un-select all rules via the Bulk Selection button from the Utility bar
cy.get(SELECT_ALL_RULES_BTN).click();
// Current selection should be 0 rules
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
// Bulk selection button should be back to displaying all rules
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount);
});
});
it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => {
waitForPrebuiltDetectionRulesToBeLoaded();
cy.get(SELECT_ALL_RULES_BTN).click();
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount);
});
// Un-select all rules via the Un-select All checkbox from the table
cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click();
// Current selection should be 0 rules
cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0');
// Bulk selection button should be back to displaying all rules
getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => {
cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount);
});
});
}
);

20
x-pack/test/security_solution_cypress/cypress/e2e/exceptions/entry/use_value_list.cy.ts
View file

@ -17,7 +17,6 @@ import {
visitRuleDetailsPage,
} from '../../../tasks/rule_details';
import { getNewRule } from '../../../objects/rule';
import { cleanKibana } from '../../../tasks/common';
import { login } from '../../../tasks/login';
import { visit } from '../../../tasks/navigation';
import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management';
@ -29,6 +28,7 @@ import {
deleteValueListsFile,
importValueList,
KNOWN_VALUE_LIST_FILES,
deleteValueLists,
} from '../../../tasks/lists';
import { createRule } from '../../../tasks/api_calls/rules';
import {
@ -48,11 +48,17 @@ const goToRulesAndOpenValueListModal = () => {
};
describe('Use Value list in exception entry', { tags: ['@ess', '@serverless'] }, () => {
beforeEach(() => {
cleanKibana();
login();
createListsIndex();
before(() => {
cy.task('esArchiverLoad', { archiveName: 'exceptions' });
});
after(() => {
cy.task('esArchiverUnload', 'exceptions');
});
beforeEach(() => {
login();
deleteValueLists([KNOWN_VALUE_LIST_FILES.TEXT]);
createListsIndex();
importValueList(KNOWN_VALUE_LIST_FILES.TEXT, 'keyword');
createRule(
@ -66,10 +72,6 @@ describe('Use Value list in exception entry', { tags: ['@ess', '@serverless'] },
).then((rule) => visitRuleDetailsPage(rule.body.id, { tab: 'rule_exceptions' }));
});
afterEach(() => {
cy.task('esArchiverUnload', 'exceptions');
});
it('Should use value list in exception entry, and validate deleting value list prompt', () => {
const ITEM_NAME = 'Exception item with value list';
const ITEM_FIELD = 'agent.name';

6
x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/connector_options.cy.ts
View file

@ -27,15 +27,9 @@ import {
import { goToCreateNewCase } from '../../../tasks/all_cases';
import { CASES_URL } from '../../../urls/navigation';
import { CONNECTOR_CARD_DETAILS, CONNECTOR_TITLE } from '../../../screens/case_details';
import { cleanKibana } from '../../../tasks/common';
// FLAKY: https://github.com/elastic/kibana/issues/165712
describe('Cases connector incident fields', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
login();
});
beforeEach(() => {
login();
cy.intercept('GET', '/api/cases/configure/connectors/_find', getMockConnectorsResponse());

2
x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts
View file

@ -41,7 +41,6 @@ import { OVERVIEW_CASE_DESCRIPTION, OVERVIEW_CASE_NAME } from '../../../screens/
import { goToCaseDetails, goToCreateNewCase } from '../../../tasks/all_cases';
import { createTimeline } from '../../../tasks/api_calls/timelines';
import { openCaseTimeline } from '../../../tasks/case_details';
import { cleanKibana } from '../../../tasks/common';
import {
attachTimeline,
backToCases,
@ -58,7 +57,6 @@ import { ELASTICSEARCH_USERNAME } from '../../../env_var_names_constants';
// Tracked by https://github.com/elastic/security-team/issues/7696
describe('Cases', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
createTimeline(getCase1().timeline).then((response) =>
cy
.wrap({

4
x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts
View file

@ -11,7 +11,7 @@ import { visitWithTimeRange } from '../../../tasks/navigation';
import { ALERTS_URL, ENTITY_ANALYTICS_URL } from '../../../urls/navigation';
import { cleanKibana, deleteAlertsAndRules } from '../../../tasks/common';
import { deleteAlertsAndRules } from '../../../tasks/common';
import {
ANOMALIES_TABLE,
@ -67,8 +67,6 @@ const OLDEST_DATE = moment('2019-01-19T16:22:56.217Z').format(DATE_FORMAT);
describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
login();
deleteRiskEngineConfiguration();
});

76
x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_charts.cy.ts
View file

@ -25,46 +25,50 @@ import {
} from '../../../screens/search_bar';
import { TOASTER } from '../../../screens/alerts_detection_rules';
describe('Histogram legend hover actions', { tags: ['@ess', '@serverless'] }, () => {
const ruleConfigs = getNewRule();
describe(
'Histogram legend hover actions',
{ tags: ['@ess', '@serverless', '@brokenInServerlessQA'] },
() => {
const ruleConfigs = getNewRule();
before(() => {
cleanKibana();
});
before(() => {
cleanKibana();
});
beforeEach(() => {
login();
createRule(getNewRule({ rule_id: 'new custom rule' }));
visitWithTimeRange(ALERTS_URL);
selectAlertsHistogram();
});
beforeEach(() => {
login();
createRule(getNewRule({ rule_id: 'new custom rule' }));
visitWithTimeRange(ALERTS_URL);
selectAlertsHistogram();
});
it('Filter in/out should add a filter to KQL bar', function () {
const expectedNumberOfAlerts = 2;
clickAlertsHistogramLegend();
clickAlertsHistogramLegendFilterFor(ruleConfigs.name);
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should(
'have.text',
`kibana.alert.rule.name: ${ruleConfigs.name}`
);
cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`);
it('Filter in/out should add a filter to KQL bar', function () {
const expectedNumberOfAlerts = 2;
clickAlertsHistogramLegend();
clickAlertsHistogramLegendFilterFor(ruleConfigs.name);
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should(
'have.text',
`kibana.alert.rule.name: ${ruleConfigs.name}`
);
cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`);
clickAlertsHistogramLegend();
clickAlertsHistogramLegendFilterOut(ruleConfigs.name);
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should(
'have.text',
`NOT kibana.alert.rule.name: ${ruleConfigs.name}`
);
cy.get(ALERTS_COUNT).should('not.exist');
clickAlertsHistogramLegend();
clickAlertsHistogramLegendFilterOut(ruleConfigs.name);
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should(
'have.text',
`NOT kibana.alert.rule.name: ${ruleConfigs.name}`
);
cy.get(ALERTS_COUNT).should('not.exist');
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click();
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist');
});
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click();
cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist');
});
it('Add To Timeline', function () {
clickAlertsHistogramLegend();
clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name);
it('Add To Timeline', function () {
clickAlertsHistogramLegend();
clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name);
cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`);
});
});
cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`);
});
}
);

2
x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts
View file

@ -182,7 +182,7 @@ describe('Alert details flyout', { tags: ['@ess', '@serverless'] }, () => {
});
});
describe('Localstorage management', () => {
describe('Localstorage management', { tags: ['@brokenInServerlessQA'] }, () => {
const ARCHIVED_RULE_ID = '7015a3e2-e4ea-11ed-8c11-49608884878f';
const ARCHIVED_RULE_NAME = 'Endpoint Security';

2
x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_threat_intelligence_tab.cy.ts
View file

@ -10,7 +10,6 @@ import { getNewRule } from '../../../../objects/rule';
import { expandDocumentDetailsExpandableFlyoutLeftSection } from '../../../../tasks/expandable_flyout/alert_details_right_panel';
import { expandFirstAlertExpandableFlyout } from '../../../../tasks/expandable_flyout/common';
import { INDICATOR_MATCH_ENRICHMENT_SECTION } from '../../../../screens/alerts_details';
import { cleanKibana } from '../../../../tasks/common';
import { waitForAlertsToPopulate } from '../../../../tasks/create_new_rule';
import { login } from '../../../../tasks/login';
import { visit } from '../../../../tasks/navigation';
@ -28,7 +27,6 @@ describe(
{ tags: ['@ess', '@serverless'] },
() => {
beforeEach(() => {
cleanKibana();
login();
createRule(getNewRule());
visit(ALERTS_URL);

2
x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/ransomware_prevention.cy.ts
View file

@ -14,11 +14,9 @@ import { ALERTS_HISTOGRAM_SERIES, ALERT_RULE_NAME, MESSAGE } from '../../../scre
import { TIMELINE_QUERY, TIMELINE_VIEW_IN_ANALYZER } from '../../../screens/timeline';
import { selectAlertsHistogram } from '../../../tasks/alerts';
import { createTimeline } from '../../../tasks/timelines';
import { cleanKibana } from '../../../tasks/common';
describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
cy.task('esArchiverLoad', {
archiveName: 'ransomware_prevention',
useCreate: true,

3
x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/export.cy.ts
View file

@ -15,15 +15,12 @@ import {
import { TIMELINE_TEMPLATES_URL } from '../../../urls/navigation';
import { createTimelineTemplate } from '../../../tasks/api_calls/timelines';
import { cleanKibana } from '../../../tasks/common';
import { searchByTitle } from '../../../tasks/table_pagination';
// FLAKY: https://github.com/elastic/kibana/issues/165760
// FLAKY: https://github.com/elastic/kibana/issues/165645
describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
createTimelineTemplate(getTimelineTemplate()).then((response) => {
cy.wrap(response).as('templateResponse');
cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('templateId');

6
x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/export.cy.ts
View file

@ -19,13 +19,12 @@ import { TOASTER } from '../../../screens/alerts_detection_rules';
import { TIMELINE_CHECKBOX } from '../../../screens/timelines';
import { createTimeline } from '../../../tasks/api_calls/timelines';
import { expectedExportedTimeline, getTimeline } from '../../../objects/timeline';
import { cleanKibana } from '../../../tasks/common';
import { deleteTimelines } from '../../../tasks/common';
// FLAKY: https://github.com/elastic/kibana/issues/165744
describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
login();
deleteTimelines();
cy.intercept({
method: 'POST',
path: '/api/timeline/_export?file_name=timelines_export.ndjson',
@ -38,7 +37,6 @@ describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => {
cy.wrap(response).as('timelineResponse2');
cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('timelineId2');
});
visit(TIMELINES_URL);
});
beforeEach(() => {

11
x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/open_timeline.cy.ts
View file

@ -22,8 +22,6 @@ import { addNoteToTimeline } from '../../../tasks/api_calls/notes';
import { createTimeline } from '../../../tasks/api_calls/timelines';
import { cleanKibana } from '../../../tasks/common';
import { login } from '../../../tasks/login';
import { visit } from '../../../tasks/navigation';
import {
@ -39,10 +37,8 @@ import { TIMELINES_URL } from '../../../urls/navigation';
describe('Open timeline', { tags: ['@serverless', '@ess'] }, () => {
describe('Open timeline modal', () => {
before(function () {
cleanKibana();
login();
visit(TIMELINES_URL);
createTimeline(getTimeline())
.then((response) => response.body.data.persistTimeline.timeline.savedObjectId)
.then((timelineId: string) => {
@ -63,14 +59,9 @@ describe('Open timeline', { tags: ['@serverless', '@ess'] }, () => {
});
});
beforeEach(function () {
login();
visit(TIMELINES_URL);
it('should display timeline info', function () {
openTimelineFromSettings();
openTimelineById(this.timelineId);
});
it('should display timeline info', () => {
cy.get(OPEN_TIMELINE_MODAL).should('be.visible');
cy.contains(getTimeline().title).should('exist');
cy.get(TIMELINES_DESCRIPTION).last().should('have.text', getTimeline().description);

6
x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/row_renderers.cy.ts
View file

@ -15,7 +15,7 @@ import {
TIMELINE_ROW_RENDERERS_SURICATA_SIGNATURE_TOOLTIP,
TIMELINE_ROW_RENDERERS_SURICATA_LINK_TOOLTIP,
} from '../../../screens/timeline';
import { cleanKibana, deleteTimelines, waitForWelcomePanelToBeLoaded } from '../../../tasks/common';
import { deleteTimelines, waitForWelcomePanelToBeLoaded } from '../../../tasks/common';
import { waitForAllHostsToBeLoaded } from '../../../tasks/hosts/all_hosts';
import { login } from '../../../tasks/login';
@ -26,10 +26,6 @@ import { populateTimeline } from '../../../tasks/timeline';
import { hostsUrl } from '../../../urls/navigation';
describe('Row renderers', { tags: ['@ess', '@serverless'] }, () => {
before(() => {
cleanKibana();
});
beforeEach(() => {
deleteTimelines();
login();

8
x-pack/test/security_solution_cypress/cypress/objects/timeline.ts
View file

@ -108,9 +108,9 @@ export const expectedExportedTimelineTemplate = (
templateTimelineVersion: 1,
timelineType: 'template',
created: timelineTemplateBody.created,
createdBy: 'system_indices_superuser',
createdBy: Cypress.env('ELASTICSEARCH_USERNAME'),
updated: timelineTemplateBody.updated,
updatedBy: 'system_indices_superuser',
updatedBy: Cypress.env('ELASTICSEARCH_USERNAME'),
sort: [],
eventNotes: [],
globalNotes: [],
@ -143,9 +143,9 @@ export const expectedExportedTimeline = (timelineResponse: Cypress.Response<Time
description: timelineBody.description,
title: timelineBody.title,
created: timelineBody.created,
createdBy: 'system_indices_superuser',
createdBy: Cypress.env('ELASTICSEARCH_USERNAME'),
updated: timelineBody.updated,
updatedBy: 'system_indices_superuser',
updatedBy: Cypress.env('ELASTICSEARCH_USERNAME'),
timelineType: 'default',
sort: [],
eventNotes: [],

6
x-pack/test/security_solution_cypress/cypress/tasks/api_calls/timelines.ts
View file

@ -53,7 +53,11 @@ export const createTimeline = (timeline: CompleteTimeline) =>
: {}),
},
},
headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' },
headers: {
'kbn-xsrf': 'cypress-creds',
'x-elastic-internal-origin': 'security-solution',
'elastic-api-version': '2023-10-31',
},
});
export const createTimelineTemplate = (timeline: CompleteTimeline) =>

7
x-pack/test/security_solution_cypress/package.json
View file

@ -23,10 +23,13 @@
"cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'",
"cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/exceptions/shared_exception_lists_management/**/*.cy.ts'",
"cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true",
"cypress:run:qa:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless_qa.config.ts --env CLOUD_SERVERLESS=true",
"cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'",
"cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'",
"cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5",
"cypress:burn:serverless": "yarn cypress:serverless --env burn=5"
"cypress:burn:serverless": "yarn cypress:serverless --env burn=2",
"cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts",
"cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'",
"cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'",
"cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'"
}
}