github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.12] [Security Solution][Detection Engine] fixes flaky ES|QL tests (#173251) (#174099)

Browse source 
# Backport

This will backport the following commits from `main` to `8.12`:
- [[Security Solution][Detection Engine] fixes flaky ES|QL tests
(#173251)](https://github.com/elastic/kibana/pull/173251)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Vitalii
Dmyterko","email":"92328789+vitaliidm@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-01-02T16:00:08Z","message":"[Security
Solution][Detection Engine] fixes flaky ES|QL tests (#173251)\n\n##
Summary\r\n\r\n- fixes flaky ES|QL
tests\r\nhttps://github.com/elastic/kibana/issues/173006\r\n- the reason
is an issue in cypress
itself\r\nhttps://github.com/cypress-io/cypress/issues/22113. It doesn't
have a\r\nfix yet, but I tried some suggested workaround, as
calling\r\n`uncaught:exception` handler after page is opened, so it is
registered\r\nin the same origin. Which seemed to help, since there was
no failures in\r\n300 runs. Usual rate of flakiness before the fix was
3-7 failures per\r\n100 runs. Hopefully, it will work and we won't see
any errors in future.\r\n\r\n\r\n### Checklist\r\n\r\nDelete any items
that are not applicable to this PR.\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n- 100 runs for all Detection
tests:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4625\r\n-
2 x 100 for just failing
ones:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4620,\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4605\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"b365051dc24156d810004917efa64dbcbaa64a59","branchLabelMapping":{"^v8.13.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Detections
and Resp","Team: SecuritySolution","backport:prev-minor","Team:Detection
Engine","v8.12.0","v8.13.0"],"title":"[Security Solution][Detection
Engine] fixes flaky ES|QL
tests","number":173251,"url":"https://github.com/elastic/kibana/pull/173251","mergeCommit":{"message":"[Security
Solution][Detection Engine] fixes flaky ES|QL tests (#173251)\n\n##
Summary\r\n\r\n- fixes flaky ES|QL
tests\r\nhttps://github.com/elastic/kibana/issues/173006\r\n- the reason
is an issue in cypress
itself\r\nhttps://github.com/cypress-io/cypress/issues/22113. It doesn't
have a\r\nfix yet, but I tried some suggested workaround, as
calling\r\n`uncaught:exception` handler after page is opened, so it is
registered\r\nin the same origin. Which seemed to help, since there was
no failures in\r\n300 runs. Usual rate of flakiness before the fix was
3-7 failures per\r\n100 runs. Hopefully, it will work and we won't see
any errors in future.\r\n\r\n\r\n### Checklist\r\n\r\nDelete any items
that are not applicable to this PR.\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n- 100 runs for all Detection
tests:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4625\r\n-
2 x 100 for just failing
ones:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4620,\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4605\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"b365051dc24156d810004917efa64dbcbaa64a59"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.12","label":"v8.12.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.13.0","branchLabelMappingKey":"^v8.13.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/173251","number":173251,"mergeCommit":{"message":"[Security
Solution][Detection Engine] fixes flaky ES|QL tests (#173251)\n\n##
Summary\r\n\r\n- fixes flaky ES|QL
tests\r\nhttps://github.com/elastic/kibana/issues/173006\r\n- the reason
is an issue in cypress
itself\r\nhttps://github.com/cypress-io/cypress/issues/22113. It doesn't
have a\r\nfix yet, but I tried some suggested workaround, as
calling\r\n`uncaught:exception` handler after page is opened, so it is
registered\r\nin the same origin. Which seemed to help, since there was
no failures in\r\n300 runs. Usual rate of flakiness before the fix was
3-7 failures per\r\n100 runs. Hopefully, it will work and we won't see
any errors in future.\r\n\r\n\r\n### Checklist\r\n\r\nDelete any items
that are not applicable to this PR.\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n- 100 runs for all Detection
tests:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4625\r\n-
2 x 100 for just failing
ones:\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4620,\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4605\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"b365051dc24156d810004917efa64dbcbaa64a59"}}]}]
BACKPORT-->

Co-authored-by: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com>
This commit is contained in:
Kibana Machine 2024-01-02 12:18:46 -05:00 • committed by GitHub
parent d46c147510
commit 7b18537c60
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 45 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

41
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts
View file

@ -20,6 +20,7 @@ import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details';
import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules';
import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common';
import {
expandEsqlQueryBar,
fillAboutRuleAndContinue,
fillDefineEsqlRuleAndContinue,
fillScheduleRuleAndContinue,
@ -34,12 +35,22 @@ import { visit } from '../../../../tasks/navigation';
import { CREATE_RULE_URL } from '../../../../urls/navigation';
// https://github.com/cypress-io/cypress/issues/22113
// issue is inside monaco editor, used in ES|QL query input
// calling it after visiting page in each tests, seems fixes the issue
// the only other alternative is patching ResizeObserver, which is something I would like to avoid
const workaroundForResizeObserver = () =>
cy.on('uncaught:exception', (err) => {
if (err.message.includes('ResizeObserver loop limit exceeded')) {
return false;
}
});
describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
const rule = getEsqlRule();
const expectedNumberOfRules = 1;
// FLAKY: https://github.com/elastic/kibana/issues/172618
describe.skip('creation', () => {
describe('creation', () => {
beforeEach(() => {
deleteAlertsAndRules();
login();
@ -47,8 +58,10 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
it('creates an ES|QL rule', function () {
visit(CREATE_RULE_URL);
workaroundForResizeObserver();
selectEsqlRuleType();
expandEsqlQueryBar();
// ensures ES|QL rule in technical preview on create page
cy.get(ESQL_TYPE).contains('Technical Preview');
@ -73,8 +86,10 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
// this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API
it('creates an ES|QL rule and overrides its name', function () {
visit(CREATE_RULE_URL);
workaroundForResizeObserver();
selectEsqlRuleType();
expandEsqlQueryBar();
fillDefineEsqlRuleAndContinue(rule);
fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' });
@ -86,23 +101,26 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});
});
// FLAKY: https://github.com/elastic/kibana/issues/172881
describe.skip('ES|QL query validation', () => {
describe('ES|QL query validation', () => {
beforeEach(() => {
login();
visit(CREATE_RULE_URL);
});
it('shows error when ES|QL query is empty', function () {
selectEsqlRuleType();
workaroundForResizeObserver();
selectEsqlRuleType();
expandEsqlQueryBar();
getDefineContinueButton().click();
cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required');
});
it('proceeds further once invalid query is fixed', function () {
selectEsqlRuleType();
workaroundForResizeObserver();
selectEsqlRuleType();
expandEsqlQueryBar();
getDefineContinueButton().click();
cy.get(ESQL_QUERY_BAR).contains('required');
@ -115,9 +133,11 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});
it('shows error when non-aggregating ES|QL query does not [metadata] operator', function () {
workaroundForResizeObserver();
const invalidNonAggregatingQuery = 'from auditbeat* | limit 5';
selectEsqlRuleType();
expandEsqlQueryBar();
fillEsqlQueryBar(invalidNonAggregatingQuery);
getDefineContinueButton().click();
@ -127,11 +147,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});
it('shows error when non-aggregating ES|QL query does not return _id field', function () {
workaroundForResizeObserver();
const invalidNonAggregatingQuery =
'from auditbeat* [metadata _id, _version, _index] | keep agent.* | limit 5';
selectEsqlRuleType();
expandEsqlQueryBar();
fillEsqlQueryBar(invalidNonAggregatingQuery);
getDefineContinueButton().click();
@ -141,12 +163,13 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
});
it('shows error when ES|QL query is invalid', function () {
workaroundForResizeObserver();
const invalidEsqlQuery =
'from auditbeat* [metadata _id, _version, _index] | not_existing_operator';
visit(CREATE_RULE_URL);
selectEsqlRuleType();
expandEsqlQueryBar();
fillEsqlQueryBar(invalidEsqlQuery);
getDefineContinueButton().click();

9
x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts
View file

@ -9,7 +9,7 @@ import { getEsqlRule } from '../../../../objects/rule';
import { ESQL_QUERY_DETAILS, RULE_NAME_OVERRIDE_DETAILS } from '../../../../screens/rule_details';
import { ESQL_QUERY_BAR, ESQL_QUERY_BAR_EXPAND_BTN } from '../../../../screens/create_new_rule';
import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule';
import { createRule } from '../../../../tasks/api_calls/rules';
@ -18,6 +18,7 @@ import { getDetails } from '../../../../tasks/rule_details';
import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common';
import {
clearEsqlQueryBar,
expandEsqlQueryBar,
fillEsqlQueryBar,
fillOverrideEsqlRuleName,
goToAboutStepTab,
@ -44,8 +45,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => {
it('edits ES|QL rule and checks details page', () => {
visit(RULES_MANAGEMENT_URL);
editFirstRule();
// expands query bar, so query is not obscured on narrow screens
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
expandEsqlQueryBar();
// ensure once edit form opened, correct query is displayed in ES|QL input
cy.get(ESQL_QUERY_BAR).contains(rule.query);
@ -78,8 +78,7 @@ describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => {
visit(RULES_MANAGEMENT_URL);
editFirstRule();
// expands query bar, so query is not obscured on narrow screens
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
expandEsqlQueryBar();
// ensure once edit form opened, correct query is displayed in ES|QL input
cy.get(ESQL_QUERY_BAR).contains(rule.query);

9
x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts
View file

@ -55,6 +55,7 @@ import {
EQL_TYPE,
ESQL_TYPE,
ESQL_QUERY_BAR,
ESQL_QUERY_BAR_EXPAND_BTN,
ESQL_QUERY_BAR_INPUT_AREA,
FALSE_POSITIVES_INPUT,
IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK,
@ -537,6 +538,14 @@ export const fillEsqlQueryBar = (query: string) => {
cy.get(ESQL_QUERY_BAR_INPUT_AREA).type(query);
};
/**
* expands query bar, so query is not obscured on narrow screens
* and validation message is not covered by input menu tooltip
*/
export const expandEsqlQueryBar = () => {
cy.get(ESQL_QUERY_BAR_EXPAND_BTN).click();
};
export const fillDefineEsqlRuleAndContinue = (rule: EsqlRuleCreateProps) => {
cy.get(ESQL_QUERY_BAR).contains('ES|QL query');
fillEsqlQueryBar(rule.query);