[Security Solution] Fix flaky e2e tests for Related Integrations (#142134)

**Related to:** https://github.com/elastic/kibana/pull/142040, https://github.com/elastic/kibana/pull/135181

## Summary

- Fixes flakiness in Cypress tests for the "related integrations" feature.
- Unskips tests that have been skipped in https://github.com/elastic/kibana/pull/142040

x-pack/plugins/security_solution/cypress/e2e/detection_rules/related_integrations.cy.ts

@@ -5,47 +5,52 @@
  * 2.0.
  */
 
+import { DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation';
+
+import { FIELD } from '../../screens/alerts_details';
+import { INTEGRATIONS, INTEGRATIONS_STATUS } from '../../screens/rule_details';
 import {
   INTEGRATIONS_POPOVER,
   INTEGRATIONS_POPOVER_TITLE,
   RULE_NAME,
 } from '../../screens/alerts_detection_rules';
-import { INTEGRATIONS, INTEGRATIONS_STATUS } from '../../screens/rule_details';
-import {
-  enableRule,
-  goToTheRuleDetailsOf,
-  openIntegrationsPopover,
-  waitForRuleToChangeStatus,
-} from '../../tasks/alerts_detection_rules';
+
+import { cleanFleet } from '../../tasks/api_calls/fleet';
 import { importRule } from '../../tasks/api_calls/rules';
-import { cleanKibana, cleanPackages } from '../../tasks/common';
-import { login, visit } from '../../tasks/login';
-import { DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation';
-import { installAwsCloudFrontWithPolicy } from '../../tasks/integrations';
-import { expandFirstAlert } from '../../tasks/alerts';
-import { waitForAlertsToPopulate } from '../../tasks/create_new_rule';
-import { filterBy, openTable } from '../../tasks/alerts_details';
-import { FIELD } from '../../screens/alerts_details';
 import {
   disableRelatedIntegrations,
   enableRelatedIntegrations,
 } from '../../tasks/api_calls/kibana_advanced_settings';
 
-/*
- Note that the rule we are using for testing purposes has the following characteristics, changing that may affect the coverage.
+import { cleanKibana } from '../../tasks/common';
+import { login, visit } from '../../tasks/login';
+import { expandFirstAlert } from '../../tasks/alerts';
+import { filterBy, openTable } from '../../tasks/alerts_details';
+import { waitForAlertsToPopulate } from '../../tasks/create_new_rule';
+import { installAwsCloudFrontWithPolicy } from '../../tasks/integrations';
+import {
+  enableRule,
+  goToTheRuleDetailsOf,
+  openIntegrationsPopover,
+  waitForRulesTableToShow,
+  waitForRuleToChangeStatus,
+} from '../../tasks/alerts_detection_rules';
 
- - Single-integration
-   - Package: system
- - Multi-integration package 
-   - Package: aws
-   - Integration: cloudtrail
-   - Integration: cloudfront   
- - Not existing package:
-    - Package: unknown
- - Not existing integration & existing package:
-    - Package: aws
-    - Integration: unknown
- */
+/*
+Note that the rule we are using for testing purposes has the following characteristics, changing that may affect the coverage.
+
+- Single-integration
+  - Package: system
+- Multi-integration package
+  - Package: aws
+  - Integration: cloudtrail
+  - Integration: cloudfront
+- Not existing package:
+  - Package: unknown
+- Not existing integration & existing package:
+  - Package: aws
+  - Integration: unknown
+*/
 
 describe('Related integrations', () => {
   before(() => {
@@ -62,8 +67,12 @@ describe('Related integrations', () => {
     };
 
     before(() => {
-      cleanPackages();
+      cleanFleet();
+    });
+
+    beforeEach(() => {
       visit(DETECTIONS_RULE_MANAGEMENT_URL);
+      waitForRulesTableToShow();
     });
 
     it('should display a badge with the installed integrations on the rule management page', () => {
@@ -117,19 +126,24 @@ describe('Related integrations', () => {
       };
 
       before(() => {
-        cleanPackages();
-        installAwsCloudFrontWithPolicy();
-        visit(DETECTIONS_RULE_MANAGEMENT_URL);
+        cleanFleet().then(() => {
+          installAwsCloudFrontWithPolicy();
+        });
       });
 
-      it.skip('should display a badge with the installed integrations on the rule management page', () => {
+      beforeEach(() => {
+        visit(DETECTIONS_RULE_MANAGEMENT_URL);
+        waitForRulesTableToShow();
+      });
+
+      it('should display a badge with the installed integrations on the rule management page', () => {
         cy.get(INTEGRATIONS_POPOVER).should(
           'have.text',
           `${rule.enabledIntegrations}/${rule.integrations.length} integrations`
         );
       });
 
-      it.skip('should display a popover when clicking the badge with the installed integrations on the rule management page', () => {
+      it('should display a popover when clicking the badge with the installed integrations on the rule management page', () => {
         openIntegrationsPopover();
 
         cy.get(INTEGRATIONS_POPOVER_TITLE).should(
@@ -148,7 +162,7 @@ describe('Related integrations', () => {
         });
       });
 
-      it.skip('should display the integrations on the definition section', () => {
+      it('should display the integrations on the definition section', () => {
         goToTheRuleDetailsOf(rule.name);
 
         cy.get(INTEGRATIONS).should('have.length', rule.integrations.length);
@@ -169,7 +183,6 @@ describe('Related integrations', () => {
         const expectedRelatedIntegrationsText =
           '{"package":"system","version":"1.17.0"}{"package":"aws","integration":"cloudtrail","version":"1.17.0"}{"package":"aws","integration":"cloudfront","version":"1.17.0"}{"package":"aws","integration":"unknown","version":"1.17.0"}';
 
-        visit(DETECTIONS_RULE_MANAGEMENT_URL);
         enableRule(firstRule);
         waitForRuleToChangeStatus();
         goToTheRuleDetailsOf(rule.name);
@@ -193,15 +206,20 @@ describe('Related integrations', () => {
     };
 
     before(() => {
-      cleanPackages();
-      disableRelatedIntegrations();
-      visit(DETECTIONS_RULE_MANAGEMENT_URL);
+      cleanFleet().then(() => {
+        disableRelatedIntegrations();
+      });
     });
 
     after(() => {
       enableRelatedIntegrations();
     });
 
+    beforeEach(() => {
+      visit(DETECTIONS_RULE_MANAGEMENT_URL);
+      waitForRulesTableToShow();
+    });
+
     it('should not display a badge with the installed integrations on the rule management page', () => {
       cy.get(RULE_NAME).should('have.text', rule.name);
       cy.get(INTEGRATIONS).should('not.exist');

x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts

@@ -253,9 +253,37 @@ export const sortByEnabledRules = () => {
   cy.get(SORT_RULES_BTN).contains('Enabled').click({ force: true });
 };
 
+/**
+ * Because the Rule Management page is relatively slow, in order to avoid timeouts and flakiness,
+ * we almost always want to wait until the Rules table is "loaded" before we do anything with it.
+ *
+ * This task should be sufficient for the vast majority of the tests. It waits for the table
+ * to show up on the page, but doesn't wait until it is fully loaded and filled with rows.
+ *
+ * @example
+ * beforeEach(() => {
+ *   visit(DETECTIONS_RULE_MANAGEMENT_URL); // returns on "load" event, still lots of work to do
+ *   waitForRulesTableToShow(); // a lot has done in React and the table shows up on the page
+ * });
+ */
+export const waitForRulesTableToShow = () => {
+  // Wait up to 5 minutes for the table to show up as in CI containers this can be very slow
+  cy.get(RULES_TABLE, { timeout: 300000 }).should('exist');
+};
+
+/**
+ * Because the Rule Management page is relatively slow, in order to avoid timeouts and flakiness,
+ * we almost always want to wait until the Rules table is "loaded" before we do anything with it.
+ *
+ * This task can be needed for some tests that e.g. check the table load/refetch/pagination logic.
+ * It waits for the table's own loading indicator to show up and disappear.
+ *
+ * NOTE: Normally, we should not rely on loading indicators in tests, because due to their
+ * dynamic nature it's possible to introduce race conditions and flakiness.
+ */
 export const waitForRulesTableToBeLoaded = () => {
-  cy.get(RULES_TABLE_INITIAL_LOADING_INDICATOR).should('exist');
   // Wait up to 5 minutes for the rules to load as in CI containers this can be very slow
+  cy.get(RULES_TABLE_INITIAL_LOADING_INDICATOR, { timeout: 300000 }).should('exist');
   cy.get(RULES_TABLE_INITIAL_LOADING_INDICATOR, { timeout: 300000 }).should('not.exist');
 };
 

x-pack/plugins/security_solution/cypress/tasks/api_calls/fleet.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Deletes all existing Fleet packages, package policies and agent policies.
+ */
+export const cleanFleet = () => {
+  // NOTE: order does matter.
+  return deletePackagePolicies()
+    .then(() => {
+      deletePackages();
+    })
+    .then(() => {
+      deleteAgentPolicies();
+    });
+};
+
+const deleteAgentPolicies = () => {
+  return cy
+    .request({
+      method: 'GET',
+      url: 'api/fleet/agent_policies',
+      headers: { 'kbn-xsrf': 'cypress-creds' },
+    })
+    .then((response) => {
+      response.body.items.forEach((item: { id: string }) => {
+        cy.request({
+          method: 'POST',
+          url: `api/fleet/agent_policies/delete`,
+          headers: { 'kbn-xsrf': 'cypress-creds' },
+          body: {
+            agentPolicyId: item.id,
+          },
+        });
+      });
+    });
+};
+
+const deletePackagePolicies = () => {
+  return cy
+    .request({
+      method: 'GET',
+      url: 'api/fleet/package_policies',
+      headers: { 'kbn-xsrf': 'cypress-creds' },
+    })
+    .then((response) => {
+      cy.request({
+        method: 'POST',
+        url: `api/fleet/package_policies/delete`,
+        headers: { 'kbn-xsrf': 'cypress-creds' },
+        body: {
+          packagePolicyIds: response.body.items.map((item: { id: string }) => item.id),
+        },
+      });
+    });
+};
+
+const deletePackages = () => {
+  return cy
+    .request({
+      method: 'GET',
+      url: 'api/fleet/epm/packages',
+      headers: { 'kbn-xsrf': 'cypress-creds' },
+    })
+    .then((response) => {
+      response.body.items.forEach((item: { status: string; name: string; version: string }) => {
+        if (item.status === 'installed') {
+          cy.request({
+            method: 'DELETE',
+            url: `api/fleet/epm/packages/${item.name}/${item.version}`,
+            headers: { 'kbn-xsrf': 'cypress-creds' },
+          });
+        }
+      });
+    });
+};

x-pack/plugins/security_solution/cypress/tasks/common.ts

@@ -62,48 +62,6 @@ export const cleanKibana = () => {
   deleteTimelines();
 };
 
-export const cleanPackages = () => {
-  deletePolicies();
-  deletePackages();
-};
-
-export const deletePolicies = () => {
-  cy.request({
-    method: 'GET',
-    url: 'api/fleet/agent_policies',
-    headers: { 'kbn-xsrf': 'cypress-creds' },
-  }).then((response) => {
-    response.body.items.forEach((item: { id: string }) => {
-      cy.request({
-        method: 'POST',
-        url: `api/fleet/agent_policies/delete`,
-        headers: { 'kbn-xsrf': 'cypress-creds' },
-        body: {
-          agentPolicyId: item.id,
-        },
-      });
-    });
-  });
-};
-
-export const deletePackages = () => {
-  cy.request({
-    method: 'GET',
-    url: 'api/fleet/epm/packages',
-    headers: { 'kbn-xsrf': 'cypress-creds' },
-  }).then((response) => {
-    response.body.items.forEach((item: { status: string; name: string; version: string }) => {
-      if (item.status === 'installed') {
-        cy.request({
-          method: 'DELETE',
-          url: `api/fleet/epm/packages/${item.name}/${item.version}`,
-          headers: { 'kbn-xsrf': 'cypress-creds' },
-        });
-      }
-    });
-  });
-};
-
 export const deleteAlertsAndRules = () => {
   const kibanaIndexUrl = `${Cypress.env('ELASTICSEARCH_URL')}/.kibana_\*`;
 

x-pack/plugins/security_solution/cypress/tasks/integrations.ts

@@ -18,6 +18,8 @@ export const installAwsCloudFrontWithPolicy = () => {
   visit('app/integrations/detail/aws-1.17.0/overview?integration=cloudfront');
   cy.get(ADD_INTEGRATION_BTN).click();
   cy.get(QUEUE_URL).type('http://www.example.com');
+
+  // Fleet installs an integration very slowly, so we have to increase the timeout here.
   cy.get(SAVE_AND_CONTINUE_BTN).click();
-  cy.get(INTEGRATION_ADDED_POP_UP).should('exist');
+  cy.get(INTEGRATION_ADDED_POP_UP, { timeout: 120000 }).should('exist');
 };