[Security Solution] Unskip tests for the prebuilt rules bootstrap endpoint (#205106)

**Fixes: https://github.com/elastic/kibana/issues/197108**
**Fixes: https://github.com/elastic/kibana/issues/202037**
**Fixes: https://github.com/elastic/kibana/issues/203632**

## Summary

Builds on top of https://github.com/elastic/kibana/pull/203799 and:

- simplifies the package deletion code (1 [API
call](https://www.elastic.co/docs/api/doc/kibana/operation/operation-delete-fleet-epm-packages-pkgname-pkgversion)
instead of 2, per package)
- adds retry logic
- refactors the testing utils a bit

Flaky test runs:

-
[ESS](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7632)
- 190 runs
-
[Serverless](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7633)
- 100 runs

## Details

Here's how the test logs look like:

```
 info Starting tests

 └-: Rules Management - Prebuilt Rules - Prebuilt Rules Management
   └-> "before all" hook: beforeTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management"
   └-: @ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules
     └-> "before all" hook: beforeTestSuite.trigger for "should install fleet packages required for detection engine to function"
     └-> should install fleet packages required for detection engine to function
       └-> "before each" hook: global before each for "should install fleet packages required for detection engine to function"
       └-> "before each" hook for "should install fleet packages required for detection engine to function"
         │ debg Deleting security_detection_engine package
         │ debg Deleted security_detection_engine package (was not installed) {
         │        statusCode: 400,
         │        error: 'Bad Request',
         │        message: 'security_detection_engine is not installed'
         │      }
         │ debg Deleting endpoint package
         │ debg Deleted endpoint package (was not installed) {
         │        statusCode: 400,
         │        error: 'Bad Request',
         │        message: 'endpoint is not installed'
         │      }
       └- ✓ pass  (21.6s)
     └-> should skip installing fleet packages if they are already installed
       └-> "before each" hook: global before each for "should skip installing fleet packages if they are already installed"
       └-> "before each" hook for "should skip installing fleet packages if they are already installed"
         │ debg Deleting security_detection_engine package
         │ debg Deleted security_detection_engine package (was installed)
         │ debg Deleting endpoint package
         │ debg Deleted endpoint package (was installed)
       └- ✓ pass  (22.4s)
     └-> "after all" hook: afterTestSuite.trigger for "should skip installing fleet packages if they are already installed"
   └-> "after all" hook: afterTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management"

2 passing (1.0m)
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed

x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts

@@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
-  const retry = getService('retry');
+  const retryService = getService('retry');
 
   /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder,
   /* in order to assert that, in production environments, the latest stable version of the package
@@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
       await deleteAllPrebuiltRuleAssets(es, log);
-      await deletePrebuiltRulesFleetPackage(supertest);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
     });
 
     it('should install latest stable version and ignore prerelease packages', async () => {
@@ -49,7 +49,7 @@ export default ({ getService }: FtrProviderContext): void => {
       const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI(
         es,
         supertest,
-        retry
+        retryService
       );
 
       expect(fleetPackageInstallationResponse.items.length).toBe(1);

x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts

@@ -11,18 +11,24 @@ import {
 } from '@kbn/security-solution-plugin/common/detection_engine/constants';
 import expect from 'expect';
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
-import { deletePrebuiltRulesFleetPackage } from '../../../../utils';
-import { deleteEndpointFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_endpoint_fleet_package';
+import {
+  deleteAllPrebuiltRuleAssets,
+  deleteEndpointFleetPackage,
+  deletePrebuiltRulesFleetPackage,
+} from '../../../../utils';
 
 export default ({ getService }: FtrProviderContext): void => {
+  const es = getService('es');
+  const log = getService('log');
   const supertest = getService('supertest');
+  const retryService = getService('retry');
   const securitySolutionApi = getService('securitySolutionApi');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/203632
-  describe.skip('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => {
+  describe('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => {
     beforeEach(async () => {
-      await deletePrebuiltRulesFleetPackage(supertest);
-      await deleteEndpointFleetPackage(supertest);
+      await deleteAllPrebuiltRuleAssets(es, log);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
+      await deleteEndpointFleetPackage({ supertest, es, log, retryService });
     });
 
     it('should install fleet packages required for detection engine to function', async () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts

@@ -13,18 +13,18 @@ import {
 import { deleteAllRules } from '../../../../../../../common/utils/security_solution';
 import { deleteAllPrebuiltRuleAssets } from '../../../../utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets';
 import { deleteAllTimelines } from '../../../../utils/rules/prebuilt_rules/delete_all_timelines';
-import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package';
+import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_fleet_packages';
 import { installPrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package';
 
 export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
-  const retry = getService('retry');
+  const retryService = getService('retry');
 
   describe('@ess @serverless @skipInServerlessMKI install_prebuilt_rules_from_real_package', () => {
     beforeEach(async () => {
-      await deletePrebuiltRulesFleetPackage(supertest);
+      await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService });
       await deleteAllRules(supertest, log);
       await deleteAllTimelines(es, log);
       await deleteAllPrebuiltRuleAssets(es, log);
@@ -48,7 +48,7 @@ export default ({ getService }: FtrProviderContext): void => {
         es,
         supertest,
         overrideExistingPackage: true,
-        retryService: retry,
+        retryService,
       });
 
       // Verify that status is updated after package installation

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts

@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { epmRouteService } from '@kbn/fleet-plugin/common';
-import { ENDPOINT_PACKAGE_NAME } from '@kbn/security-solution-plugin/common/detection_engine/constants';
-import type SuperTest from 'supertest';
-
-/**
- * Delete the endpoint package using fleet API.
- *
- * @param supertest Supertest instance
- */
-export async function deleteEndpointFleetPackage(supertest: SuperTest.Agent) {
-  const resp = await supertest
-    .get(epmRouteService.getInfoPath(ENDPOINT_PACKAGE_NAME))
-    .set('kbn-xsrf', 'true')
-    .set('elastic-api-version', '2023-10-31')
-    .send();
-
-  if (resp.status === 200 && resp.body.item.status === 'installed') {
-    await supertest
-      .delete(epmRouteService.getRemovePath(ENDPOINT_PACKAGE_NAME, resp.body.item.version))
-      .set('kbn-xsrf', 'true')
-      .send({ force: true });
-  }
-}

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { epmRouteService } from '@kbn/fleet-plugin/common';
+import { RetryService } from '@kbn/ftr-common-functional-services';
+import {
+  ENDPOINT_PACKAGE_NAME,
+  PREBUILT_RULES_PACKAGE_NAME,
+} from '@kbn/security-solution-plugin/common/detection_engine/constants';
+import { ToolingLog } from '@kbn/tooling-log';
+import type SuperTest from 'supertest';
+import { refreshSavedObjectIndices } from '../../refresh_index';
+
+interface DeleteFleetPackageDeps {
+  supertest: SuperTest.Agent;
+  retryService: RetryService;
+  log: ToolingLog;
+  es: Client;
+}
+
+interface DeleteFleetPackageArgs {
+  packageName: string;
+  dependencies: DeleteFleetPackageDeps;
+}
+
+/**
+ * Delete the security_detection_engine package using fleet API.
+ */
+export async function deletePrebuiltRulesFleetPackage(params: DeleteFleetPackageDeps) {
+  await deleteFleetPackage({
+    packageName: PREBUILT_RULES_PACKAGE_NAME,
+    dependencies: params,
+  });
+}
+
+/**
+ * Delete the endpoint package using fleet API.
+ */
+export async function deleteEndpointFleetPackage(params: DeleteFleetPackageDeps) {
+  await deleteFleetPackage({
+    packageName: ENDPOINT_PACKAGE_NAME,
+    dependencies: params,
+  });
+}
+
+async function deleteFleetPackage(params: DeleteFleetPackageArgs): Promise<void> {
+  const { packageName, dependencies } = params;
+  const { supertest, retryService, log, es } = dependencies;
+
+  await retryService.tryWithRetries(
+    'deleteFleetPackage',
+    async () => {
+      log.debug(`Deleting ${packageName} package`);
+
+      const response = await supertest
+        .delete(epmRouteService.getRemovePath(packageName))
+        .set('kbn-xsrf', 'true')
+        .set('elastic-api-version', '2023-10-31')
+        .send({ force: true });
+
+      if (response.status === 200) {
+        log.debug(`Deleted ${packageName} package (was installed)`);
+        return;
+      } else if (
+        response.status === 400 &&
+        response.body.message === `${packageName} is not installed`
+      ) {
+        log.debug(`Deleted ${packageName} package (was not installed)`, response.body);
+        return;
+      } else {
+        log.warning(`Error deleting ${packageName} package`, response.body);
+        throw response.error;
+      }
+    },
+    {
+      retryCount: 2, // overall max 3 attempts
+      timeout: 3 * 60000, // total timeout applied to all attempts altogether
+    }
+  );
+
+  await refreshSavedObjectIndices(es);
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts

@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { epmRouteService } from '@kbn/fleet-plugin/common';
-import { PREBUILT_RULES_PACKAGE_NAME } from '@kbn/security-solution-plugin/common/detection_engine/constants';
-import type SuperTest from 'supertest';
-
-/**
- * Delete the security_detection_engine package using fleet API.
- *
- * @param supertest Supertest instance
- */
-export async function deletePrebuiltRulesFleetPackage(supertest: SuperTest.Agent) {
-  const resp = await supertest
-    .get(epmRouteService.getInfoPath(PREBUILT_RULES_PACKAGE_NAME))
-    .set('kbn-xsrf', 'true')
-    .set('elastic-api-version', '2023-10-31')
-    .send();
-
-  if (resp.status === 200 && resp.body.item.status === 'installed') {
-    await supertest
-      .delete(epmRouteService.getRemovePath(PREBUILT_RULES_PACKAGE_NAME, resp.body.item.version))
-      .set('kbn-xsrf', 'true')
-      .send({ force: true });
-  }
-}

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts

@@ -7,7 +7,7 @@
 export * from './create_prebuilt_rule_saved_objects';
 export * from './delete_all_prebuilt_rule_assets';
 export * from './delete_all_timelines';
-export * from './delete_prebuilt_rules_fleet_package';
+export * from './delete_fleet_packages';
 export * from './get_installed_rules';
 export * from './get_prebuilt_rules_and_timelines_status';
 export * from './get_prebuilt_rules_status';