[8.15] [Detection Engine] Addresses Flakiness in ML FTR tests (#188155) (#188259)

# Backport This will backport the following commits from `main` to `8.15`: - [[Detection Engine] Addresses Flakiness in ML FTR tests (#188155)](https://github.com/elastic/kibana/pull/188155)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
2025-04-23 17:28:26 -04:00 · 2024-07-12 17:49:13 -05:00 · 2024-07-12 17:49:13 -05:00 · 38677570cb
commit 38677570cb
parent bc5b5da7fd
3 changed files with 46 additions and 9 deletions
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts
@ -40,12 +40,12 @@ import {
  importFile,
 } from '../../../../../lists_and_exception_lists/utils';
 import {
-  executeSetupModuleRequest,
  forceStartDatafeeds,
  getAlerts,
  getPreviewAlerts,
  previewRule,
  previewRuleWithExceptionEntries,
+  setupMlModulesWithRetry,
 } from '../../../../utils';
 import {
  createRule,
@ -86,13 +86,12 @@ export default ({ getService }: FtrProviderContext) => {
    rule_id: 'ml-rule-id',
  };

-  // FLAKY: https://github.com/elastic/kibana/issues/171426
-  describe.skip('@ess @serverless @serverlessQA Machine learning type rules', () => {
+  describe('@ess @serverless @serverlessQA Machine learning type rules', () => {
    before(async () => {
      // Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
      // as the job looks for certain indices on start
      await esArchiver.load(auditPath);
-      await executeSetupModuleRequest({ module: siemModule, rspCode: 200, supertest });
+      await setupMlModulesWithRetry({ module: siemModule, supertest, retry });
      await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
      await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
    });
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts
@ -27,7 +27,6 @@ import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder'
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
 import {
  dataGeneratorFactory,
-  executeSetupModuleRequest,
  forceStartDatafeeds,
  getAlerts,
  getOpenAlerts,
@ -36,6 +35,7 @@ import {
  previewRule,
  previewRuleWithExceptionEntries,
  setAlertStatus,
+  setupMlModulesWithRetry,
 } from '../../../../utils';
 import {
  createRule,
@ -51,6 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
  const es = getService('es');
  const log = getService('log');
  const config = getService('config');
+  const retry = getService('retry');

  const isServerless = config.get('serverless');
  const dataPathBuilder = new EsArchivePathBuilder(isServerless);
@ -87,14 +88,13 @@ export default ({ getService }: FtrProviderContext) => {
  // The tests described in this file rely on the
  // 'alertSuppressionForMachineLearningRuleEnabled' feature flag, and are thus
  // skipped in MKI
-  // Failing: See https://github.com/elastic/kibana/issues/187478
-  describe.skip('@ess @serverless @skipInServerlessMKI Machine Learning Detection Rule - Alert Suppression', () => {
+  describe('@ess @serverless @skipInServerlessMKI Machine Learning Detection Rule - Alert Suppression', () => {
    describe('with an active ML Job', () => {
      before(async () => {
        // Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
        // as the job looks for certain indices on start
        await esArchiver.load(auditbeatArchivePath);
-        await executeSetupModuleRequest({ module: mlModuleName, rspCode: 200, supertest });
+        await setupMlModulesWithRetry({ module: mlModuleName, retry, supertest });
        await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
        await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
        await deleteAllAnomalies(log, es);
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts
@ -6,9 +6,18 @@
 */

 import type SuperTest from 'supertest';
+import { RetryService } from '@kbn/ftr-common-functional-services';
 import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants';
 import { getCommonRequestHeader } from '../../../../../functional/services/ml/common_api';

+interface ModuleJob {
+  id: string;
+  success: boolean;
+  error?: {
+    status: number;
+  };
+}
+
 export const executeSetupModuleRequest = async ({
  module,
  rspCode,
@ -17,7 +26,7 @@ export const executeSetupModuleRequest = async ({
  module: string;
  rspCode: number;
  supertest: SuperTest.Agent;
-}) => {
+}): Promise<{ jobs: ModuleJob[] }> => {
  const { body } = await supertest
    .post(`/internal/ml/modules/setup/${module}`)
    .set(getCommonRequestHeader('1'))
@ -34,6 +43,35 @@ export const executeSetupModuleRequest = async ({
  return body;
 };

+export const setupMlModulesWithRetry = async ({
+  module,
+  retry,
+  supertest,
+}: {
+  module: string;
+  retry: RetryService;
+  supertest: SuperTest.Agent;
+}) =>
+  retry.try(async () => {
+    const response = await executeSetupModuleRequest({
+      module,
+      rspCode: 200,
+      supertest,
+    });
+
+    const allJobsSucceeded = response?.jobs.every((job) => {
+      return job.success || (job.error?.status && job.error.status < 500);
+    });
+
+    if (!allJobsSucceeded) {
+      throw new Error(
+        `Expected all jobs to set up successfully, but got ${JSON.stringify(response)}`
+      );
+    }
+
+    return response;
+  });
+
 export const forceStartDatafeeds = async ({
  jobId,
  rspCode,