[Response Ops] Add framework services for rule types to respect alert circuit breaker (#139501)

* Adding service functions to alert factory

* Fixing types

* Fixing types

* Small refactor

* Adding functional test

* Cleanup

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts

@@ -8,7 +8,7 @@
 import sinon from 'sinon';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { Alert } from './alert';
-import { createAlertFactory } from './create_alert_factory';
+import { createAlertFactory, getPublicAlertFactory } from './create_alert_factory';
 import { processAlerts } from '../lib';
 
 jest.mock('../lib', () => ({
@@ -238,4 +238,78 @@ describe('createAlertFactory()', () => {
       `Set doesSetRecoveryContext to true on rule type to get access to recovered alerts.`
     );
   });
+
+  test('throws error when checking limit usage if alertLimit.getValue is called but alertLimit.setLimitReached is not', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 1000,
+    });
+
+    const limit = alertFactory.alertLimit.getValue();
+    expect(limit).toEqual(1000);
+
+    expect(() => {
+      alertFactory.alertLimit.checkLimitUsage();
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"Rule has not reported whether alert limit has been reached after requesting limit value!"`
+    );
+  });
+
+  test('does not throw error when checking limit usage if alertLimit.getValue is called and alertLimit.setLimitReached is called with reached = true', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 1000,
+    });
+
+    const limit = alertFactory.alertLimit.getValue();
+    expect(limit).toEqual(1000);
+
+    alertFactory.alertLimit.setLimitReached(true);
+    alertFactory.alertLimit.checkLimitUsage();
+  });
+
+  test('does not throw error when checking limit usage if alertLimit.getValue is called and alertLimit.setLimitReached is called with reached = false', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 1000,
+    });
+
+    const limit = alertFactory.alertLimit.getValue();
+    expect(limit).toEqual(1000);
+
+    alertFactory.alertLimit.setLimitReached(false);
+    alertFactory.alertLimit.checkLimitUsage();
+  });
+});
+
+describe('getPublicAlertFactory', () => {
+  test('only returns subset of function from given alert factory', () => {
+    const alertFactory = createAlertFactory({
+      alerts: {},
+      logger,
+      maxAlerts: 1000,
+    });
+
+    expect(alertFactory.create).toBeDefined();
+    expect(alertFactory.alertLimit.getValue).toBeDefined();
+    expect(alertFactory.alertLimit.setLimitReached).toBeDefined();
+    expect(alertFactory.alertLimit.checkLimitUsage).toBeDefined();
+    expect(alertFactory.hasReachedAlertLimit).toBeDefined();
+    expect(alertFactory.done).toBeDefined();
+
+    const publicAlertFactory = getPublicAlertFactory(alertFactory);
+
+    expect(publicAlertFactory.create).toBeDefined();
+    expect(publicAlertFactory.done).toBeDefined();
+    expect(publicAlertFactory.alertLimit.getValue).toBeDefined();
+    expect(publicAlertFactory.alertLimit.setLimitReached).toBeDefined();
+
+    // @ts-expect-error
+    expect(publicAlertFactory.alertLimit.checkLimitUsage).not.toBeDefined();
+    // @ts-expect-error
+    expect(publicAlertFactory.hasReachedAlertLimit).not.toBeDefined();
+  });
 });

x-pack/plugins/alerting/server/alert/create_alert_factory.ts

@@ -11,6 +11,32 @@ import { AlertInstanceContext, AlertInstanceState } from '../types';
 import { Alert, PublicAlert } from './alert';
 import { processAlerts } from '../lib';
 
+export interface AlertFactory<
+  State extends AlertInstanceState,
+  Context extends AlertInstanceContext,
+  ActionGroupIds extends string
+> {
+  create: (id: string) => PublicAlert<State, Context, ActionGroupIds>;
+  alertLimit: {
+    getValue: () => number;
+    setLimitReached: (reached: boolean) => void;
+    checkLimitUsage: () => void;
+  };
+  hasReachedAlertLimit: () => boolean;
+  done: () => AlertFactoryDoneUtils<State, Context, ActionGroupIds>;
+}
+
+export type PublicAlertFactory<
+  State extends AlertInstanceState,
+  Context extends AlertInstanceContext,
+  ActionGroupIds extends string
+> = Pick<AlertFactory<State, Context, ActionGroupIds>, 'create' | 'done'> & {
+  alertLimit: Pick<
+    AlertFactory<State, Context, ActionGroupIds>['alertLimit'],
+    'getValue' | 'setLimitReached'
+  >;
+};
+
 export interface AlertFactoryDoneUtils<
   State extends AlertInstanceState,
   Context extends AlertInstanceContext,
@@ -38,7 +64,7 @@ export function createAlertFactory<
   logger,
   maxAlerts,
   canSetRecoveryContext = false,
-}: CreateAlertFactoryOpts<State, Context>) {
+}: CreateAlertFactoryOpts<State, Context>): AlertFactory<State, Context, ActionGroupIds> {
   // Keep track of which alerts we started with so we can determine which have recovered
   const originalAlerts = cloneDeep(alerts);
 
@@ -48,6 +74,12 @@ export function createAlertFactory<
   // Whether the number of alerts reported has reached max allowed
   let hasReachedAlertLimit = false;
 
+  // Whether rule type has asked for the alert limit
+  let hasRequestedAlertLimit = false;
+
+  // Whether rule type has reported back if alert limit was reached
+  let hasReportedLimitReached = false;
+
   let isDone = false;
   return {
     create: (id: string): PublicAlert<State, Context, ActionGroupIds> => {
@@ -66,6 +98,25 @@ export function createAlertFactory<
 
       return alerts[id];
     },
+    // namespace alert limit services for rule type executors to use
+    alertLimit: {
+      getValue: (): number => {
+        hasRequestedAlertLimit = true;
+        return maxAlerts;
+      },
+      setLimitReached: (reached: boolean) => {
+        hasReportedLimitReached = true;
+        hasReachedAlertLimit = reached;
+      },
+      checkLimitUsage: () => {
+        // If the rule type has requested the value but never reported back, throw an error
+        if (hasRequestedAlertLimit && !hasReportedLimitReached) {
+          throw new Error(
+            `Rule has not reported whether alert limit has been reached after requesting limit value!`
+          );
+        }
+      },
+    },
     hasReachedAlertLimit: (): boolean => hasReachedAlertLimit,
     done: (): AlertFactoryDoneUtils<State, Context, ActionGroupIds> => {
       isDone = true;
@@ -94,3 +145,20 @@ export function createAlertFactory<
     },
   };
 }
+
+export function getPublicAlertFactory<
+  State extends AlertInstanceState = AlertInstanceState,
+  Context extends AlertInstanceContext = AlertInstanceContext,
+  ActionGroupIds extends string = string
+>(
+  alertFactory: AlertFactory<State, Context, ActionGroupIds>
+): PublicAlertFactory<State, Context, ActionGroupIds> {
+  return {
+    create: (...args): PublicAlert<State, Context, ActionGroupIds> => alertFactory.create(...args),
+    alertLimit: {
+      getValue: (): number => alertFactory.alertLimit.getValue(),
+      setLimitReached: (...args): void => alertFactory.alertLimit.setLimitReached(...args),
+    },
+    done: (): AlertFactoryDoneUtils<State, Context, ActionGroupIds> => alertFactory.done(),
+  };
+}

x-pack/plugins/alerting/server/mocks.ts

@@ -104,7 +104,10 @@ const createRuleExecutorServicesMock = <
   return {
     alertFactory: {
       create: jest.fn().mockReturnValue(alertFactoryMockCreate),
-      hasReachedAlertLimit: jest.fn().mockReturnValue(false),
+      alertLimit: {
+        getValue: jest.fn().mockReturnValue(1000),
+        setLimitReached: jest.fn(),
+      },
       done: jest.fn().mockReturnValue(alertFactoryMockDone),
     },
     savedObjectsClient: savedObjectsClientMock.create(),

x-pack/plugins/alerting/server/task_runner/task_runner.ts

@@ -67,6 +67,7 @@ import { AlertingEventLogger } from '../lib/alerting_event_logger/alerting_event
 import { loadRule } from './rule_loader';
 import { logAlerts } from './log_alerts';
 import { scheduleActionsForAlerts } from './schedule_actions_for_alerts';
+import { getPublicAlertFactory } from '../alert/create_alert_factory';
 import { TaskRunnerTimer, TaskRunnerTimerSpan } from './task_runner_timer';
 
 const FALLBACK_RETRY_INTERVAL = '5m';
@@ -317,6 +318,18 @@ export class TaskRunner<
           maxAlerts: this.maxAlerts,
           canSetRecoveryContext: ruleType.doesSetRecoveryContext ?? false,
         });
+
+        const checkHasReachedAlertLimit = () => {
+          const reachedLimit = alertFactory.hasReachedAlertLimit();
+          if (reachedLimit) {
+            this.logger.warn(
+              `rule execution generated greater than ${this.maxAlerts} alerts: ${ruleLabel}`
+            );
+            ruleRunMetricsStore.setHasReachedAlertLimit(true);
+          }
+          return reachedLimit;
+        };
+
         let updatedState: void | Record<string, unknown>;
         try {
           const ctx = {
@@ -341,7 +354,7 @@ export class TaskRunner<
                 searchSourceClient: wrappedSearchSourceClient.searchSourceClient,
                 uiSettingsClient: this.context.uiSettings.asScopedToClient(savedObjectsClient),
                 scopedClusterClient: wrappedScopedClusterClient.client(),
-                alertFactory,
+                alertFactory: getPublicAlertFactory(alertFactory),
                 shouldWriteAlerts: () => this.shouldLogAndScheduleActionsForAlerts(),
                 shouldStopExecution: () => this.cancelled,
               },
@@ -374,14 +387,16 @@ export class TaskRunner<
               },
             })
           );
+
+          // Rule type execution has successfully completed
+          // Check that the rule type either never requested the max alerts limit
+          // or requested it and then reported back whether it exceeded the limit
+          // If neither of these apply, this check will throw an error
+          // These errors should show up during rule type development
+          alertFactory.alertLimit.checkLimitUsage();
         } catch (err) {
           // Check if this error is due to reaching the alert limit
-          if (alertFactory.hasReachedAlertLimit()) {
-            this.logger.warn(
-              `rule execution generated greater than ${this.maxAlerts} alerts: ${ruleLabel}`
-            );
-            ruleRunMetricsStore.setHasReachedAlertLimit(true);
-          } else {
+          if (!checkHasReachedAlertLimit()) {
             this.alertingEventLogger.setExecutionFailed(
               `rule execution failure: ${ruleLabel}`,
               err.message
@@ -394,6 +409,9 @@ export class TaskRunner<
           }
         }
 
+        // Check if the rule type has reported that it reached the alert limit
+        checkHasReachedAlertLimit();
+
         this.alertingEventLogger.setExecutionSucceeded(`rule executed: ${ruleLabel}`);
 
         ruleRunMetricsStore.setSearchMetrics([

x-pack/plugins/alerting/server/types.ts

@@ -19,7 +19,6 @@ import {
   SavedObjectsClientContract,
 } from '@kbn/core/server';
 import type { PublicMethodsOf } from '@kbn/utility-types';
-import { AlertFactoryDoneUtils, PublicAlert } from './alert';
 import { RuleTypeRegistry as OrigruleTypeRegistry } from './rule_type_registry';
 import { PluginSetupContract, PluginStartContract } from './plugin';
 import { RulesClient } from './rules_client';
@@ -44,6 +43,7 @@ import {
   MappedParams,
   RuleSnooze,
 } from '../common';
+import { PublicAlertFactory } from './alert/create_alert_factory';
 export type WithoutQueryAndParams<T> = Pick<T, Exclude<keyof T, 'query' | 'params'>>;
 export type SpaceIdToNamespaceFunction = (spaceId?: string) => string | undefined;
 export type { RuleTypeParams };
@@ -68,21 +68,16 @@ export type AlertingRequestHandlerContext = CustomRequestHandlerContext<{
  * @internal
  */
 export type AlertingRouter = IRouter<AlertingRequestHandlerContext>;
-
 export interface RuleExecutorServices<
-  InstanceState extends AlertInstanceState = AlertInstanceState,
-  InstanceContext extends AlertInstanceContext = AlertInstanceContext,
+  State extends AlertInstanceState = AlertInstanceState,
+  Context extends AlertInstanceContext = AlertInstanceContext,
   ActionGroupIds extends string = never
 > {
   searchSourceClient: ISearchStartSearchSource;
   savedObjectsClient: SavedObjectsClientContract;
   uiSettingsClient: IUiSettingsClient;
   scopedClusterClient: IScopedClusterClient;
-  alertFactory: {
-    create: (id: string) => PublicAlert<InstanceState, InstanceContext, ActionGroupIds>;
-    hasReachedAlertLimit: () => boolean;
-    done: () => AlertFactoryDoneUtils<InstanceState, InstanceContext, ActionGroupIds>;
-  };
+  alertFactory: PublicAlertFactory<State, Context, ActionGroupIds>;
   shouldWriteAlerts: () => boolean;
   shouldStopExecution: () => boolean;
 }

x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts

@@ -73,7 +73,10 @@ function createRule(shouldWriteAlerts: boolean = true) {
         scheduleActions,
       } as any;
     },
-    hasReachedAlertLimit: () => false,
+    alertLimit: {
+      getValue: () => 1000,
+      setLimitReached: () => {},
+    },
     done: () => ({ getRecoveredAlerts: () => [] }),
   };
 

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/preview_rules_route.ts

@@ -185,7 +185,10 @@ export const previewRulesRoute = async (
               | 'getContext'
               | 'hasContext'
             >;
-            hasReachedAlertLimit: () => boolean;
+            alertLimit: {
+              getValue: () => number;
+              setLimitReached: () => void;
+            };
             done: () => { getRecoveredAlerts: () => [] };
           }
         ) => {
@@ -287,7 +290,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -304,7 +310,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -321,7 +330,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -338,7 +350,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -353,7 +368,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -368,7 +386,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );
@@ -383,7 +404,10 @@ export const previewRulesRoute = async (
               () => true,
               {
                 create: alertInstanceFactoryStub,
-                hasReachedAlertLimit: () => false,
+                alertLimit: {
+                  getValue: () => 1000,
+                  setLimitReached: () => {},
+                },
                 done: () => ({ getRecoveredAlerts: () => [] }),
               }
             );

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts

@@ -76,7 +76,10 @@ export const createRuleTypeMocks = (
     scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
     alertFactory: {
       create: jest.fn(() => ({ scheduleActions })),
-      hasReachedAlertLimit: () => false,
+      alertLimit: {
+        getValue: jest.fn(() => 1000),
+        setLimitReached: jest.fn(() => {}),
+      },
       done: jest.fn().mockResolvedValue({}),
     },
     findAlerts: jest.fn(), // TODO: does this stay?

x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts

@@ -40,7 +40,10 @@ const alertFactory = (contextKeys: unknown[], testAlertActionArr: unknown[]) =>
     );
     return alertInstance;
   },
-  hasReachedAlertLimit: () => false,
+  alertLimit: {
+    getValue: () => 1000,
+    setLimitReached: () => {},
+  },
   done: () => ({ getRecoveredAlerts: () => [] }),
 });
 

x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/alert_types.ts

@@ -5,9 +5,10 @@
  * 2.0.
  */
 
+import uuid from 'uuid';
 import { CoreSetup } from '@kbn/core/server';
 import { schema, TypeOf } from '@kbn/config-schema';
-import { curry, times } from 'lodash';
+import { curry, range, times } from 'lodash';
 import {
   RuleType,
   AlertInstanceState,
@@ -274,6 +275,60 @@ function getFailingAlertType() {
   return result;
 }
 
+function getExceedsAlertLimitRuleType() {
+  const paramsSchema = schema.object({
+    index: schema.string(),
+    getsLimit: schema.boolean(),
+    reportsLimitReached: schema.boolean(),
+  });
+  type ParamsType = TypeOf<typeof paramsSchema>;
+  const result: RuleType<ParamsType, never, {}, {}, {}, 'default'> = {
+    id: 'test.exceedsAlertLimit',
+    name: 'Test: ExceedsAlertLimit',
+    validate: {
+      params: paramsSchema,
+    },
+    actionGroups: [
+      {
+        id: 'default',
+        name: 'Default',
+      },
+    ],
+    producer: 'alertsFixture',
+    defaultActionGroupId: 'default',
+    minimumLicenseRequired: 'basic',
+    isExportable: true,
+    async executor({ services, params, state }) {
+      let limit: number | null = null;
+      if (params.getsLimit) {
+        limit = services.alertFactory.alertLimit.getValue();
+      }
+
+      const alertsToCreate = limit ? limit : 25;
+
+      range(alertsToCreate)
+        .map(() => uuid.v4())
+        .forEach((id: string) => {
+          services.alertFactory.create(id).scheduleActions('default');
+        });
+
+      if (params.reportsLimitReached) {
+        services.alertFactory.alertLimit.setLimitReached(true);
+      }
+
+      // Index something
+      await services.scopedClusterClient.asCurrentUser.index({
+        index: params.index,
+        refresh: 'wait_for',
+        body: {
+          numAlerts: alertsToCreate,
+        },
+      });
+    },
+  };
+  return result;
+}
+
 function getAuthorizationAlertType(core: CoreSetup<FixtureStartDeps>) {
   const paramsSchema = schema.object({
     callClusterAuthorizationIndex: schema.string(),
@@ -786,4 +841,5 @@ export function defineAlertTypes(
   alerting.registerType(getLongRunningPatternRuleType(false));
   alerting.registerType(getCancellableRuleType());
   alerting.registerType(getPatternSuccessOrFailureAlertType());
+  alerting.registerType(getExceedsAlertLimitRuleType());
 }

x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/server/plugin.ts

@@ -72,6 +72,7 @@ export class FixturePlugin implements Plugin<void, void, FixtureSetupDeps, Fixtu
         'test.patternSuccessOrFailure',
         'test.throw',
         'test.longRunning',
+        'test.exceedsAlertLimit',
       ],
       privileges: {
         all: {
@@ -98,6 +99,7 @@ export class FixturePlugin implements Plugin<void, void, FixtureSetupDeps, Fixtu
                 'test.patternSuccessOrFailure',
                 'test.throw',
                 'test.longRunning',
+                'test.exceedsAlertLimit',
               ],
             },
           },
@@ -127,6 +129,7 @@ export class FixturePlugin implements Plugin<void, void, FixtureSetupDeps, Fixtu
                 'test.patternSuccessOrFailure',
                 'test.throw',
                 'test.longRunning',
+                'test.exceedsAlertLimit',
               ],
             },
           },

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/circuit_breaker/alert_limit_services.ts

@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+
+import { Spaces } from '../../../../scenarios';
+import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
+import {
+  ESTestIndexTool,
+  ES_TEST_INDEX_NAME,
+  getUrlPrefix,
+  ObjectRemover,
+  getEventLog,
+} from '../../../../../common/lib';
+
+// eslint-disable-next-line import/no-default-export
+export default function maxAlertsRuleTests({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+  const es = getService('es');
+  const esTestIndexTool = new ESTestIndexTool(es, retry);
+
+  describe('rule that hits max alerts circuit breaker', () => {
+    const objectRemover = new ObjectRemover(supertest);
+
+    beforeEach(async () => {
+      await esTestIndexTool.destroy();
+      await esTestIndexTool.setup();
+    });
+
+    afterEach(async () => {
+      await objectRemover.removeAll();
+      await esTestIndexTool.destroy();
+    });
+
+    it('short circuits rule execution if rule type lets framework handle alert limit', async () => {
+      const ruleId = await createRule({
+        name: 'no alert limit handling',
+        getsLimit: false,
+        reportsLimitReached: false,
+      });
+
+      const events = await retry.try(async () => {
+        return await getEventLog({
+          getService,
+          spaceId: Spaces.space1.id,
+          type: 'alert',
+          id: ruleId,
+          provider: 'alerting',
+          actions: new Map([['execute', { gte: 1 }]]),
+        });
+      });
+
+      // check that there's a warning in the execute event
+      const executeEvent = events[0];
+      expect(executeEvent?.event?.outcome).to.eql('success');
+      expect(executeEvent?.event?.reason).to.eql('maxAlerts');
+      expect(executeEvent?.kibana?.alerting?.status).to.eql('warning');
+      expect(executeEvent?.message).to.eql(
+        'Rule reported more than the maximum number of alerts in a single run. Alerts may be missed and recovery notifications may be delayed'
+      );
+
+      // check there are no docs written out in the ES_TEST_INDEX
+      const results = await es.search(
+        { index: ES_TEST_INDEX_NAME, body: { query: { match_all: {} } } },
+        { meta: true }
+      );
+      // @ts-expect-error doesn't handle total: number
+      const value = results.body.hits.total.value?.value || results.body.hits.total.value;
+      expect(value).to.eql(0);
+    });
+
+    it('ends in error if rule type requests alert limit but does not report back whether it reached the limit', async () => {
+      const ruleId = await createRule({
+        name: 'no alert limit handling',
+        getsLimit: true,
+        reportsLimitReached: false,
+      });
+
+      const events = await retry.try(async () => {
+        return await getEventLog({
+          getService,
+          spaceId: Spaces.space1.id,
+          type: 'alert',
+          id: ruleId,
+          provider: 'alerting',
+          actions: new Map([['execute', { gte: 1 }]]),
+        });
+      });
+
+      // check that there's an error in the execute event
+      const executeEvent = events[0];
+      expect(executeEvent?.event?.outcome).to.eql('failure');
+      expect(executeEvent?.event?.reason).to.eql('execute');
+      expect(executeEvent?.kibana?.alerting?.status).to.eql('error');
+      expect(executeEvent?.error?.message).to.eql(
+        `Rule has not reported whether alert limit has been reached after requesting limit value!`
+      );
+
+      // check there are docs written out in the ES_TEST_INDEX
+      const results = await es.search(
+        { index: ES_TEST_INDEX_NAME, body: { query: { match_all: {} } } },
+        { meta: true }
+      );
+      // @ts-expect-error doesn't handle total: number
+      const value = results.body.hits.total.value?.value || results.body.hits.total.value;
+      expect(value).to.eql(1);
+
+      const hit = results.body.hits.hits[0];
+      expect(hit._source).to.eql({
+        numAlerts: 20,
+      });
+    });
+
+    it('completes execution if rule type requests alert limit and reports back whether it reached the limit', async () => {
+      const ruleId = await createRule({
+        name: 'no alert limit handling',
+        getsLimit: true,
+        reportsLimitReached: true,
+      });
+
+      const events = await retry.try(async () => {
+        return await getEventLog({
+          getService,
+          spaceId: Spaces.space1.id,
+          type: 'alert',
+          id: ruleId,
+          provider: 'alerting',
+          actions: new Map([['execute', { gte: 1 }]]),
+        });
+      });
+
+      // check that there's a warning in the execute event
+      const executeEvent = events[0];
+      expect(executeEvent?.event?.outcome).to.eql('success');
+      expect(executeEvent?.event?.reason).to.eql('maxAlerts');
+      expect(executeEvent?.kibana?.alerting?.status).to.eql('warning');
+      expect(executeEvent?.message).to.eql(
+        'Rule reported more than the maximum number of alerts in a single run. Alerts may be missed and recovery notifications may be delayed'
+      );
+
+      // check there are docs written out in the ES_TEST_INDEX
+      const results = await es.search(
+        { index: ES_TEST_INDEX_NAME, body: { query: { match_all: {} } } },
+        { meta: true }
+      );
+      // @ts-expect-error doesn't handle total: number
+      const value = results.body.hits.total.value?.value || results.body.hits.total.value;
+      expect(value).to.eql(1);
+
+      const hit = results.body.hits.hits[0];
+      expect(hit._source).to.eql({
+        numAlerts: 20,
+      });
+    });
+
+    interface CreateRuleParams {
+      name: string;
+      getsLimit: boolean;
+      reportsLimitReached: boolean;
+    }
+
+    async function createRule(params: CreateRuleParams): Promise<string> {
+      const { status, body: createdRule } = await supertest
+        .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`)
+        .set('kbn-xsrf', 'foo')
+        .send({
+          name: params.name,
+          consumer: 'alerts',
+          enabled: true,
+          rule_type_id: 'test.exceedsAlertLimit',
+          schedule: { interval: '1m' },
+          actions: [],
+          notify_when: 'onActiveAlert',
+          params: {
+            index: ES_TEST_INDEX_NAME,
+            getsLimit: params.getsLimit,
+            reportsLimitReached: params.reportsLimitReached,
+          },
+        });
+
+      expect(status).to.be(200);
+
+      const ruleId = createdRule.id;
+      objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting');
+
+      return ruleId;
+    }
+  });
+}

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/circuit_breaker/index.ts

@@ -10,6 +10,14 @@ import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
 // eslint-disable-next-line import/no-default-export
 export default function alertingCircuitBreakerTests({ loadTestFile }: FtrProviderContext) {
   describe('circuit_breakers', () => {
-    loadTestFile(require.resolve('./max_alerts'));
+    /**
+     * This tests the expected behavior for a rule type that hits the alert limit in a single execution.
+     */
+    loadTestFile(require.resolve('./alert_limit_services'));
+    /**
+     * This tests the expected behavior for the active and recovered alerts generated over
+     * a sequence of rule executions that hit the alert limit.
+     */
+    loadTestFile(require.resolve('./index_threshold_max_alerts'));
   });
 }

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/circuit_breaker/index_threshold_max_alerts.ts

@@ -29,7 +29,7 @@ export default function maxAlertsRuleTests({ getService }: FtrProviderContext) {
   const es = getService('es');
   const esTestIndexTool = new ESTestIndexTool(es, retry);
 
-  describe('rule that hits max alerts circuit breaker', () => {
+  describe('index threshold rule that hits max alerts circuit breaker', () => {
     const objectRemover = new ObjectRemover(supertest);
 
     beforeEach(async () => {