[ResponseOps][Flapping] Update notifyWhen check (#165167)

Resolves https://github.com/elastic/kibana/issues/165002

## Summary

Updates notifyWhen check inside the task runner to include notifyWhen
set in the action frequency. Adds an additional check at the action
level in the execution handler.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts

@@ -11,7 +11,6 @@ import {
   DEFAULT_FLAPPING_SETTINGS,
   RecoveredActionGroup,
   RuleAlertData,
-  RuleNotifyWhen,
 } from '../types';
 import * as LegacyAlertsClientModule from './legacy_alerts_client';
 import { LegacyAlertsClient } from './legacy_alerts_client';
@@ -114,7 +113,7 @@ describe('Alerts Client', () => {
           ruleRunMetricsStore,
           shouldLogAlerts: false,
           flappingSettings: DEFAULT_FLAPPING_SETTINGS,
-          notifyWhen: RuleNotifyWhen.CHANGE,
+          notifyOnActionGroupChange: true,
           maintenanceWindowIds: [],
         };
       });

x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts

@@ -6,7 +6,7 @@
  */
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { UntypedNormalizedRuleType } from '../rule_type_registry';
-import { AlertInstanceContext, RecoveredActionGroup, RuleNotifyWhen } from '../types';
+import { AlertInstanceContext, RecoveredActionGroup } from '../types';
 import { LegacyAlertsClient } from './legacy_alerts_client';
 import { createAlertFactory, getPublicAlertFactory } from '../alert/create_alert_factory';
 import { Alert } from '../alert/alert';
@@ -283,7 +283,7 @@ describe('Legacy Alerts Client', () => {
       ruleRunMetricsStore,
       shouldLogAlerts: true,
       flappingSettings: DEFAULT_FLAPPING_SETTINGS,
-      notifyWhen: RuleNotifyWhen.CHANGE,
+      notifyOnActionGroupChange: true,
       maintenanceWindowIds: ['window-id1', 'window-id2'],
     });
 
@@ -312,7 +312,7 @@ describe('Legacy Alerts Client', () => {
         lookBackWindow: 20,
         statusChangeThreshold: 4,
       },
-      RuleNotifyWhen.CHANGE,
+      true,
       'default',
       {},
       {

x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts

@@ -136,7 +136,7 @@ export class LegacyAlertsClient<
     ruleRunMetricsStore,
     shouldLogAlerts,
     flappingSettings,
-    notifyWhen,
+    notifyOnActionGroupChange,
     maintenanceWindowIds,
   }: ProcessAndLogAlertsOpts) {
     const {
@@ -163,7 +163,7 @@ export class LegacyAlertsClient<
 
     const alerts = getAlertsForNotification<State, Context, ActionGroupIds, RecoveryActionGroupId>(
       flappingSettings,
-      notifyWhen,
+      notifyOnActionGroupChange,
       this.options.ruleType.defaultActionGroupId,
       processedAlertsNew,
       processedAlertsActive,

x-pack/plugins/alerting/server/alerts_client/types.ts

@@ -16,7 +16,6 @@ import {
   SummarizedAlerts,
   RawAlertInstance,
   RuleAlertData,
-  RuleNotifyWhenType,
   WithoutReservedActionGroups,
 } from '../types';
 import { AlertingEventLogger } from '../lib/alerting_event_logger/alerting_event_logger';
@@ -82,7 +81,7 @@ export interface ProcessAndLogAlertsOpts {
   shouldLogAlerts: boolean;
   ruleRunMetricsStore: RuleRunMetricsStore;
   flappingSettings: RulesSettingsFlappingProperties;
-  notifyWhen: RuleNotifyWhenType | null;
+  notifyOnActionGroupChange: boolean;
   maintenanceWindowIds: string[];
 }
 

x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts

@@ -9,7 +9,6 @@ import { DEFAULT_FLAPPING_SETTINGS, DISABLE_FLAPPING_SETTINGS } from '../../comm
 import { getAlertsForNotification } from '.';
 import { Alert } from '../alert';
 import { alertsWithAnyUUID } from '../test_utils';
-import { RuleNotifyWhen } from '../types';
 
 describe('getAlertsForNotification', () => {
   test('should set pendingRecoveredCount to zero for all active alerts', () => {
@@ -20,7 +19,7 @@ describe('getAlertsForNotification', () => {
 
     const { newAlerts, activeAlerts } = getAlertsForNotification(
       DEFAULT_FLAPPING_SETTINGS,
-      RuleNotifyWhen.CHANGE,
+      true,
       'default',
       {
         '1': alert1,
@@ -85,7 +84,7 @@ describe('getAlertsForNotification', () => {
       currentRecoveredAlerts,
     } = getAlertsForNotification(
       DEFAULT_FLAPPING_SETTINGS,
-      RuleNotifyWhen.CHANGE,
+      true,
       'default',
       {},
       {},
@@ -212,7 +211,7 @@ describe('getAlertsForNotification', () => {
     const { newAlerts, activeAlerts, recoveredAlerts, currentRecoveredAlerts } =
       getAlertsForNotification(
         DISABLE_FLAPPING_SETTINGS,
-        RuleNotifyWhen.CHANGE,
+        true,
         'default',
         {},
         {},
@@ -337,7 +336,7 @@ describe('getAlertsForNotification', () => {
       currentRecoveredAlerts,
     } = getAlertsForNotification(
       DEFAULT_FLAPPING_SETTINGS,
-      RuleNotifyWhen.ACTIVE,
+      false,
       'default',
       {},
       {},

x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts

@@ -8,12 +8,7 @@
 import { keys } from 'lodash';
 import { RulesSettingsFlappingProperties } from '../../common/rules_settings';
 import { Alert } from '../alert';
-import {
+import { AlertInstanceState, AlertInstanceContext } from '../types';
-  AlertInstanceState,
-  AlertInstanceContext,
-  RuleNotifyWhenType,
-  RuleNotifyWhen,
-} from '../types';
 
 export function getAlertsForNotification<
   State extends AlertInstanceState,
@@ -22,7 +17,7 @@ export function getAlertsForNotification<
   RecoveryActionGroupId extends string
 >(
   flappingSettings: RulesSettingsFlappingProperties,
-  notifyWhen: RuleNotifyWhenType | null,
+  notifyOnActionGroupChange: boolean,
   actionGroupId: string,
   newAlerts: Record<string, Alert<State, Context, ActionGroupIds>> = {},
   activeAlerts: Record<string, Alert<State, Context, ActionGroupIds>> = {},
@@ -62,8 +57,9 @@ export function getAlertsForNotification<
           );
           activeAlerts[id] = newAlert;
 
-          // rules with "on status change" should return notifications
+          // rule with "on status change" or rule with at least one
-          if (notifyWhen === RuleNotifyWhen.CHANGE) {
+          // action with "on status change" should return notifications
+          if (notifyOnActionGroupChange) {
             currentActiveAlerts[id] = newAlert;
           }
 

x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts

@@ -28,7 +28,7 @@ import { alertingEventLoggerMock } from '../lib/alerting_event_logger/alerting_e
 import { TaskRunnerContext } from './task_runner_factory';
 import { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server';
 import { Alert } from '../alert';
-import { AlertInstanceState, AlertInstanceContext } from '../../common';
+import { AlertInstanceState, AlertInstanceContext, RuleNotifyWhen } from '../../common';
 import { asSavedObjectExecutionSource } from '@kbn/actions-plugin/server';
 import sinon from 'sinon';
 import { mockAAD } from './fixtures';
@@ -155,6 +155,7 @@ const generateAlert = ({
   throttledActions = {},
   lastScheduledActionsGroup = 'default',
   maintenanceWindowIds,
+  pendingRecoveredCount,
 }: {
   id: number;
   group?: ActiveActionGroup | 'recovered';
@@ -164,6 +165,7 @@ const generateAlert = ({
   throttledActions?: ThrottledActions;
   lastScheduledActionsGroup?: string;
   maintenanceWindowIds?: string[];
+  pendingRecoveredCount?: number;
 }) => {
   const alert = new Alert<AlertInstanceState, AlertInstanceContext, 'default' | 'other-group'>(
     String(id),
@@ -176,6 +178,7 @@ const generateAlert = ({
           group: lastScheduledActionsGroup,
           actions: throttledActions,
         },
+        pendingRecoveredCount,
       },
     }
   );
@@ -1778,6 +1781,157 @@ describe('Execution Handler', () => {
     );
   });
 
+  test('does not schedule actions with notifyWhen not set to "on status change" for alerts that are flapping', async () => {
+    const executionHandler = new ExecutionHandler(
+      generateExecutionParams({
+        ...defaultExecutionParams,
+        rule: {
+          ...defaultExecutionParams.rule,
+          actions: [
+            {
+              ...defaultExecutionParams.rule.actions[0],
+              frequency: {
+                summary: false,
+                notifyWhen: RuleNotifyWhen.ACTIVE,
+                throttle: null,
+              },
+            },
+          ],
+        },
+      })
+    );
+
+    await executionHandler.run({
+      ...generateAlert({ id: 1, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+      ...generateAlert({ id: 2, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+      ...generateAlert({ id: 3, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+    });
+
+    expect(actionsClient.bulkEnqueueExecution).not.toHaveBeenCalled();
+  });
+
+  test('does schedule actions with notifyWhen is set to "on status change" for alerts that are flapping', async () => {
+    const executionHandler = new ExecutionHandler(
+      generateExecutionParams({
+        ...defaultExecutionParams,
+        rule: {
+          ...defaultExecutionParams.rule,
+          actions: [
+            {
+              ...defaultExecutionParams.rule.actions[0],
+              frequency: {
+                summary: false,
+                notifyWhen: RuleNotifyWhen.CHANGE,
+                throttle: null,
+              },
+            },
+          ],
+        },
+      })
+    );
+
+    await executionHandler.run({
+      ...generateAlert({ id: 1, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+      ...generateAlert({ id: 2, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+      ...generateAlert({ id: 3, pendingRecoveredCount: 1, lastScheduledActionsGroup: 'recovered' }),
+    });
+
+    expect(actionsClient.bulkEnqueueExecution).toHaveBeenCalledTimes(1);
+    expect(actionsClient.bulkEnqueueExecution.mock.calls[0]).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          Object {
+            "actionTypeId": "test",
+            "apiKey": "MTIzOmFiYw==",
+            "consumer": "rule-consumer",
+            "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28",
+            "id": "1",
+            "params": Object {
+              "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 1 goes here",
+              "contextVal": "My  goes here",
+              "foo": true,
+              "stateVal": "My  goes here",
+            },
+            "relatedSavedObjects": Array [
+              Object {
+                "id": "1",
+                "namespace": "test1",
+                "type": "alert",
+                "typeId": "test",
+              },
+            ],
+            "source": Object {
+              "source": Object {
+                "id": "1",
+                "type": "alert",
+              },
+              "type": "SAVED_OBJECT",
+            },
+            "spaceId": "test1",
+          },
+          Object {
+            "actionTypeId": "test",
+            "apiKey": "MTIzOmFiYw==",
+            "consumer": "rule-consumer",
+            "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28",
+            "id": "1",
+            "params": Object {
+              "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 2 goes here",
+              "contextVal": "My  goes here",
+              "foo": true,
+              "stateVal": "My  goes here",
+            },
+            "relatedSavedObjects": Array [
+              Object {
+                "id": "1",
+                "namespace": "test1",
+                "type": "alert",
+                "typeId": "test",
+              },
+            ],
+            "source": Object {
+              "source": Object {
+                "id": "1",
+                "type": "alert",
+              },
+              "type": "SAVED_OBJECT",
+            },
+            "spaceId": "test1",
+          },
+          Object {
+            "actionTypeId": "test",
+            "apiKey": "MTIzOmFiYw==",
+            "consumer": "rule-consumer",
+            "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28",
+            "id": "1",
+            "params": Object {
+              "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 3 goes here",
+              "contextVal": "My  goes here",
+              "foo": true,
+              "stateVal": "My  goes here",
+            },
+            "relatedSavedObjects": Array [
+              Object {
+                "id": "1",
+                "namespace": "test1",
+                "type": "alert",
+                "typeId": "test",
+              },
+            ],
+            "source": Object {
+              "source": Object {
+                "id": "1",
+                "type": "alert",
+              },
+              "type": "SAVED_OBJECT",
+            },
+            "spaceId": "test1",
+          },
+        ],
+      ]
+    `);
+  });
+
   describe('rule url', () => {
     const ruleWithUrl = {
       ...rule,

x-pack/plugins/alerting/server/task_runner/execution_handler.ts

@@ -36,6 +36,7 @@ import {
   RuleTypeState,
   SanitizedRule,
   RuleAlertData,
+  RuleNotifyWhen,
 } from '../../common';
 import {
   generateActionHash,
@@ -621,6 +622,16 @@ export class ExecutionHandler<
           );
           continue;
         }
+
+        // only actions with notifyWhen set to "on status change" should return
+        // notifications for flapping pending recovered alerts
+        if (
+          alert.getPendingRecoveredCount() > 0 &&
+          action.frequency?.notifyWhen !== RuleNotifyWhen.CHANGE
+        ) {
+          continue;
+        }
+
         if (action.group === actionGroup && !this.isAlertMuted(alertId)) {
           if (
             this.isRecoveredAlert(action.group) ||

x-pack/plugins/alerting/server/task_runner/task_runner.ts

@@ -6,7 +6,7 @@
  */
 
 import apm from 'elastic-apm-node';
-import { omit } from 'lodash';
+import { omit, some } from 'lodash';
 import { UsageCounter } from '@kbn/usage-collection-plugin/server';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from '@kbn/core/server';
@@ -50,6 +50,7 @@ import {
   MaintenanceWindow,
   RuleAlertData,
   SanitizedRule,
+  RuleNotifyWhen,
 } from '../../common';
 import { NormalizedRuleType, UntypedNormalizedRuleType } from '../rule_type_registry';
 import { getEsErrorMessage } from '../lib/errors';
@@ -546,7 +547,9 @@ export class TaskRunner<
         ruleRunMetricsStore,
         shouldLogAlerts: this.shouldLogAndScheduleActionsForAlerts(),
         flappingSettings,
-        notifyWhen,
+        notifyOnActionGroupChange:
+          notifyWhen === RuleNotifyWhen.CHANGE ||
+          some(actions, (action) => action.frequency?.notifyWhen === RuleNotifyWhen.CHANGE),
         maintenanceWindowIds,
       });
     });

x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts

@@ -14,7 +14,6 @@ import {
   AlertInstanceState,
   AlertInstanceContext,
   Rule,
-  RuleNotifyWhen,
   RuleAlertData,
 } from '../types';
 import { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server';
@@ -772,7 +771,7 @@ describe('Task Runner', () => {
           lookBackWindow: 20,
           statusChangeThreshold: 4,
         },
-        notifyWhen: RuleNotifyWhen.ACTIVE,
+        notifyOnActionGroupChange: false,
         maintenanceWindowIds: [],
       });
       expect(alertsClientNotToUse.processAndLogAlerts).not.toHaveBeenCalled();

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts

@@ -725,6 +725,114 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
           expect(flapping).to.eql(result);
         });
 
+        it('should generate expected events for flapping alerts that settle on active where the action notifyWhen is set to "on status change"', async () => {
+          await supertest
+            .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
+            .set('kbn-xsrf', 'foo')
+            .auth('superuser', 'superuser')
+            .send({
+              enabled: true,
+              look_back_window: 6,
+              status_change_threshold: 4,
+            })
+            .expect(200);
+          const { body: createdAction } = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/actions/connector`)
+            .set('kbn-xsrf', 'foo')
+            .send({
+              name: 'MY action',
+              connector_type_id: 'test.noop',
+              config: {},
+              secrets: {},
+            })
+            .expect(200);
+
+          // pattern of when the alert should fire
+          const instance = [true, false, false, true, false, true, false, true, false].concat(
+            ...new Array(8).fill(true),
+            false
+          );
+          const pattern = {
+            instance,
+          };
+
+          const response = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/alerting/rule`)
+            .set('kbn-xsrf', 'foo')
+            .send(
+              getTestRuleData({
+                rule_type_id: 'test.patternFiring',
+                schedule: { interval: '1s' },
+                throttle: null,
+                notify_when: null,
+                params: {
+                  pattern,
+                },
+                actions: [
+                  {
+                    id: createdAction.id,
+                    group: 'default',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: null,
+                      notify_when: RuleNotifyWhen.CHANGE,
+                    },
+                  },
+                  {
+                    id: createdAction.id,
+                    group: 'recovered',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: null,
+                      notify_when: RuleNotifyWhen.CHANGE,
+                    },
+                  },
+                ],
+              })
+            );
+
+          expect(response.status).to.eql(200);
+          const alertId = response.body.id;
+          objectRemover.add(space.id, alertId, 'rule', 'alerting');
+
+          // get the events we're expecting
+          const events = await retry.try(async () => {
+            return await getEventLog({
+              getService,
+              spaceId: space.id,
+              type: 'alert',
+              id: alertId,
+              provider: 'alerting',
+              actions: new Map([
+                // make sure the counts of the # of events per type are as expected
+                ['execute-start', { gte: 6 }],
+                ['execute', { gte: 6 }],
+                ['execute-action', { equal: 6 }],
+                ['new-instance', { equal: 3 }],
+                ['active-instance', { gte: 6 }],
+                ['recovered-instance', { equal: 3 }],
+              ]),
+            });
+          });
+
+          const flapping = events
+            .filter(
+              (event) =>
+                event?.event?.action === 'active-instance' ||
+                event?.event?.action === 'recovered-instance'
+            )
+            .map((event) => event?.kibana?.alert?.flapping);
+          const result = [false, false, false, false, false].concat(
+            new Array(9).fill(true),
+            false,
+            false,
+            false
+          );
+          expect(flapping).to.eql(result);
+        });
+
         it('should generate expected events for flapping alerts settle on recovered', async () => {
           await supertest
             .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
@@ -818,6 +926,109 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
           );
         });
 
+        it('should generate expected events for flapping alerts settle on recovered where the action notifyWhen is set to "on status change"', async () => {
+          await supertest
+            .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
+            .set('kbn-xsrf', 'foo')
+            .auth('superuser', 'superuser')
+            .send({
+              enabled: true,
+              look_back_window: 6,
+              status_change_threshold: 4,
+            })
+            .expect(200);
+          const { body: createdAction } = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/actions/connector`)
+            .set('kbn-xsrf', 'foo')
+            .send({
+              name: 'MY action',
+              connector_type_id: 'test.noop',
+              config: {},
+              secrets: {},
+            })
+            .expect(200);
+
+          // pattern of when the alert should fire
+          const instance = [true, false, false, true, false, true, false, true, false, true].concat(
+            new Array(11).fill(false)
+          );
+          const pattern = {
+            instance,
+          };
+
+          const response = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/alerting/rule`)
+            .set('kbn-xsrf', 'foo')
+            .send(
+              getTestRuleData({
+                rule_type_id: 'test.patternFiring',
+                schedule: { interval: '1s' },
+                throttle: null,
+                notify_when: null,
+                params: {
+                  pattern,
+                },
+                actions: [
+                  {
+                    id: createdAction.id,
+                    group: 'default',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: null,
+                      notify_when: RuleNotifyWhen.CHANGE,
+                    },
+                  },
+                  {
+                    id: createdAction.id,
+                    group: 'recovered',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: null,
+                      notify_when: RuleNotifyWhen.CHANGE,
+                    },
+                  },
+                ],
+              })
+            );
+
+          expect(response.status).to.eql(200);
+          const alertId = response.body.id;
+          objectRemover.add(space.id, alertId, 'rule', 'alerting');
+
+          // get the events we're expecting
+          const events = await retry.try(async () => {
+            return await getEventLog({
+              getService,
+              spaceId: space.id,
+              type: 'alert',
+              id: alertId,
+              provider: 'alerting',
+              actions: new Map([
+                // make sure the counts of the # of events per type are as expected
+                ['execute-start', { gte: 6 }],
+                ['execute', { gte: 6 }],
+                ['execute-action', { equal: 6 }],
+                ['new-instance', { equal: 3 }],
+                ['active-instance', { gte: 3 }],
+                ['recovered-instance', { equal: 3 }],
+              ]),
+            });
+          });
+
+          const flapping = events
+            .filter(
+              (event) =>
+                event?.event?.action === 'active-instance' ||
+                event?.event?.action === 'recovered-instance'
+            )
+            .map((event) => event?.kibana?.alert?.flapping);
+          expect(flapping).to.eql(
+            [false, false, false, false, false].concat(new Array(8).fill(true))
+          );
+        });
+
         it('should generate expected events for flapping alerts over a period of time longer than the look back', async () => {
           await supertest
             .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
@@ -917,7 +1128,7 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
           expect(flapping).to.eql(result);
         });
 
-        it('should generate expected events for flapping alerts that settle on active where notifyWhen is not set to "on status change"', async () => {
+        it('should generate expected events for flapping alerts that settle on active where notifyWhen is NOT set to "on status change"', async () => {
           await supertest
             .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
             .set('kbn-xsrf', 'foo')
@@ -955,7 +1166,7 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
               getTestRuleData({
                 rule_type_id: 'test.patternFiring',
                 schedule: { interval: '1s' },
-                throttle: null,
+                throttle: '1s',
                 params: {
                   pattern,
                 },
@@ -1014,7 +1225,205 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
           expect(flapping).to.eql(result);
         });
 
-        it('should generate expected events for flapping alerts that settle on recovered where notifyWhen is not set to "on status change"', async () => {
+        it('should generate expected events for flapping alerts that settle on active where the action notifyWhen is NOT set to "on status change"', async () => {
+          await supertest
+            .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
+            .set('kbn-xsrf', 'foo')
+            .auth('superuser', 'superuser')
+            .send({
+              enabled: true,
+              look_back_window: 6,
+              status_change_threshold: 4,
+            })
+            .expect(200);
+          const { body: createdAction } = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/actions/connector`)
+            .set('kbn-xsrf', 'foo')
+            .send({
+              name: 'MY action',
+              connector_type_id: 'test.noop',
+              config: {},
+              secrets: {},
+            })
+            .expect(200);
+
+          // pattern of when the alert should fire
+          const instance = [true, false, false, true, false, true, false, true, false].concat(
+            ...new Array(8).fill(true),
+            false
+          );
+          const pattern = {
+            instance,
+          };
+
+          const response = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/alerting/rule`)
+            .set('kbn-xsrf', 'foo')
+            .send(
+              getTestRuleData({
+                rule_type_id: 'test.patternFiring',
+                schedule: { interval: '1s' },
+                throttle: null,
+                notify_when: null,
+                params: {
+                  pattern,
+                },
+                actions: [
+                  {
+                    id: createdAction.id,
+                    group: 'default',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: '1s',
+                      notify_when: RuleNotifyWhen.THROTTLE,
+                    },
+                  },
+                  {
+                    id: createdAction.id,
+                    group: 'recovered',
+                    params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: '1s',
+                      notify_when: RuleNotifyWhen.THROTTLE,
+                    },
+                  },
+                ],
+              })
+            );
+
+          expect(response.status).to.eql(200);
+          const alertId = response.body.id;
+          objectRemover.add(space.id, alertId, 'rule', 'alerting');
+
+          // get the events we're expecting
+          const events = await retry.try(async () => {
+            return await getEventLog({
+              getService,
+              spaceId: space.id,
+              type: 'alert',
+              id: alertId,
+              provider: 'alerting',
+              actions: new Map([
+                // make sure the counts of the # of events per type are as expected
+                ['execute-start', { gte: 15 }],
+                ['execute', { gte: 15 }],
+                ['execute-action', { equal: 15 }],
+                ['new-instance', { equal: 3 }],
+                ['active-instance', { gte: 6 }],
+                ['recovered-instance', { equal: 3 }],
+              ]),
+            });
+          });
+
+          const flapping = events
+            .filter(
+              (event) =>
+                event?.event?.action === 'active-instance' ||
+                event?.event?.action === 'recovered-instance'
+            )
+            .map((event) => event?.kibana?.alert?.flapping);
+          const result = [false, false, false, false, false].concat(
+            new Array(7).fill(true),
+            false,
+            false,
+            false
+          );
+          expect(flapping).to.eql(result);
+        });
+
+        it('should generate expected events for flapping alerts that settle on recovered where notifyWhen is NOT set to "on status change"', async () => {
+          await supertest
+            .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
+            .set('kbn-xsrf', 'foo')
+            .auth('superuser', 'superuser')
+            .send({
+              enabled: true,
+              look_back_window: 6,
+              status_change_threshold: 4,
+            })
+            .expect(200);
+          const { body: createdAction } = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/actions/connector`)
+            .set('kbn-xsrf', 'foo')
+            .send({
+              name: 'MY action',
+              connector_type_id: 'test.noop',
+              config: {},
+              secrets: {},
+            })
+            .expect(200);
+
+          // pattern of when the alert should fire
+          const instance = [true, false, false, true, false, true, false, true, false, true].concat(
+            new Array(11).fill(false)
+          );
+          const pattern = {
+            instance,
+          };
+
+          const response = await supertest
+            .post(`${getUrlPrefix(space.id)}/api/alerting/rule`)
+            .set('kbn-xsrf', 'foo')
+            .send(
+              getTestRuleData({
+                rule_type_id: 'test.patternFiring',
+                schedule: { interval: '1s' },
+                throttle: '1s',
+                params: {
+                  pattern,
+                },
+                actions: [
+                  {
+                    id: createdAction.id,
+                    group: 'default',
+                    params: {},
+                  },
+                  {
+                    id: createdAction.id,
+                    group: 'recovered',
+                    params: {},
+                  },
+                ],
+              })
+            );
+
+          expect(response.status).to.eql(200);
+          const alertId = response.body.id;
+          objectRemover.add(space.id, alertId, 'rule', 'alerting');
+
+          // get the events we're expecting
+          const events = await retry.try(async () => {
+            return await getEventLog({
+              getService,
+              spaceId: space.id,
+              type: 'alert',
+              id: alertId,
+              provider: 'alerting',
+              actions: new Map([
+                // make sure the counts of the # of events per type are as expected
+                ['execute-start', { gte: 8 }],
+                ['execute', { gte: 8 }],
+                ['execute-action', { equal: 8 }],
+                ['new-instance', { equal: 3 }],
+                ['active-instance', { gte: 3 }],
+                ['recovered-instance', { equal: 3 }],
+              ]),
+            });
+          });
+
+          const flapping = events
+            .filter(
+              (event) =>
+                event?.event?.action === 'active-instance' ||
+                event?.event?.action === 'recovered-instance'
+            )
+            .map((event) => event?.kibana?.alert?.flapping);
+          expect(flapping).to.eql([false, false, false, false, false, true, true, true]);
+        });
+
+        it('should generate expected events for flapping alerts that settle on recovered where the action notifyWhen is NOT set to "on status change"', async () => {
           await supertest
             .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`)
             .set('kbn-xsrf', 'foo')
@@ -1052,6 +1461,7 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
                 rule_type_id: 'test.patternFiring',
                 schedule: { interval: '1s' },
                 throttle: null,
+                notify_when: null,
                 params: {
                   pattern,
                 },
@@ -1060,11 +1470,21 @@ export default function eventLogTests({ getService }: FtrProviderContext) {
                     id: createdAction.id,
                     group: 'default',
                     params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: '1s',
+                      notify_when: RuleNotifyWhen.THROTTLE,
+                    },
                   },
                   {
                     id: createdAction.id,
                     group: 'recovered',
                     params: {},
+                    frequency: {
+                      summary: false,
+                      throttle: '1s',
+                      notify_when: RuleNotifyWhen.THROTTLE,
+                    },
                   },
                 ],
               })