Save ES Query Rule type alerts in alert-as-data index (#161685)

Resolves: #159493

This PR replaces `AlertFactory` in ES Query rule type with
`AlertsClient` so the alerts are persistent in an alert-as-data index.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+// ---------------------------------- WARNING ----------------------------------
+// this file was generated, and should not be edited by hand
+// ---------------------------------- WARNING ----------------------------------
+import * as rt from 'io-ts';
+import { Either } from 'fp-ts/lib/Either';
+import { AlertSchema } from './alert_schema';
+const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/;
+export const IsoDateString = new rt.Type<string, string, unknown>(
+  'IsoDateString',
+  rt.string.is,
+  (input, context): Either<rt.Errors, string> => {
+    if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) {
+      return rt.success(input);
+    } else {
+      return rt.failure(input, context);
+    }
+  },
+  rt.identity
+);
+export type IsoDateStringC = typeof IsoDateString;
+export const schemaDate = IsoDateString;
+export const schemaDateArray = rt.array(IsoDateString);
+export const schemaDateRange = rt.partial({
+  gte: schemaDate,
+  lte: schemaDate,
+});
+export const schemaDateRangeArray = rt.array(schemaDateRange);
+export const schemaUnknown = rt.unknown;
+export const schemaUnknownArray = rt.array(rt.unknown);
+export const schemaString = rt.string;
+export const schemaStringArray = rt.array(schemaString);
+export const schemaNumber = rt.number;
+export const schemaNumberArray = rt.array(schemaNumber);
+export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
+export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
+export const schemaBoolean = rt.boolean;
+export const schemaBooleanArray = rt.array(schemaBoolean);
+const schemaGeoPointCoords = rt.type({
+  type: schemaString,
+  coordinates: schemaNumberArray,
+});
+const schemaGeoPointString = schemaString;
+const schemaGeoPointLatLon = rt.type({
+  lat: schemaNumber,
+  lon: schemaNumber,
+});
+const schemaGeoPointLocation = rt.type({
+  location: schemaNumberArray,
+});
+const schemaGeoPointLocationString = rt.type({
+  location: schemaString,
+});
+export const schemaGeoPoint = rt.union([
+  schemaGeoPointCoords,
+  schemaGeoPointString,
+  schemaGeoPointLatLon,
+  schemaGeoPointLocation,
+  schemaGeoPointLocationString,
+]);
+export const schemaGeoPointArray = rt.array(schemaGeoPoint);
+// prettier-ignore
+const StackAlertRequired = rt.type({
+});
+const StackAlertOptional = rt.partial({
+  kibana: rt.partial({
+    alert: rt.partial({
+      evaluation: rt.partial({
+        conditions: schemaString,
+        value: schemaString,
+      }),
+      title: schemaString,
+    }),
+  }),
+});
+
+// prettier-ignore
+export const StackAlertSchema = rt.intersection([StackAlertRequired, StackAlertOptional, AlertSchema]);
+// prettier-ignore
+export type StackAlert = rt.TypeOf<typeof StackAlertSchema>;

packages/kbn-alerts-as-data-utils/src/schemas/index.ts

@@ -23,6 +23,7 @@ export type { ObservabilityMetricsAlert } from './generated/observability_metric
 export type { ObservabilitySloAlert } from './generated/observability_slo_schema';
 export type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema';
 export type { SecurityAlert } from './generated/security_schema';
+export type { StackAlert } from './generated/stack_schema';
 
 export type AADAlert =
   | Alert

x-pack/plugins/alerting/server/alerts_client/alerts_client.mock.ts

@@ -30,9 +30,11 @@ const createPublicAlertsClientMock = () => {
   return jest.fn().mockImplementation(() => {
     return {
       create: jest.fn(),
-      getAlertLimitValue: jest.fn(),
+      report: jest.fn(),
+      getAlertLimitValue: jest.fn().mockReturnValue(1000),
       setAlertLimitReached: jest.fn(),
-      getRecoveredAlerts: jest.fn(),
+      getRecoveredAlerts: jest.fn().mockReturnValue([]),
+      setAlertData: jest.fn(),
     };
   });
 };

x-pack/plugins/alerting/server/alerts_client/lib/index.ts

@@ -14,4 +14,5 @@ export {
   getHitsWithCount,
   getLifecycleAlertsQueries,
   getContinualAlertsQuery,
+  expandFlattenedAlert,
 } from './get_summarized_alerts_query';

x-pack/plugins/alerting/server/alerts_client/lib/strip_framework_fields.ts

@@ -6,11 +6,16 @@
  */
 
 import { omit } from 'lodash';
-import { ALERT_REASON, ALERT_WORKFLOW_STATUS, TAGS } from '@kbn/rule-data-utils';
+import { ALERT_REASON, ALERT_WORKFLOW_STATUS, TAGS, ALERT_URL } from '@kbn/rule-data-utils';
 import { alertFieldMap } from '@kbn/alerts-as-data-utils';
 import { RuleAlertData } from '../../types';
 
-const allowedFrameworkFields = new Set<string>([ALERT_REASON, ALERT_WORKFLOW_STATUS, TAGS]);
+const allowedFrameworkFields = new Set<string>([
+  ALERT_REASON,
+  ALERT_WORKFLOW_STATUS,
+  TAGS,
+  ALERT_URL,
+]);
 
 /**
  * Remove framework fields from the alert payload reported by

x-pack/plugins/alerting/server/task_runner/execution_handler.ts

@@ -292,7 +292,7 @@ export class ExecutionHandler<
                 alertActionGroupName: this.ruleTypeActionGroups!.get(actionGroup)!,
                 context: alert.getContext(),
                 actionId: action.id,
-                state: alert.getScheduledActionOptions()?.state || {},
+                state: alert.getState(),
                 kibanaBaseUrl: this.taskRunnerContext.kibanaBaseUrl,
                 alertParams: this.rule.params,
                 actionParams: action.params,

x-pack/plugins/stack_alerts/server/rule_types/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const STACK_AAD_INDEX_NAME = 'stack';

x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts

@@ -45,11 +45,19 @@ jest.mock('./lib/fetch_search_source_query', () => ({
     mockFetchSearchSourceQuery(...args),
 }));
 
-const scheduleActions = jest.fn();
-const replaceState = jest.fn(() => ({ scheduleActions }));
-const mockCreateAlert = jest.fn(() => ({ replaceState }));
 const mockGetRecoveredAlerts = jest.fn().mockReturnValue([]);
 const mockSetLimitReached = jest.fn();
+const mockReport = jest.fn();
+const mockSetAlertData = jest.fn();
+const mockGetAlertLimitValue = jest.fn().mockReturnValue(1000);
+
+const mockAlertClient = {
+  report: mockReport,
+  getAlertLimitValue: mockGetAlertLimitValue,
+  setAlertLimitReached: mockSetLimitReached,
+  getRecoveredAlerts: mockGetRecoveredAlerts,
+  setAlertData: mockSetAlertData,
+};
 
 const mockNow = jest.getRealSystemTime();
 
@@ -87,16 +95,7 @@ describe('es_query executor', () => {
         get: () => ({ attributes: { consumer: 'alerts' } }),
       },
       searchSourceClient: searchSourceClientMock,
-      alertFactory: {
+      alertsClient: mockAlertClient,
-        create: mockCreateAlert,
-        alertLimit: {
-          getValue: jest.fn().mockReturnValue(1000),
-          setLimitReached: mockSetLimitReached,
-        },
-        done: () => ({
-          getRecoveredAlerts: mockGetRecoveredAlerts,
-        }),
-      },
       alertWithLifecycle: jest.fn(),
       logger,
       shouldWriteAlerts: () => true,
@@ -210,7 +209,7 @@ describe('es_query executor', () => {
         params: { ...defaultProps, threshold: [500], thresholdComparator: '>=' as Comparator },
       });
 
-      expect(mockCreateAlert).not.toHaveBeenCalled();
+      expect(mockReport).not.toHaveBeenCalled();
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);
     });
@@ -237,10 +236,10 @@ describe('es_query executor', () => {
         params: { ...defaultProps, threshold: [200], thresholdComparator: '>=' as Comparator },
       });
 
-      expect(mockCreateAlert).toHaveBeenCalledTimes(1);
+      expect(mockReport).toHaveBeenCalledTimes(1);
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(1, 'query matched');
+      expect(mockReport).toHaveBeenNthCalledWith(1, {
-      expect(scheduleActions).toHaveBeenCalledTimes(1);
+        actionGroup: 'query matched',
-      expect(scheduleActions).toHaveBeenNthCalledWith(1, 'query matched', {
+        context: {
           conditions: 'Number of matching documents is greater than or equal to 200',
           date: new Date(mockNow).toISOString(),
           hits: [],
@@ -253,6 +252,31 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' matched query",
           value: 491,
+        },
+        id: 'query matched',
+        state: {
+          dateEnd: new Date(mockNow).toISOString(),
+          dateStart: new Date(mockNow).toISOString(),
+          latestTimestamp: undefined,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+              reason: `rule 'test-rule-name' is active:
+
+- Value: 491
+- Conditions Met: Number of matching documents is greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' matched query",
+              evaluation: {
+                conditions: 'Number of matching documents is greater than or equal to 200',
+                value: 491,
+              },
+            },
+          },
+        },
       });
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);
@@ -297,12 +321,10 @@ describe('es_query executor', () => {
         },
       });
 
-      expect(mockCreateAlert).toHaveBeenCalledTimes(3);
+      expect(mockReport).toHaveBeenCalledTimes(3);
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(1, 'host-1');
+      expect(mockReport).toHaveBeenNthCalledWith(1, {
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(2, 'host-2');
+        actionGroup: 'query matched',
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(3, 'host-3');
+        context: {
-      expect(scheduleActions).toHaveBeenCalledTimes(3);
-      expect(scheduleActions).toHaveBeenNthCalledWith(1, 'query matched', {
           conditions:
             'Number of matching documents for group "host-1" is greater than or equal to 200',
           date: new Date(mockNow).toISOString(),
@@ -316,8 +338,36 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' matched query for group host-1",
           value: 291,
+        },
+        id: 'host-1',
+        state: {
+          dateEnd: new Date(mockNow).toISOString(),
+          dateStart: new Date(mockNow).toISOString(),
+          latestTimestamp: undefined,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+              reason: `rule 'test-rule-name' is active:
+
+- Value: 291
+- Conditions Met: Number of matching documents for group "host-1" is greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' matched query for group host-1",
+              evaluation: {
+                conditions:
+                  'Number of matching documents for group "host-1" is greater than or equal to 200',
+                value: 291,
+              },
+            },
+          },
+        },
       });
-      expect(scheduleActions).toHaveBeenNthCalledWith(2, 'query matched', {
+      expect(mockReport).toHaveBeenNthCalledWith(2, {
+        actionGroup: 'query matched',
+        context: {
           conditions:
             'Number of matching documents for group "host-2" is greater than or equal to 200',
           date: new Date(mockNow).toISOString(),
@@ -331,8 +381,36 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' matched query for group host-2",
           value: 477,
+        },
+        id: 'host-2',
+        state: {
+          dateEnd: new Date(mockNow).toISOString(),
+          dateStart: new Date(mockNow).toISOString(),
+          latestTimestamp: undefined,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+              reason: `rule 'test-rule-name' is active:
+
+- Value: 477
+- Conditions Met: Number of matching documents for group "host-2" is greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' matched query for group host-2",
+              evaluation: {
+                conditions:
+                  'Number of matching documents for group "host-2" is greater than or equal to 200',
+                value: 477,
+              },
+            },
+          },
+        },
       });
-      expect(scheduleActions).toHaveBeenNthCalledWith(3, 'query matched', {
+      expect(mockReport).toHaveBeenNthCalledWith(3, {
+        actionGroup: 'query matched',
+        context: {
           conditions:
             'Number of matching documents for group "host-3" is greater than or equal to 200',
           date: new Date(mockNow).toISOString(),
@@ -346,6 +424,32 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' matched query for group host-3",
           value: 999,
+        },
+        id: 'host-3',
+        state: {
+          dateEnd: new Date(mockNow).toISOString(),
+          dateStart: new Date(mockNow).toISOString(),
+          latestTimestamp: undefined,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+              reason: `rule 'test-rule-name' is active:
+
+- Value: 999
+- Conditions Met: Number of matching documents for group \"host-3\" is greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' matched query for group host-3",
+              evaluation: {
+                conditions:
+                  'Number of matching documents for group "host-3" is greater than or equal to 200',
+                value: 999,
+              },
+            },
+          },
+        },
       });
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);
@@ -389,21 +493,20 @@ describe('es_query executor', () => {
         },
       });
 
-      expect(mockCreateAlert).toHaveBeenCalledTimes(3);
+      expect(mockReport).toHaveBeenCalledTimes(3);
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(1, 'host-1');
+      expect(mockReport).toHaveBeenNthCalledWith(1, expect.objectContaining({ id: 'host-1' }));
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(2, 'host-2');
+      expect(mockReport).toHaveBeenNthCalledWith(2, expect.objectContaining({ id: 'host-2' }));
-      expect(mockCreateAlert).toHaveBeenNthCalledWith(3, 'host-3');
+      expect(mockReport).toHaveBeenNthCalledWith(3, expect.objectContaining({ id: 'host-3' }));
-      expect(scheduleActions).toHaveBeenCalledTimes(3);
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(true);
     });
 
     it('should correctly handle recovered alerts for ungrouped alert', async () => {
-      const mockSetContext = jest.fn();
       mockGetRecoveredAlerts.mockReturnValueOnce([
         {
+          alert: {
             getId: () => 'query matched',
-          setContext: mockSetContext,
+          },
         },
       ]);
       mockFetchEsQuery.mockResolvedValueOnce({
@@ -427,9 +530,11 @@ describe('es_query executor', () => {
         params: { ...defaultProps, threshold: [500], thresholdComparator: '>=' as Comparator },
       });
 
-      expect(mockCreateAlert).not.toHaveBeenCalled();
+      expect(mockReport).not.toHaveBeenCalled();
-      expect(mockSetContext).toHaveBeenCalledTimes(1);
+      expect(mockSetAlertData).toHaveBeenCalledTimes(1);
-      expect(mockSetContext).toHaveBeenNthCalledWith(1, {
+      expect(mockSetAlertData).toHaveBeenNthCalledWith(1, {
+        id: 'query matched',
+        context: {
           conditions: 'Number of matching documents is NOT greater than or equal to 500',
           date: new Date(mockNow).toISOString(),
           hits: [],
@@ -442,21 +547,41 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' recovered",
           value: 0,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              evaluation: {
+                conditions: 'Number of matching documents is NOT greater than or equal to 500',
+                value: 0,
+              },
+              reason: `rule 'test-rule-name' is recovered:
+
+- Value: 0
+- Conditions Met: Number of matching documents is NOT greater than or equal to 500 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' recovered",
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+            },
+          },
+        },
       });
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);
     });
 
     it('should correctly handle recovered alerts for grouped alerts', async () => {
-      const mockSetContext = jest.fn();
       mockGetRecoveredAlerts.mockReturnValueOnce([
         {
+          alert: {
             getId: () => 'host-1',
-          setContext: mockSetContext,
+          },
         },
         {
+          alert: {
             getId: () => 'host-2',
-          setContext: mockSetContext,
+          },
         },
       ]);
       mockFetchEsQuery.mockResolvedValueOnce({
@@ -478,9 +603,11 @@ describe('es_query executor', () => {
         },
       });
 
-      expect(mockCreateAlert).not.toHaveBeenCalled();
+      expect(mockReport).not.toHaveBeenCalled();
-      expect(mockSetContext).toHaveBeenCalledTimes(2);
+      expect(mockSetAlertData).toHaveBeenCalledTimes(2);
-      expect(mockSetContext).toHaveBeenNthCalledWith(1, {
+      expect(mockSetAlertData).toHaveBeenNthCalledWith(1, {
+        id: 'host-1',
+        context: {
           conditions: `Number of matching documents for group "host-1" is NOT greater than or equal to 200`,
           date: new Date(mockNow).toISOString(),
           hits: [],
@@ -493,8 +620,30 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' recovered",
           value: 0,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              evaluation: {
+                conditions:
+                  'Number of matching documents for group "host-1" is NOT greater than or equal to 200',
+                value: 0,
+              },
+              reason: `rule 'test-rule-name' is recovered:
+
+- Value: 0
+- Conditions Met: Number of matching documents for group \"host-1\" is NOT greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' recovered",
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+            },
+          },
+        },
       });
-      expect(mockSetContext).toHaveBeenNthCalledWith(2, {
+      expect(mockSetAlertData).toHaveBeenNthCalledWith(2, {
+        id: 'host-2',
+        context: {
           conditions: `Number of matching documents for group "host-2" is NOT greater than or equal to 200`,
           date: new Date(mockNow).toISOString(),
           hits: [],
@@ -507,6 +656,26 @@ describe('es_query executor', () => {
 - Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
           title: "rule 'test-rule-name' recovered",
           value: 0,
+        },
+        payload: {
+          kibana: {
+            alert: {
+              evaluation: {
+                conditions:
+                  'Number of matching documents for group "host-2" is NOT greater than or equal to 200',
+                value: 0,
+              },
+              reason: `rule 'test-rule-name' is recovered:
+
+- Value: 0
+- Conditions Met: Number of matching documents for group \"host-2\" is NOT greater than or equal to 200 over 5m
+- Timestamp: ${new Date(mockNow).toISOString()}
+- Link: https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id`,
+              title: "rule 'test-rule-name' recovered",
+              url: 'https://localhost:5601/app/management/insightsAndAlerting/triggersActions/rule/test-rule-id',
+            },
+          },
+        },
       });
       expect(mockSetLimitReached).toHaveBeenCalledTimes(1);
       expect(mockSetLimitReached).toHaveBeenCalledWith(false);

x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts

@@ -9,6 +9,10 @@ import { i18n } from '@kbn/i18n';
 import { CoreSetup } from '@kbn/core/server';
 import { parseDuration } from '@kbn/alerting-plugin/server';
 import { isGroupAggregation, UngroupedGroupId } from '@kbn/triggers-actions-ui-plugin/common';
+import { ALERT_EVALUATION_VALUE, ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils';
+
+import { expandFlattenedAlert } from '@kbn/alerting-plugin/server/alerts_client/lib';
+import { ALERT_TITLE, ALERT_EVALUATION_CONDITIONS } from './fields';
 import { ComparatorFns } from '../../../common';
 import {
   addMessages,
@@ -32,11 +36,11 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
     spaceId,
     logger,
   } = options;
-  const { alertFactory, scopedClusterClient, searchSourceClient, share, dataViews } = services;
+  const { alertsClient, scopedClusterClient, searchSourceClient, share, dataViews } = services;
   const currentTimestamp = new Date().toISOString();
   const publicBaseUrl = core.http.basePath.publicBaseUrl ?? '';
   const spacePrefix = spaceId !== 'default' ? `/s/${spaceId}` : '';
-  const alertLimit = alertFactory.alertLimit.getValue();
+  const alertLimit = alertsClient?.getAlertLimitValue();
   const compareFn = ComparatorFns.get(params.thresholdComparator);
   if (compareFn == null) {
     throw new Error(getInvalidComparatorError(params.thresholdComparator));
@@ -108,19 +112,29 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
         ...(isGroupAgg ? { group: alertId } : {}),
       }),
     } as EsQueryRuleActionContext;
+
     const actionContext = addMessages({
       ruleName: name,
       baseContext: baseActiveContext,
       params,
       ...(isGroupAgg ? { group: alertId } : {}),
     });
-    const alert = alertFactory.create(
+
-      alertId === UngroupedGroupId && !isGroupAgg ? ConditionMetAlertInstanceId : alertId
+    const id = alertId === UngroupedGroupId && !isGroupAgg ? ConditionMetAlertInstanceId : alertId;
-    );
+
-    alert
+    alertsClient!.report({
-      // store the params we would need to recreate the query that led to this alert instance
+      id,
-      .replaceState({ latestTimestamp, dateStart, dateEnd })
+      actionGroup: ActionGroupId,
-      .scheduleActions(ActionGroupId, actionContext);
+      state: { latestTimestamp, dateStart, dateEnd },
+      context: actionContext,
+      payload: expandFlattenedAlert({
+        [ALERT_URL]: actionContext.link,
+        [ALERT_REASON]: actionContext.message,
+        [ALERT_TITLE]: actionContext.title,
+        [ALERT_EVALUATION_CONDITIONS]: actionContext.conditions,
+        [ALERT_EVALUATION_VALUE]: actionContext.value,
+      }),
+    });
     if (!isGroupAgg) {
       // update the timestamp based on the current search results
       const firstValidTimefieldSort = getValidTimefieldSort(
@@ -131,12 +145,11 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
       }
     }
   }
+  alertsClient!.setAlertLimitReached(parsedResults.truncated);
 
-  alertFactory.alertLimit.setLimitReached(parsedResults.truncated);
+  const { getRecoveredAlerts } = alertsClient!;
-
-  const { getRecoveredAlerts } = alertFactory.done();
   for (const recoveredAlert of getRecoveredAlerts()) {
-    const alertId = recoveredAlert.getId();
+    const alertId = recoveredAlert.alert.getId();
     const baseRecoveryContext: EsQueryRuleActionContext = {
       title: name,
       date: currentTimestamp,
@@ -159,7 +172,17 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
       isRecovered: true,
       ...(isGroupAgg ? { group: alertId } : {}),
     });
-    recoveredAlert.setContext(recoveryContext);
+    alertsClient?.setAlertData({
+      id: alertId,
+      context: recoveryContext,
+      payload: expandFlattenedAlert({
+        [ALERT_URL]: recoveryContext.link,
+        [ALERT_REASON]: recoveryContext.message,
+        [ALERT_TITLE]: recoveryContext.title,
+        [ALERT_EVALUATION_CONDITIONS]: recoveryContext.conditions,
+        [ALERT_EVALUATION_VALUE]: recoveryContext.value,
+      }),
+    });
   }
   return { state: { latestTimestamp } };
 }

x-pack/plugins/stack_alerts/server/rule_types/es_query/fields.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
+
+const ALERT_TITLE = `${ALERT_NAMESPACE}.title` as const;
+// kibana.alert.evaluation.conditions - human readable string that shows the consditions set by the user
+const ALERT_EVALUATION_CONDITIONS = `${ALERT_NAMESPACE}.evaluation.conditions` as const;
+
+export { ALERT_TITLE, ALERT_EVALUATION_CONDITIONS };

x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts

@@ -8,11 +8,7 @@
 import { v4 as uuidv4 } from 'uuid';
 import type { Writable } from '@kbn/utility-types';
 import { RuleExecutorServices } from '@kbn/alerting-plugin/server';
-import {
+import { RuleExecutorServicesMock, alertsMock } from '@kbn/alerting-plugin/server/mocks';
-  RuleExecutorServicesMock,
-  alertsMock,
-  AlertInstanceMock,
-} from '@kbn/alerting-plugin/server/mocks';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import type { DataViewSpec } from '@kbn/data-views-plugin/common';
 import { createStubDataView } from '@kbn/data-views-plugin/common/data_view.stub';
@@ -31,8 +27,23 @@ import { DEFAULT_FLAPPING_SETTINGS } from '@kbn/alerting-plugin/common/rules_set
 const logger = loggingSystemMock.create().get();
 const coreSetup = coreMock.createSetup();
 const ruleType = getRuleType(coreSetup);
+const mockNow = jest.getRealSystemTime();
 
 describe('ruleType', () => {
+  beforeAll(() => {
+    jest.useFakeTimers();
+    jest.setSystemTime(mockNow);
+  });
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  afterAll(() => {
+    jest.useRealTimers();
+  });
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
   it('rule type creation structure is the expected value', async () => {
     expect(ruleType.id).toBe('.es-query');
     expect(ruleType.name).toBe('Elasticsearch query');
@@ -168,7 +179,7 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      expect(ruleServices.alertFactory.create).not.toHaveBeenCalled();
+      expect(ruleServices.alertsClient.report).not.toHaveBeenCalled();
 
       expect(result).toMatchInlineSnapshot(`
         Object {
@@ -215,13 +226,17 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      expect(ruleServices.alertFactory.create).toHaveBeenCalledWith(ConditionMetAlertInstanceId);
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+        expect.objectContaining({
-      expect(instance.replaceState).toHaveBeenCalledWith({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: undefined,
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result).toMatchObject({
         state: {
@@ -269,13 +284,18 @@ describe('ruleType', () => {
         },
       });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      expect(instance.replaceState).toHaveBeenCalledWith({
+        expect.objectContaining({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             // ensure the invalid "latestTimestamp" in the state is stored as an ISO string going forward
             latestTimestamp: new Date(previousTimestamp).toISOString(),
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result).toMatchObject({
         state: {
@@ -318,12 +338,17 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      expect(instance.replaceState).toHaveBeenCalledWith({
+        expect.objectContaining({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: undefined,
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result).toMatchObject({
         state: {
@@ -363,12 +388,17 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      expect(instance.replaceState).toHaveBeenCalledWith({
+        expect.objectContaining({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: undefined,
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result?.state).toMatchObject({
         latestTimestamp: new Date(oldestDocumentTimestamp).toISOString(),
@@ -394,13 +424,17 @@ describe('ruleType', () => {
         state: result?.state as EsQueryRuleState,
       });
 
-      const existingInstance: AlertInstanceMock =
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-        ruleServices.alertFactory.create.mock.results[1].value;
+        expect.objectContaining({
-      expect(existingInstance.replaceState).toHaveBeenCalledWith({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: new Date(oldestDocumentTimestamp).toISOString(),
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(secondResult).toMatchObject({
         state: {
@@ -446,12 +480,17 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      expect(instance.replaceState).toHaveBeenCalledWith({
+        expect.objectContaining({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: undefined,
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result).toMatchObject({
         state: {
@@ -498,12 +537,17 @@ describe('ruleType', () => {
 
       const result = await invokeExecutor({ params, ruleServices });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
-      expect(instance.replaceState).toHaveBeenCalledWith({
+        expect.objectContaining({
+          id: ConditionMetAlertInstanceId,
+          actionGroup: ActionGroupId,
+          state: {
             latestTimestamp: undefined,
             dateStart: expect.any(String),
             dateEnd: expect.any(String),
-      });
+          },
+        })
+      );
 
       expect(result).toMatchObject({
         state: {
@@ -599,7 +643,7 @@ describe('ruleType', () => {
 
       await invokeExecutor({ params, ruleServices });
 
-      expect(ruleServices.alertFactory.create).not.toHaveBeenCalled();
+      expect(ruleServices.alertsClient.report).not.toHaveBeenCalled();
     });
 
     it('rule executor throws an error when index does not have time field', async () => {
@@ -637,10 +681,33 @@ describe('ruleType', () => {
         hits: { total: 3, hits: [{}, {}, {}] },
       });
 
-      await invokeExecutor({ params, ruleServices });
+      await invokeExecutor({
+        params,
+        ruleServices,
+        state: { latestTimestamp: new Date(mockNow).toISOString(), dateStart: '', dateEnd: '' },
+      });
 
-      const instance: AlertInstanceMock = ruleServices.alertFactory.create.mock.results[0].value;
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledTimes(1);
-      expect(instance.scheduleActions).toHaveBeenCalled();
+
+      expect(ruleServices.alertsClient.report).toHaveBeenCalledWith(
+        expect.objectContaining({
+          actionGroup: 'query matched',
+          id: 'query matched',
+          payload: expect.objectContaining({
+            kibana: {
+              alert: {
+                url: expect.any(String),
+                reason: expect.any(String),
+                title: "rule 'rule-name' matched query",
+                evaluation: {
+                  conditions: 'Number of matching documents is greater than or equal to 3',
+                  value: 3,
+                },
+              },
+            },
+          }),
+        })
+      );
     });
   });
 });
@@ -711,7 +778,7 @@ async function invokeExecutor({
     spaceId: uuidv4(),
     rule: {
       id: uuidv4(),
-      name: uuidv4(),
+      name: 'rule-name',
       tags: [],
       consumer: '',
       producer: '',

x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.ts

@@ -8,6 +8,11 @@
 import { i18n } from '@kbn/i18n';
 import { CoreSetup } from '@kbn/core/server';
 import { extractReferences, injectReferences } from '@kbn/data-plugin/common';
+import { IRuleTypeAlerts } from '@kbn/alerting-plugin/server';
+import { ALERT_EVALUATION_VALUE } from '@kbn/rule-data-utils';
+import { StackAlert } from '@kbn/alerts-as-data-utils';
+import { STACK_AAD_INDEX_NAME } from '..';
+import { ALERT_TITLE, ALERT_EVALUATION_CONDITIONS } from './fields';
 import { RuleType } from '../../types';
 import { ActionContext } from './action_context';
 import {
@@ -30,7 +35,9 @@ export function getRuleType(
   EsQueryRuleState,
   {},
   ActionContext,
-  typeof ActionGroupId
+  typeof ActionGroupId,
+  never,
+  StackAlert
 > {
   const ruleTypeName = i18n.translate('xpack.stackAlerts.esQuery.alertTypeTitle', {
     defaultMessage: 'Elasticsearch query',
@@ -135,6 +142,18 @@ export function getRuleType(
     }
   );
 
+  const alerts: IRuleTypeAlerts<StackAlert> = {
+    context: STACK_AAD_INDEX_NAME,
+    mappings: {
+      fieldMap: {
+        [ALERT_TITLE]: { type: 'keyword', array: false, required: false },
+        [ALERT_EVALUATION_CONDITIONS]: { type: 'keyword', array: false, required: false },
+        [ALERT_EVALUATION_VALUE]: { type: 'keyword', array: false, required: false },
+      },
+    },
+    shouldWrite: true,
+  };
+
   return {
     id: ES_QUERY_ID,
     name: ruleTypeName,
@@ -188,5 +207,6 @@ export function getRuleType(
     },
     producer: STACK_ALERTS_FEATURE_ID,
     doesSetRecoveryContext: true,
+    alerts,
   };
 }

x-pack/plugins/stack_alerts/server/rule_types/es_query/types.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { StackAlert } from '@kbn/alerts-as-data-utils';
 import { RuleExecutorOptions, RuleTypeParams } from '../../types';
 import { ActionContext } from './action_context';
 import { EsQueryRuleParams, EsQueryRuleState } from './rule_type_params';
@@ -24,5 +25,6 @@ export type ExecutorOptions<P extends RuleTypeParams> = RuleExecutorOptions<
   EsQueryRuleState,
   {},
   ActionContext,
-  typeof ActionGroupId
+  typeof ActionGroupId,
+  StackAlert
 >;

x-pack/plugins/stack_alerts/server/rule_types/index.ts

@@ -10,6 +10,8 @@ import { register as registerIndexThreshold } from './index_threshold';
 import { register as registerGeoContainment } from './geo_containment';
 import { register as registerEsQuery } from './es_query';
 
+export * from './constants';
+
 export function registerBuiltInRuleTypes(params: RegisterRuleTypesParams) {
   registerIndexThreshold(params);
   registerGeoContainment(params);

x-pack/plugins/stack_alerts/tsconfig.json

@@ -42,6 +42,8 @@
     "@kbn/logging-mocks",
     "@kbn/share-plugin",
     "@kbn/discover-plugin",
+    "@kbn/rule-data-utils",
+    "@kbn/alerts-as-data-utils",
   ],
   "exclude": [
     "target/**/*",

x-pack/test/alerting_api_integration/packages/helpers/es_test_index_tool.ts

@@ -110,6 +110,31 @@ export class ESTestIndexTool {
     return await this.es.search(params, { meta: true });
   }
 
+  async getAll(size: number = 10) {
+    const params = {
+      index: this.index,
+      size,
+      body: {
+        query: {
+          match_all: {},
+        },
+      },
+    };
+    return await this.es.search(params, { meta: true });
+  }
+
+  async removeAll() {
+    const params = {
+      index: this.index,
+      body: {
+        query: {
+          match_all: {},
+        },
+      },
+    };
+    return await this.es.deleteByQuery(params);
+  }
+
   async waitForDocs(source: string, reference: string, numDocs: number = 1) {
     return await this.retry.try(async () => {
       const searchResult = await this.search(source, reference);

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/common.ts

@@ -6,6 +6,7 @@
  */
 
 import { ESTestIndexTool, ES_TEST_INDEX_NAME } from '@kbn/alerting-api-integration-helpers';
+import { STACK_AAD_INDEX_NAME } from '@kbn/stack-alerts-plugin/server/rule_types';
 import { FtrProviderContext } from '../../../../../../common/ftr_provider_context';
 import { Spaces } from '../../../../../scenarios';
 import { getUrlPrefix, ObjectRemover } from '../../../../../../common/lib';
@@ -69,6 +70,11 @@ export function getRuleServices(getService: FtrProviderContext['getService']) {
   const esTestIndexTool = new ESTestIndexTool(es, retry);
   const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME);
   const esTestIndexToolDataStream = new ESTestIndexTool(es, retry, ES_TEST_DATA_STREAM_NAME);
+  const esTestIndexToolAAD = new ESTestIndexTool(
+    es,
+    retry,
+    `.internal.alerts-${STACK_AAD_INDEX_NAME}.alerts-default-000001`
+  );
 
   async function createEsDocumentsInGroups(
     groups: number,
@@ -112,6 +118,14 @@ export function getRuleServices(getService: FtrProviderContext['getService']) {
     );
   }
 
+  async function getAllAADDocs(size: number): Promise<any> {
+    return await esTestIndexToolAAD.getAll(size);
+  }
+
+  async function removeAllAADDocs(): Promise<any> {
+    return await esTestIndexToolAAD.removeAll();
+  }
+
   return {
     retry,
     es,
@@ -121,5 +135,7 @@ export function getRuleServices(getService: FtrProviderContext['getService']) {
     createEsDocumentsInGroups,
     createGroupedEsDocumentsInGroups,
     waitForDocs,
+    getAllAADDocs,
+    removeAllAADDocs,
   };
 }

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/rule.ts

@@ -38,6 +38,8 @@ export default function ruleTests({ getService }: FtrProviderContext) {
     esTestIndexToolDataStream,
     createEsDocumentsInGroups,
     createGroupedEsDocumentsInGroups,
+    removeAllAADDocs,
+    getAllAADDocs,
   } = getRuleServices(getService);
 
   describe('rule', async () => {
@@ -66,6 +68,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       await esTestIndexTool.destroy();
       await esTestIndexToolOutput.destroy();
       await deleteDataStream(es, ES_TEST_DATA_STREAM_NAME);
+      await removeAllAADDocs();
     });
 
     [
@@ -135,6 +138,9 @@ export default function ruleTests({ getService }: FtrProviderContext) {
         await initData();
 
         const docs = await waitForDocs(2);
+        const messagePattern =
+          /rule 'always fire' is active:\n\n- Value: \d+\n- Conditions Met: Number of matching documents is greater than -1 over 20s\n- Timestamp: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
+
         for (let i = 0; i < docs.length; i++) {
           const doc = docs[i];
           const { previousTimestamp, hits } = doc._source;
@@ -142,8 +148,6 @@ export default function ruleTests({ getService }: FtrProviderContext) {
 
           expect(name).to.be('always fire');
           expect(title).to.be(`rule 'always fire' matched query`);
-          const messagePattern =
-            /rule 'always fire' is active:\n\n- Value: \d+\n- Conditions Met: Number of matching documents is greater than -1 over 20s\n- Timestamp: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
           expect(message).to.match(messagePattern);
           expect(hits).not.to.be.empty();
 
@@ -155,6 +159,17 @@ export default function ruleTests({ getService }: FtrProviderContext) {
             expect(previousTimestamp).not.to.be.empty();
           }
         }
+
+        const aadDocs = await getAllAADDocs(1);
+
+        const alertDoc = aadDocs.body.hits.hits[0]._source.kibana.alert;
+        expect(alertDoc.reason).to.match(messagePattern);
+        expect(alertDoc.title).to.be("rule 'always fire' matched query");
+        expect(alertDoc.evaluation.conditions).to.be(
+          'Number of matching documents is greater than -1'
+        );
+        expect(alertDoc.evaluation.value).greaterThan(0);
+        expect(alertDoc.url).to.contain('/s/space1/app/');
       })
     );
 

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts

@@ -84,7 +84,13 @@ export default function checkAlertSchemasTest({ getService }: FtrProviderContext
           }
         });
 
-      const { stdout } = await execa('git', ['ls-files', '--modified']);
+      const { stdout } = await execa('git', [
+        'ls-files',
+        '--modified',
+        '--others',
+        '--exclude-standard',
+      ]);
+
       expect(stdout).not.to.contain('packages/kbn-alerts-as-data-utils/src/schemas/generated');
     });
   });

x-pack/test/tsconfig.json

@@ -138,6 +138,7 @@
     "@kbn/ml-category-validator",
     "@kbn/observability-ai-assistant-plugin",
     "@kbn/stack-connectors-plugin",
+    "@kbn/stack-alerts-plugin",
     "@kbn/aiops-utils"
   ]
 }