Implement writing rule execution events to event_log (#112286)

x-pack/plugins/event_log/generated/mappings.json

@@ -267,6 +267,42 @@
                         }
                     }
                 },
+                "alert": {
+                    "properties": {
+                        "rule": {
+                            "properties": {
+                                "execution": {
+                                    "properties": {
+                                        "uuid": {
+                                            "type": "keyword",
+                                            "ignore_above": 1024
+                                        },
+                                        "status": {
+                                            "type": "keyword",
+                                            "ignore_above": 1024
+                                        },
+                                        "status_order": {
+                                            "type": "long"
+                                        },
+                                        "metrics": {
+                                            "properties": {
+                                                "total_indexing_duration_ms": {
+                                                    "type": "long"
+                                                },
+                                                "total_search_duration_ms": {
+                                                    "type": "long"
+                                                },
+                                                "execution_gap_duration_s": {
+                                                    "type": "long"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
                 "saved_objects": {
                     "type": "nested",
                     "properties": {
@@ -292,6 +328,13 @@
                         }
                     }
                 },
+                "space_ids": {
+                    "type": "keyword",
+                    "ignore_above": 1024,
+                    "meta": {
+                        "isArray": "true"
+                    }
+                },
                 "version": {
                     "type": "version"
                 }

x-pack/plugins/event_log/generated/schemas.ts

@@ -116,6 +116,28 @@ export const EventSchema = schema.maybe(
             status: ecsString(),
           })
         ),
+        alert: schema.maybe(
+          schema.object({
+            rule: schema.maybe(
+              schema.object({
+                execution: schema.maybe(
+                  schema.object({
+                    uuid: ecsString(),
+                    status: ecsString(),
+                    status_order: ecsNumber(),
+                    metrics: schema.maybe(
+                      schema.object({
+                        total_indexing_duration_ms: ecsNumber(),
+                        total_search_duration_ms: ecsNumber(),
+                        execution_gap_duration_s: ecsNumber(),
+                      })
+                    ),
+                  })
+                ),
+              })
+            ),
+          })
+        ),
         saved_objects: schema.maybe(
           schema.arrayOf(
             schema.object({
@@ -127,6 +149,7 @@ export const EventSchema = schema.maybe(
             })
           )
         ),
+        space_ids: ecsStringMulti(),
         version: ecsVersion(),
       })
     ),

x-pack/plugins/event_log/scripts/mappings.js

@@ -49,6 +49,42 @@ exports.EcsCustomPropertyMappings = {
           },
         },
       },
+      alert: {
+        properties: {
+          rule: {
+            properties: {
+              execution: {
+                properties: {
+                  uuid: {
+                    type: 'keyword',
+                    ignore_above: 1024,
+                  },
+                  status: {
+                    type: 'keyword',
+                    ignore_above: 1024,
+                  },
+                  status_order: {
+                    type: 'long',
+                  },
+                  metrics: {
+                    properties: {
+                      total_indexing_duration_ms: {
+                        type: 'long',
+                      },
+                      total_search_duration_ms: {
+                        type: 'long',
+                      },
+                      execution_gap_duration_s: {
+                        type: 'long',
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
       // array of saved object references, for "linking" via search
       saved_objects: {
         type: 'nested',
@@ -77,6 +113,10 @@ exports.EcsCustomPropertyMappings = {
           },
         },
       },
+      space_ids: {
+        type: 'keyword',
+        ignore_above: 1024,
+      },
       version: {
         type: 'version',
       },
@@ -105,4 +145,10 @@ exports.EcsPropertiesToGenerate = [
 /**
  * These properties can have multiple values (are arrays in the generated event schema).
  */
-exports.EcsEventLogMultiValuedProperties = ['tags', 'event.category', 'event.type', 'rule.author'];
+exports.EcsEventLogMultiValuedProperties = [
+  'tags',
+  'event.category',
+  'event.type',
+  'rule.author',
+  'kibana.space_ids',
+];

x-pack/plugins/security_solution/kibana.json

@@ -12,15 +12,16 @@
     "actions",
     "alerting",
     "cases",
-    "ruleRegistry",
     "data",
     "dataEnhanced",
     "embeddable",
+    "eventLog",
     "features",
-    "taskManager",
     "inspector",
     "licensing",
     "maps",
+    "ruleRegistry",
+    "taskManager",
     "timelines",
     "triggersActionsUi",
     "uiActions"

x-pack/plugins/security_solution/server/config.ts

@@ -12,6 +12,7 @@ import {
   getExperimentalAllowedValues,
   isValidExperimentalValue,
 } from '../common/experimental_features';
+import { UnderlyingLogClient } from './lib/detection_engine/rule_execution_log/types';
 
 const allowedExperimentalValues = getExperimentalAllowedValues();
 
@@ -103,6 +104,19 @@ export const configSchema = schema.object({
     },
   }),
 
+  /**
+   * Rule Execution Log Configuration
+   */
+  ruleExecutionLog: schema.object({
+    underlyingClient: schema.oneOf(
+      [
+        schema.literal(UnderlyingLogClient.eventLog),
+        schema.literal(UnderlyingLogClient.savedObjects),
+      ],
+      { defaultValue: UnderlyingLogClient.savedObjects }
+    ),
+  }),
+
   /**
    * Host Endpoint Configuration
    */

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts

@@ -11,6 +11,7 @@ import { serverMock } from './server';
 import { requestMock } from './request';
 import { responseMock } from './response_factory';
 import { ConfigType } from '../../../../config';
+import { UnderlyingLogClient } from '../../rule_execution_log/types';
 
 export { requestMock, requestContextMock, responseMock, serverMock };
 
@@ -29,6 +30,9 @@ export const createMockConfig = (): ConfigType => ({
   alertIgnoreFields: [],
   prebuiltRulesFromFileSystem: true,
   prebuiltRulesFromSavedObjects: false,
+  ruleExecutionLog: {
+    underlyingClient: UnderlyingLogClient.savedObjects,
+  },
 });
 
 export const mockGetCurrentUser = {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts

@@ -14,7 +14,7 @@ export const ruleExecutionLogClientMock = {
     update: jest.fn(),
     delete: jest.fn(),
     logStatusChange: jest.fn(),
-    logExecutionMetric: jest.fn(),
+    logExecutionMetrics: jest.fn(),
   }),
 };
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const RULE_EXECUTION_LOG_PROVIDER = 'rule-execution.security';
+
+export const ALERT_SAVED_OBJECT_TYPE = 'alert';
+
+export enum RuleExecutionLogAction {
+  'status-change' = 'status-change',
+  'execution-metrics' = 'execution-metrics',
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IEventLogService } from '../../../../../../event_log/server';
+import {
+  FindBulkExecutionLogArgs,
+  FindExecutionLogArgs,
+  IRuleExecutionLogClient,
+  LogExecutionMetricsArgs,
+  LogStatusChangeArgs,
+  UpdateExecutionLogArgs,
+} from '../types';
+import { EventLogClient } from './event_log_client';
+
+export class EventLogAdapter implements IRuleExecutionLogClient {
+  private eventLogClient: EventLogClient;
+
+  constructor(eventLogService: IEventLogService) {
+    this.eventLogClient = new EventLogClient(eventLogService);
+  }
+
+  public async find({ ruleId, logsCount = 1, spaceId }: FindExecutionLogArgs) {
+    return []; // TODO Implement
+  }
+
+  public async findBulk({ ruleIds, logsCount = 1, spaceId }: FindBulkExecutionLogArgs) {
+    return {}; // TODO Implement
+  }
+
+  public async update({ attributes, spaceId, ruleName, ruleType }: UpdateExecutionLogArgs) {
+    // execution events are immutable, so we just log a status change istead of updating previous
+    if (attributes.status) {
+      this.eventLogClient.logStatusChange({
+        ruleName,
+        ruleType,
+        ruleId: attributes.alertId,
+        newStatus: attributes.status,
+        spaceId,
+      });
+    }
+  }
+
+  public async delete(id: string) {
+    // execution events are immutable, nothing to do here
+  }
+
+  public async logExecutionMetrics({
+    ruleId,
+    spaceId,
+    ruleType,
+    ruleName,
+    metrics,
+  }: LogExecutionMetricsArgs) {
+    this.eventLogClient.logExecutionMetrics({
+      ruleId,
+      ruleName,
+      ruleType,
+      spaceId,
+      metrics: {
+        executionGapDuration: metrics.executionGap?.asSeconds(),
+        totalIndexingDuration: metrics.indexingDurations?.reduce(
+          (acc, cur) => acc + Number(cur),
+          0
+        ),
+        totalSearchDuration: metrics.searchDurations?.reduce((acc, cur) => acc + Number(cur), 0),
+      },
+    });
+  }
+
+  public async logStatusChange(args: LogStatusChangeArgs) {
+    if (args.metrics) {
+      this.logExecutionMetrics({
+        ruleId: args.ruleId,
+        ruleName: args.ruleName,
+        ruleType: args.ruleType,
+        spaceId: args.spaceId,
+        metrics: args.metrics,
+      });
+    }
+
+    this.eventLogClient.logStatusChange(args);
+  }
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsUtils } from '../../../../../../../../src/core/server';
+import {
+  IEventLogger,
+  IEventLogService,
+  SAVED_OBJECT_REL_PRIMARY,
+} from '../../../../../../event_log/server';
+import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas';
+import { LogStatusChangeArgs } from '../types';
+import {
+  RuleExecutionLogAction,
+  RULE_EXECUTION_LOG_PROVIDER,
+  ALERT_SAVED_OBJECT_TYPE,
+} from './constants';
+
+const spaceIdToNamespace = SavedObjectsUtils.namespaceStringToId;
+
+const statusSeverityDict: Record<RuleExecutionStatus, number> = {
+  [RuleExecutionStatus.succeeded]: 0,
+  [RuleExecutionStatus['going to run']]: 10,
+  [RuleExecutionStatus.warning]: 20,
+  [RuleExecutionStatus['partial failure']]: 20,
+  [RuleExecutionStatus.failed]: 30,
+};
+
+interface FindExecutionLogArgs {
+  ruleIds: string[];
+  spaceId: string;
+  logsCount?: number;
+  statuses?: RuleExecutionStatus[];
+}
+
+interface LogExecutionMetricsArgs {
+  ruleId: string;
+  ruleName: string;
+  ruleType: string;
+  spaceId: string;
+  metrics: EventLogExecutionMetrics;
+}
+
+interface EventLogExecutionMetrics {
+  totalSearchDuration?: number;
+  totalIndexingDuration?: number;
+  executionGapDuration?: number;
+}
+
+interface IExecLogEventLogClient {
+  find: (args: FindExecutionLogArgs) => Promise<{}>;
+  logStatusChange: (args: LogStatusChangeArgs) => void;
+  logExecutionMetrics: (args: LogExecutionMetricsArgs) => void;
+}
+
+export class EventLogClient implements IExecLogEventLogClient {
+  private sequence = 0;
+  private eventLogger: IEventLogger;
+
+  constructor(eventLogService: IEventLogService) {
+    this.eventLogger = eventLogService.getLogger({
+      event: { provider: RULE_EXECUTION_LOG_PROVIDER },
+    });
+  }
+
+  public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) {
+    return {}; // TODO implement
+  }
+
+  public logExecutionMetrics({
+    ruleId,
+    ruleName,
+    ruleType,
+    metrics,
+    spaceId,
+  }: LogExecutionMetricsArgs) {
+    this.eventLogger.logEvent({
+      rule: {
+        id: ruleId,
+        name: ruleName,
+        category: ruleType,
+      },
+      event: {
+        kind: 'metric',
+        action: RuleExecutionLogAction['execution-metrics'],
+        sequence: this.sequence++,
+      },
+      kibana: {
+        alert: {
+          rule: {
+            execution: {
+              metrics: {
+                execution_gap_duration_s: metrics.executionGapDuration,
+                total_search_duration_ms: metrics.totalSearchDuration,
+                total_indexing_duration_ms: metrics.totalIndexingDuration,
+              },
+            },
+          },
+        },
+        space_ids: [spaceId],
+        saved_objects: [
+          {
+            rel: SAVED_OBJECT_REL_PRIMARY,
+            type: ALERT_SAVED_OBJECT_TYPE,
+            id: ruleId,
+            namespace: spaceIdToNamespace(spaceId),
+          },
+        ],
+      },
+    });
+  }
+
+  public logStatusChange({
+    ruleId,
+    ruleName,
+    ruleType,
+    newStatus,
+    message,
+    spaceId,
+  }: LogStatusChangeArgs) {
+    this.eventLogger.logEvent({
+      rule: {
+        id: ruleId,
+        name: ruleName,
+        category: ruleType,
+      },
+      event: {
+        kind: 'event',
+        action: RuleExecutionLogAction['status-change'],
+        sequence: this.sequence++,
+      },
+      message,
+      kibana: {
+        alert: {
+          rule: {
+            execution: {
+              status: newStatus,
+              status_order: statusSeverityDict[newStatus],
+            },
+          },
+        },
+        space_ids: [spaceId],
+        saved_objects: [
+          {
+            rel: SAVED_OBJECT_REL_PRIMARY,
+            type: ALERT_SAVED_OBJECT_TYPE,
+            id: ruleId,
+            namespace: spaceIdToNamespace(spaceId),
+          },
+        ],
+      },
+    });
+  }
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IEventLogService } from '../../../../../../event_log/server';
+import { RuleExecutionLogAction, RULE_EXECUTION_LOG_PROVIDER } from './constants';
+
+export const registerEventLogProvider = (eventLogService: IEventLogService) => {
+  eventLogService.registerProviderActions(
+    RULE_EXECUTION_LOG_PROVIDER,
+    Object.keys(RuleExecutionLogAction)
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts

@@ -6,34 +6,40 @@
  */
 
 import { SavedObjectsClientContract } from '../../../../../../../src/core/server';
-import { RuleRegistryAdapter } from './rule_registry_adapter/rule_registry_adapter';
+import { IEventLogService } from '../../../../../event_log/server';
+import { EventLogAdapter } from './event_log_adapter/event_log_adapter';
 import { SavedObjectsAdapter } from './saved_objects_adapter/saved_objects_adapter';
 import {
-  ExecutionMetric,
-  ExecutionMetricArgs,
+  LogExecutionMetricsArgs,
   FindBulkExecutionLogArgs,
   FindExecutionLogArgs,
-  IRuleDataPluginService,
   IRuleExecutionLogClient,
   LogStatusChangeArgs,
   UpdateExecutionLogArgs,
+  UnderlyingLogClient,
 } from './types';
 
 export interface RuleExecutionLogClientArgs {
-  ruleDataService: IRuleDataPluginService;
   savedObjectsClient: SavedObjectsClientContract;
+  eventLogService: IEventLogService;
+  underlyingClient: UnderlyingLogClient;
 }
 
-const RULE_REGISTRY_LOG_ENABLED = false;
-
 export class RuleExecutionLogClient implements IRuleExecutionLogClient {
   private client: IRuleExecutionLogClient;
 
-  constructor({ ruleDataService, savedObjectsClient }: RuleExecutionLogClientArgs) {
-    if (RULE_REGISTRY_LOG_ENABLED) {
-      this.client = new RuleRegistryAdapter(ruleDataService);
-    } else {
-      this.client = new SavedObjectsAdapter(savedObjectsClient);
+  constructor({
+    savedObjectsClient,
+    eventLogService,
+    underlyingClient,
+  }: RuleExecutionLogClientArgs) {
+    switch (underlyingClient) {
+      case UnderlyingLogClient.savedObjects:
+        this.client = new SavedObjectsAdapter(savedObjectsClient);
+        break;
+      case UnderlyingLogClient.eventLog:
+        this.client = new EventLogAdapter(eventLogService);
+        break;
     }
   }
 
@@ -53,8 +59,8 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient {
     return this.client.delete(id);
   }
 
-  public async logExecutionMetric<T extends ExecutionMetric>(args: ExecutionMetricArgs<T>) {
-    return this.client.logExecutionMetric(args);
+  public async logExecutionMetrics(args: LogExecutionMetricsArgs) {
+    return this.client.logExecutionMetrics(args);
   }
 
   public async logStatusChange(args: LogStatusChangeArgs) {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_adapter.ts

@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { merge } from 'lodash';
-import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas';
-import { RuleRegistryLogClient } from './rule_registry_log_client/rule_registry_log_client';
-import {
-  CreateExecutionLogArgs,
-  ExecutionMetric,
-  ExecutionMetricArgs,
-  FindBulkExecutionLogArgs,
-  FindExecutionLogArgs,
-  IRuleDataPluginService,
-  IRuleExecutionLogClient,
-  LogStatusChangeArgs,
-  UpdateExecutionLogArgs,
-} from '../types';
-
-/**
- * @deprecated RuleRegistryAdapter is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export class RuleRegistryAdapter implements IRuleExecutionLogClient {
-  private ruleRegistryClient: RuleRegistryLogClient;
-
-  constructor(ruleDataService: IRuleDataPluginService) {
-    this.ruleRegistryClient = new RuleRegistryLogClient(ruleDataService);
-  }
-
-  public async find({ ruleId, logsCount = 1, spaceId }: FindExecutionLogArgs) {
-    const logs = await this.ruleRegistryClient.find({
-      ruleIds: [ruleId],
-      logsCount,
-      spaceId,
-    });
-
-    return logs[ruleId].map((log) => ({
-      id: '',
-      type: '',
-      score: 0,
-      attributes: log,
-      references: [],
-    }));
-  }
-
-  public async findBulk({ ruleIds, logsCount = 1, spaceId }: FindBulkExecutionLogArgs) {
-    const [statusesById, lastErrorsById] = await Promise.all([
-      this.ruleRegistryClient.find({ ruleIds, spaceId }),
-      this.ruleRegistryClient.find({
-        ruleIds,
-        statuses: [RuleExecutionStatus.failed],
-        logsCount,
-        spaceId,
-      }),
-    ]);
-    return merge(statusesById, lastErrorsById);
-  }
-
-  private async create({ attributes, spaceId }: CreateExecutionLogArgs) {
-    if (attributes.status) {
-      await this.ruleRegistryClient.logStatusChange({
-        ruleId: attributes.alertId,
-        newStatus: attributes.status,
-        spaceId,
-      });
-    }
-
-    if (attributes.bulkCreateTimeDurations) {
-      await this.ruleRegistryClient.logExecutionMetric({
-        ruleId: attributes.alertId,
-        metric: ExecutionMetric.indexingDurationMax,
-        value: Math.max(...attributes.bulkCreateTimeDurations.map(Number)),
-        spaceId,
-      });
-    }
-
-    if (attributes.gap) {
-      await this.ruleRegistryClient.logExecutionMetric({
-        ruleId: attributes.alertId,
-        metric: ExecutionMetric.executionGap,
-        value: Number(attributes.gap),
-        spaceId,
-      });
-    }
-  }
-
-  public async update({ attributes, spaceId }: UpdateExecutionLogArgs) {
-    // execution events are immutable, so we just use 'create' here instead of 'update'
-    await this.create({ attributes, spaceId });
-  }
-
-  public async delete(id: string) {
-    // execution events are immutable, nothing to do here
-  }
-
-  public async logExecutionMetric<T extends ExecutionMetric>(args: ExecutionMetricArgs<T>) {
-    return this.ruleRegistryClient.logExecutionMetric(args);
-  }
-
-  public async logStatusChange(args: LogStatusChangeArgs) {
-    return this.ruleRegistryClient.logStatusChange(args);
-  }
-}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/constants.ts

@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-/**
- * @deprecated EVENTS_INDEX_PREFIX is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const EVENTS_INDEX_PREFIX = '.kibana_alerts-security.events';
-
-/**
- * @deprecated MESSAGE is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const MESSAGE = 'message' as const;
-
-/**
- * @deprecated EVENT_SEQUENCE is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const EVENT_SEQUENCE = 'event.sequence' as const;
-
-/**
- * @deprecated EVENT_DURATION is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const EVENT_DURATION = 'event.duration' as const;
-
-/**
- * @deprecated EVENT_END is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const EVENT_END = 'event.end' as const;
-
-/**
- * @deprecated RULE_STATUS is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const RULE_STATUS = 'kibana.rac.detection_engine.rule_status' as const;
-
-/**
- * @deprecated RULE_STATUS_SEVERITY is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const RULE_STATUS_SEVERITY = 'kibana.rac.detection_engine.rule_status_severity' as const;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/parse_rule_execution_log.ts

@@ -1,40 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { isLeft } from 'fp-ts/lib/Either';
-import { PathReporter } from 'io-ts/lib/PathReporter';
-import { technicalRuleFieldMap } from '../../../../../../../rule_registry/common/assets/field_maps/technical_rule_field_map';
-import {
-  mergeFieldMaps,
-  runtimeTypeFromFieldMap,
-} from '../../../../../../../rule_registry/common/field_map';
-import { ruleExecutionFieldMap } from './rule_execution_field_map';
-
-const ruleExecutionLogRuntimeType = runtimeTypeFromFieldMap(
-  mergeFieldMaps(technicalRuleFieldMap, ruleExecutionFieldMap)
-);
-
-/**
- * @deprecated parseRuleExecutionLog is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const parseRuleExecutionLog = (input: unknown) => {
-  const validate = ruleExecutionLogRuntimeType.decode(input);
-
-  if (isLeft(validate)) {
-    throw new Error(PathReporter.report(validate).join('\n'));
-  }
-
-  return ruleExecutionLogRuntimeType.encode(validate.right);
-};
-
-/**
- * @deprecated RuleExecutionEvent is kept here only as a reference. It will be superseded with EventLog implementation
- *
- * It's marked as `Partial` because the field map is not yet appropriate for
- * execution log events.
- */
-export type RuleExecutionEvent = Partial<ReturnType<typeof parseRuleExecutionLog>>;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_execution_field_map.ts

@@ -1,32 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import {
-  EVENT_DURATION,
-  EVENT_END,
-  EVENT_SEQUENCE,
-  MESSAGE,
-  RULE_STATUS,
-  RULE_STATUS_SEVERITY,
-} from './constants';
-
-/**
- * @deprecated ruleExecutionFieldMap is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const ruleExecutionFieldMap = {
-  [MESSAGE]: { type: 'keyword' },
-  [EVENT_SEQUENCE]: { type: 'long' },
-  [EVENT_END]: { type: 'date' },
-  [EVENT_DURATION]: { type: 'long' },
-  [RULE_STATUS]: { type: 'keyword' },
-  [RULE_STATUS_SEVERITY]: { type: 'integer' },
-} as const;
-
-/**
- * @deprecated RuleExecutionFieldMap is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export type RuleExecutionFieldMap = typeof ruleExecutionFieldMap;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts

@@ -1,270 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { estypes } from '@elastic/elasticsearch';
-import {
-  ALERT_RULE_CONSUMER,
-  ALERT_RULE_TYPE_ID,
-  EVENT_ACTION,
-  EVENT_KIND,
-  SPACE_IDS,
-  TIMESTAMP,
-  ALERT_RULE_UUID,
-} from '@kbn/rule-data-utils';
-import moment from 'moment';
-
-import { mappingFromFieldMap } from '../../../../../../../rule_registry/common/mapping_from_field_map';
-import { Dataset, IRuleDataClient } from '../../../../../../../rule_registry/server';
-import { SERVER_APP_ID } from '../../../../../../common/constants';
-import { RuleExecutionStatus } from '../../../../../../common/detection_engine/schemas/common/schemas';
-import { invariant } from '../../../../../../common/utils/invariant';
-import { IRuleStatusSOAttributes } from '../../../rules/types';
-import { makeFloatString } from '../../../signals/utils';
-import {
-  ExecutionMetric,
-  ExecutionMetricArgs,
-  IRuleDataPluginService,
-  LogStatusChangeArgs,
-} from '../../types';
-import { EVENT_SEQUENCE, MESSAGE, RULE_STATUS, RULE_STATUS_SEVERITY } from './constants';
-import { parseRuleExecutionLog, RuleExecutionEvent } from './parse_rule_execution_log';
-import { ruleExecutionFieldMap } from './rule_execution_field_map';
-import {
-  getLastEntryAggregation,
-  getMetricAggregation,
-  getMetricField,
-  sortByTimeDesc,
-} from './utils';
-
-const statusSeverityDict: Record<RuleExecutionStatus, number> = {
-  [RuleExecutionStatus.succeeded]: 0,
-  [RuleExecutionStatus['going to run']]: 10,
-  [RuleExecutionStatus.warning]: 20,
-  [RuleExecutionStatus['partial failure']]: 20,
-  [RuleExecutionStatus.failed]: 30,
-};
-
-interface FindExecutionLogArgs {
-  ruleIds: string[];
-  spaceId: string;
-  logsCount?: number;
-  statuses?: RuleExecutionStatus[];
-}
-
-interface IRuleRegistryLogClient {
-  find: (args: FindExecutionLogArgs) => Promise<{
-    [ruleId: string]: IRuleStatusSOAttributes[] | undefined;
-  }>;
-  create: (event: RuleExecutionEvent) => Promise<void>;
-  logStatusChange: (args: LogStatusChangeArgs) => Promise<void>;
-  logExecutionMetric: <T extends ExecutionMetric>(args: ExecutionMetricArgs<T>) => Promise<void>;
-}
-
-/**
- * @deprecated RuleRegistryLogClient is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export class RuleRegistryLogClient implements IRuleRegistryLogClient {
-  private sequence = 0;
-  private ruleDataClient: IRuleDataClient;
-
-  constructor(ruleDataService: IRuleDataPluginService) {
-    this.ruleDataClient = ruleDataService.initializeIndex({
-      feature: SERVER_APP_ID,
-      registrationContext: 'security',
-      dataset: Dataset.events,
-      componentTemplateRefs: [],
-      componentTemplates: [
-        {
-          name: 'mappings',
-          mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'),
-        },
-      ],
-    });
-  }
-
-  public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) {
-    if (ruleIds.length === 0) {
-      return {};
-    }
-
-    const filter: estypes.QueryDslQueryContainer[] = [
-      { terms: { [ALERT_RULE_UUID]: ruleIds } },
-      { terms: { [SPACE_IDS]: [spaceId] } },
-    ];
-
-    if (statuses) {
-      filter.push({ terms: { [RULE_STATUS]: statuses } });
-    }
-
-    const result = await this.ruleDataClient.getReader().search({
-      size: 0,
-      body: {
-        query: {
-          bool: {
-            filter,
-          },
-        },
-        aggs: {
-          rules: {
-            terms: {
-              field: ALERT_RULE_UUID,
-              size: ruleIds.length,
-            },
-            aggs: {
-              most_recent_logs: {
-                top_hits: {
-                  sort: sortByTimeDesc,
-                  size: logsCount,
-                },
-              },
-              last_failure: getLastEntryAggregation(RuleExecutionStatus.failed),
-              last_success: getLastEntryAggregation(RuleExecutionStatus.succeeded),
-              execution_gap: getMetricAggregation(ExecutionMetric.executionGap),
-              search_duration_max: getMetricAggregation(ExecutionMetric.searchDurationMax),
-              indexing_duration_max: getMetricAggregation(ExecutionMetric.indexingDurationMax),
-              indexing_lookback: getMetricAggregation(ExecutionMetric.indexingLookback),
-            },
-          },
-        },
-      },
-    });
-
-    if (result.hits.total.value === 0) {
-      return {};
-    }
-
-    invariant(result.aggregations, 'Search response should contain aggregations');
-
-    return Object.fromEntries(
-      result.aggregations.rules.buckets.map<[ruleId: string, logs: IRuleStatusSOAttributes[]]>(
-        (bucket) => [
-          bucket.key as string,
-          bucket.most_recent_logs.hits.hits.map<IRuleStatusSOAttributes>((event) => {
-            const logEntry = parseRuleExecutionLog(event._source);
-            invariant(
-              logEntry[ALERT_RULE_UUID] ?? '',
-              'Malformed execution log entry: rule.id field not found'
-            );
-
-            const lastFailure = bucket.last_failure.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.last_failure.event.hits.hits[0]._source)
-              : undefined;
-
-            const lastSuccess = bucket.last_success.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.last_success.event.hits.hits[0]._source)
-              : undefined;
-
-            const lookBack = bucket.indexing_lookback.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.indexing_lookback.event.hits.hits[0]._source)
-              : undefined;
-
-            const executionGap = bucket.execution_gap.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.execution_gap.event.hits.hits[0]._source)[
-                  getMetricField(ExecutionMetric.executionGap)
-                ]
-              : undefined;
-
-            const searchDuration = bucket.search_duration_max.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.search_duration_max.event.hits.hits[0]._source)[
-                  getMetricField(ExecutionMetric.searchDurationMax)
-                ]
-              : undefined;
-
-            const indexingDuration = bucket.indexing_duration_max.event.hits.hits[0]
-              ? parseRuleExecutionLog(bucket.indexing_duration_max.event.hits.hits[0]._source)[
-                  getMetricField(ExecutionMetric.indexingDurationMax)
-                ]
-              : undefined;
-
-            const alertId = logEntry[ALERT_RULE_UUID] ?? '';
-            const statusDate = logEntry[TIMESTAMP];
-            const lastFailureAt = lastFailure?.[TIMESTAMP];
-            const lastFailureMessage = lastFailure?.[MESSAGE];
-            const lastSuccessAt = lastSuccess?.[TIMESTAMP];
-            const lastSuccessMessage = lastSuccess?.[MESSAGE];
-            const status = (logEntry[RULE_STATUS] as RuleExecutionStatus) || null;
-            const lastLookBackDate = lookBack?.[getMetricField(ExecutionMetric.indexingLookback)];
-            const gap = executionGap ? moment.duration(executionGap).humanize() : null;
-            const bulkCreateTimeDurations = indexingDuration
-              ? [makeFloatString(indexingDuration)]
-              : null;
-            const searchAfterTimeDurations = searchDuration
-              ? [makeFloatString(searchDuration)]
-              : null;
-
-            return {
-              alertId,
-              statusDate,
-              lastFailureAt,
-              lastFailureMessage,
-              lastSuccessAt,
-              lastSuccessMessage,
-              status,
-              lastLookBackDate,
-              gap,
-              bulkCreateTimeDurations,
-              searchAfterTimeDurations,
-            };
-          }),
-        ]
-      )
-    );
-  }
-
-  public async logExecutionMetric<T extends ExecutionMetric>({
-    ruleId,
-    namespace,
-    metric,
-    value,
-    spaceId,
-  }: ExecutionMetricArgs<T>) {
-    await this.create(
-      {
-        [SPACE_IDS]: [spaceId],
-        [EVENT_ACTION]: metric,
-        [EVENT_KIND]: 'metric',
-        [getMetricField(metric)]: value,
-        [ALERT_RULE_UUID]: ruleId ?? '',
-        [TIMESTAMP]: new Date().toISOString(),
-        [ALERT_RULE_CONSUMER]: SERVER_APP_ID,
-        [ALERT_RULE_TYPE_ID]: SERVER_APP_ID,
-      },
-      namespace
-    );
-  }
-
-  public async logStatusChange({
-    ruleId,
-    newStatus,
-    namespace,
-    message,
-    spaceId,
-  }: LogStatusChangeArgs) {
-    await this.create(
-      {
-        [SPACE_IDS]: [spaceId],
-        [EVENT_ACTION]: 'status-change',
-        [EVENT_KIND]: 'event',
-        [EVENT_SEQUENCE]: this.sequence++,
-        [MESSAGE]: message,
-        [ALERT_RULE_UUID]: ruleId ?? '',
-        [RULE_STATUS_SEVERITY]: statusSeverityDict[newStatus],
-        [RULE_STATUS]: newStatus,
-        [TIMESTAMP]: new Date().toISOString(),
-        [ALERT_RULE_CONSUMER]: SERVER_APP_ID,
-        [ALERT_RULE_TYPE_ID]: SERVER_APP_ID,
-      },
-      namespace
-    );
-  }
-
-  public async create(event: RuleExecutionEvent, namespace?: string) {
-    await this.ruleDataClient.getWriter({ namespace }).bulk({
-      body: [{ index: {} }, event],
-    });
-  }
-}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/utils.ts

@@ -1,76 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { SearchSort } from '@elastic/elasticsearch/api/types';
-import { EVENT_ACTION, TIMESTAMP } from '@kbn/rule-data-utils';
-import { RuleExecutionStatus } from '../../../../../../common/detection_engine/schemas/common/schemas';
-import { ExecutionMetric } from '../../types';
-import { RULE_STATUS, EVENT_SEQUENCE, EVENT_DURATION, EVENT_END } from './constants';
-
-const METRIC_FIELDS = {
-  [ExecutionMetric.executionGap]: EVENT_DURATION,
-  [ExecutionMetric.searchDurationMax]: EVENT_DURATION,
-  [ExecutionMetric.indexingDurationMax]: EVENT_DURATION,
-  [ExecutionMetric.indexingLookback]: EVENT_END,
-};
-
-/**
- * Returns ECS field in which metric value is stored
- * @deprecated getMetricField is kept here only as a reference. It will be superseded with EventLog implementation
- *
- * @param metric - execution metric
- * @returns ECS field
- */
-export const getMetricField = <T extends ExecutionMetric>(metric: T) => METRIC_FIELDS[metric];
-
-/**
- * @deprecated sortByTimeDesc is kept here only as a reference. It will be superseded with EventLog implementation
- */
-export const sortByTimeDesc: SearchSort = [{ [TIMESTAMP]: 'desc' }, { [EVENT_SEQUENCE]: 'desc' }];
-
-/**
- * Builds aggregation to retrieve the most recent metric value
- * @deprecated getMetricAggregation is kept here only as a reference. It will be superseded with EventLog implementation
- *
- * @param metric - execution metric
- * @returns aggregation
- */
-export const getMetricAggregation = (metric: ExecutionMetric) => ({
-  filter: {
-    term: { [EVENT_ACTION]: metric },
-  },
-  aggs: {
-    event: {
-      top_hits: {
-        size: 1,
-        sort: sortByTimeDesc,
-        _source: [TIMESTAMP, getMetricField(metric)],
-      },
-    },
-  },
-});
-
-/**
- * Builds aggregation to retrieve the most recent log entry with the given status
- * @deprecated getLastEntryAggregation is kept here only as a reference. It will be superseded with EventLog implementation
- *
- * @param status - rule execution status
- * @returns aggregation
- */
-export const getLastEntryAggregation = (status: RuleExecutionStatus) => ({
-  filter: {
-    term: { [RULE_STATUS]: status },
-  },
-  aggs: {
-    event: {
-      top_hits: {
-        sort: sortByTimeDesc,
-        size: 1,
-      },
-    },
-  },
-});

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts

@@ -14,12 +14,11 @@ import {
   ruleStatusSavedObjectsClientFactory,
 } from './rule_status_saved_objects_client';
 import {
-  ExecutionMetric,
-  ExecutionMetricArgs,
+  LogExecutionMetricsArgs,
   FindBulkExecutionLogArgs,
   FindExecutionLogArgs,
   IRuleExecutionLogClient,
-  LegacyMetrics,
+  ExecutionMetrics,
   LogStatusChangeArgs,
   UpdateExecutionLogArgs,
 } from '../types';
@@ -28,14 +27,16 @@ import { assertUnreachable } from '../../../../../common';
 // 1st is mutable status, followed by 5 most recent failures
 export const MAX_RULE_STATUSES = 6;
 
-const METRIC_FIELDS = {
-  [ExecutionMetric.executionGap]: 'gap',
-  [ExecutionMetric.searchDurationMax]: 'searchAfterTimeDurations',
-  [ExecutionMetric.indexingDurationMax]: 'bulkCreateTimeDurations',
-  [ExecutionMetric.indexingLookback]: 'lastLookBackDate',
-} as const;
-
-const getMetricField = <T extends ExecutionMetric>(metric: T) => METRIC_FIELDS[metric];
+const convertMetricFields = (
+  metrics: ExecutionMetrics
+): Pick<
+  IRuleStatusSOAttributes,
+  'gap' | 'searchAfterTimeDurations' | 'bulkCreateTimeDurations'
+> => ({
+  gap: metrics.executionGap?.humanize(),
+  searchAfterTimeDurations: metrics.searchDurations,
+  bulkCreateTimeDurations: metrics.indexingDurations,
+});
 
 export class SavedObjectsAdapter implements IRuleExecutionLogClient {
   private ruleStatusClient: RuleStatusSavedObjectsClient;
@@ -66,16 +67,12 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient {
     await this.ruleStatusClient.delete(id);
   }
 
-  public async logExecutionMetric<T extends ExecutionMetric>({
-    ruleId,
-    metric,
-    value,
-  }: ExecutionMetricArgs<T>) {
+  public async logExecutionMetrics({ ruleId, metrics }: LogExecutionMetricsArgs) {
     const [currentStatus] = await this.getOrCreateRuleStatuses(ruleId);
 
     await this.ruleStatusClient.update(currentStatus.id, {
       ...currentStatus.attributes,
-      [getMetricField(metric)]: value,
+      ...convertMetricFields(metrics),
     });
   }
 
@@ -158,11 +155,11 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient {
 const buildRuleStatusAttributes: (
   status: RuleExecutionStatus,
   message?: string,
-  metrics?: LegacyMetrics
+  metrics?: ExecutionMetrics
 ) => Partial<IRuleStatusSOAttributes> = (status, message, metrics = {}) => {
   const now = new Date().toISOString();
   const baseAttributes: Partial<IRuleStatusSOAttributes> = {
-    ...metrics,
+    ...convertMetricFields(metrics),
     status:
       status === RuleExecutionStatus.warning ? RuleExecutionStatus['partial failure'] : status,
     statusDate: now,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts

@@ -5,28 +5,16 @@
  * 2.0.
  */
 
-import { PublicMethodsOf } from '@kbn/utility-types';
+import { Duration } from 'moment';
 import { SavedObjectsFindResult } from '../../../../../../../src/core/server';
-import { RuleDataPluginService } from '../../../../../rule_registry/server';
 import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
 import { IRuleStatusSOAttributes } from '../rules/types';
 
-export enum ExecutionMetric {
-  'executionGap' = 'executionGap',
-  'searchDurationMax' = 'searchDurationMax',
-  'indexingDurationMax' = 'indexingDurationMax',
-  'indexingLookback' = 'indexingLookback',
+export enum UnderlyingLogClient {
+  'savedObjects' = 'savedObjects',
+  'eventLog' = 'eventLog',
 }
 
-export type IRuleDataPluginService = PublicMethodsOf<RuleDataPluginService>;
-
-export type ExecutionMetricValue<T extends ExecutionMetric> = {
-  [ExecutionMetric.executionGap]: number;
-  [ExecutionMetric.searchDurationMax]: number;
-  [ExecutionMetric.indexingDurationMax]: number;
-  [ExecutionMetric.indexingLookback]: Date;
-}[T];
-
 export interface FindExecutionLogArgs {
   ruleId: string;
   spaceId: string;
@@ -39,29 +27,34 @@ export interface FindBulkExecutionLogArgs {
   logsCount?: number;
 }
 
-/**
- * @deprecated LegacyMetrics are only kept here for backward compatibility
- * and should be replaced by ExecutionMetric in the future
- */
-export interface LegacyMetrics {
-  searchAfterTimeDurations?: string[];
-  bulkCreateTimeDurations?: string[];
+export interface ExecutionMetrics {
+  searchDurations?: string[];
+  indexingDurations?: string[];
+  /**
+   * @deprecated lastLookBackDate is logged only by SavedObjectsAdapter and should be removed in the future
+   */
   lastLookBackDate?: string;
-  gap?: string;
+  executionGap?: Duration;
 }
 
 export interface LogStatusChangeArgs {
   ruleId: string;
+  ruleName: string;
+  ruleType: string;
   spaceId: string;
   newStatus: RuleExecutionStatus;
-  namespace?: string;
   message?: string;
-  metrics?: LegacyMetrics;
+  /**
+   * @deprecated Use RuleExecutionLogClient.logExecutionMetrics to write metrics instead
+   */
+  metrics?: ExecutionMetrics;
 }
 
 export interface UpdateExecutionLogArgs {
   id: string;
   attributes: IRuleStatusSOAttributes;
+  ruleName: string;
+  ruleType: string;
   spaceId: string;
 }
 
@@ -70,12 +63,12 @@ export interface CreateExecutionLogArgs {
   spaceId: string;
 }
 
-export interface ExecutionMetricArgs<T extends ExecutionMetric> {
+export interface LogExecutionMetricsArgs {
   ruleId: string;
+  ruleName: string;
+  ruleType: string;
   spaceId: string;
-  namespace?: string;
-  metric: T;
-  value: ExecutionMetricValue<T>;
+  metrics: ExecutionMetrics;
 }
 
 export interface FindBulkExecutionLogResponse {
@@ -90,5 +83,5 @@ export interface IRuleExecutionLogClient {
   update: (args: UpdateExecutionLogArgs) => Promise<void>;
   delete: (id: string) => Promise<void>;
   logStatusChange: (args: LogStatusChangeArgs) => Promise<void>;
-  logExecutionMetric: <T extends ExecutionMetric>(args: ExecutionMetricArgs<T>) => Promise<void>;
+  logExecutionMetrics: (args: LogExecutionMetricsArgs) => Promise<void>;
 }

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts

@@ -14,6 +14,7 @@ import { mlPluginServerMock } from '../../../../../../ml/server/mocks';
 
 import type { IRuleDataClient } from '../../../../../../rule_registry/server';
 import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks';
+import { eventLogServiceMock } from '../../../../../../event_log/server/mocks';
 import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server';
 import { ConfigType } from '../../../../config';
 import { AlertAttributes } from '../../signals/types';
@@ -55,6 +56,7 @@ export const createRuleTypeMocks = (
     references: [],
     attributes: {
       actions: [],
+      alertTypeId: 'siem.signals',
       enabled: true,
       name: 'mock rule',
       tags: [],
@@ -89,7 +91,7 @@ export const createRuleTypeMocks = (
       ruleDataClient: ruleRegistryMocks.createRuleDataClient(
         '.alerts-security.alerts'
       ) as IRuleDataClient,
-      ruleDataService: ruleRegistryMocks.createRuleDataPluginService(),
+      eventLogService: eventLogServiceMock.create(),
     },
     services,
     scheduleActions,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts

@@ -40,8 +40,9 @@ import { scheduleThrottledNotificationActions } from '../notifications/schedule_
 
 /* eslint-disable complexity */
 export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
-  ({ lists, logger, mergeStrategy, ignoreFields, ruleDataClient, ruleDataService }) =>
+  ({ lists, logger, config, ruleDataClient, eventLogService }) =>
   (type) => {
+    const { alertIgnoreFields: ignoreFields, alertMergeStrategy: mergeStrategy } = config;
     const persistenceRuleType = createPersistenceRuleTypeFactory({ ruleDataClient, logger });
     return persistenceRuleType({
       ...type,
@@ -65,13 +66,15 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
 
         const ruleStatusClient = new RuleExecutionLogClient({
           savedObjectsClient,
-          ruleDataService,
+          eventLogService,
+          underlyingClient: config.ruleExecutionLog.underlyingClient,
         });
         const ruleSO = await savedObjectsClient.get('alert', alertId);
 
         const {
           actions,
           name,
+          alertTypeId,
           schedule: { interval },
         } = ruleSO.attributes;
         const refresh = actions.length ? 'wait_for' : false;
@@ -87,9 +90,14 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
         logger.debug(buildRuleMessage(`interval: ${interval}`));
 
         let wroteWarningStatus = false;
-        await ruleStatusClient.logStatusChange({
+        const basicLogArguments = {
           spaceId,
           ruleId: alertId,
+          ruleName: name,
+          ruleType: alertTypeId,
+        };
+        await ruleStatusClient.logStatusChange({
+          ...basicLogArguments,
           newStatus: RuleExecutionStatus['going to run'],
         });
 
@@ -125,8 +133,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
                   tryCatch(
                     () =>
                       hasReadIndexPrivileges({
-                        spaceId,
-                        ruleId: alertId,
+                        ...basicLogArguments,
                         privileges,
                         logger,
                         buildRuleMessage,
@@ -138,8 +145,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
                   tryCatch(
                     () =>
                       hasTimestampFields({
-                        spaceId,
-                        ruleId: alertId,
+                        ...basicLogArguments,
                         wroteStatus: wroteStatus as boolean,
                         timestampField: hasTimestampOverride
                           ? (timestampOverride as string)
@@ -179,11 +185,10 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
           logger.warn(gapMessage);
           hasError = true;
           await ruleStatusClient.logStatusChange({
-            spaceId,
-            ruleId: alertId,
+            ...basicLogArguments,
             newStatus: RuleExecutionStatus.failed,
             message: gapMessage,
-            metrics: { gap: gapString },
+            metrics: { executionGap: remainingGap },
           });
         }
 
@@ -262,8 +267,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
           if (result.warningMessages.length) {
             const warningMessage = buildRuleMessage(result.warningMessages.join());
             await ruleStatusClient.logStatusChange({
-              spaceId,
-              ruleId: alertId,
+              ...basicLogArguments,
               newStatus: RuleExecutionStatus['partial failure'],
               message: warningMessage,
             });
@@ -327,13 +331,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
 
             if (!hasError && !wroteWarningStatus && !result.warning) {
               await ruleStatusClient.logStatusChange({
-                spaceId,
-                ruleId: alertId,
+                ...basicLogArguments,
                 newStatus: RuleExecutionStatus.succeeded,
                 message: 'succeeded',
                 metrics: {
-                  bulkCreateTimeDurations: result.bulkCreateTimes,
-                  searchAfterTimeDurations: result.searchAfterTimes,
+                  indexingDurations: result.bulkCreateTimes,
+                  searchDurations: result.searchAfterTimes,
                   lastLookBackDate: result.lastLookbackDate?.toISOString(),
                 },
               });
@@ -356,13 +359,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
             );
             logger.error(errorMessage);
             await ruleStatusClient.logStatusChange({
-              spaceId,
-              ruleId: alertId,
+              ...basicLogArguments,
               newStatus: RuleExecutionStatus.failed,
               message: errorMessage,
               metrics: {
-                bulkCreateTimeDurations: result.bulkCreateTimes,
-                searchAfterTimeDurations: result.searchAfterTimes,
+                indexingDurations: result.bulkCreateTimes,
+                searchDurations: result.searchAfterTimes,
                 lastLookBackDate: result.lastLookbackDate?.toISOString(),
               },
             });
@@ -376,13 +378,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory =
 
           logger.error(message);
           await ruleStatusClient.logStatusChange({
-            spaceId,
-            ruleId: alertId,
+            ...basicLogArguments,
             newStatus: RuleExecutionStatus.failed,
             message,
             metrics: {
-              bulkCreateTimeDurations: result.bulkCreateTimes,
-              searchAfterTimeDurations: result.searchAfterTimes,
+              indexingDurations: result.bulkCreateTimes,
+              searchDurations: result.searchAfterTimes,
               lastLookBackDate: result.lastLookbackDate?.toISOString(),
             },
           });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.test.ts

@@ -12,6 +12,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe
 import { createEqlAlertType } from './create_eql_alert_type';
 import { createRuleTypeMocks } from '../__mocks__/rule_type';
 import { getEqlRuleParams } from '../../schemas/rule_schemas.mock';
+import { createMockConfig } from '../../routes/__mocks__';
 
 jest.mock('../../rule_execution_log/rule_execution_log_client');
 
@@ -26,10 +27,9 @@ describe('Event correlation alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      ignoreFields: [],
-      mergeStrategy: 'allFields',
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
     dependencies.alerting.registerType(eqlAlertType);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts

@@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createEqlAlertType = (createOptions: CreateRuleOptions) => {
-  const {
-    experimentalFeatures,
-    lists,
-    logger,
-    ignoreFields,
-    mergeStrategy,
-    ruleDataClient,
-    version,
-    ruleDataService,
-  } = createOptions;
+  const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } =
+    createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
-    ignoreFields,
-    mergeStrategy,
+    config,
     ruleDataClient,
-    ruleDataService,
+    eventLogService,
   });
   return createSecurityRuleType<EqlRuleParams, {}, PersistenceServices, {}>({
     id: EQL_RULE_TYPE_ID,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts

@@ -44,6 +44,7 @@ describe('buildAlert', () => {
     const ruleSO = {
       attributes: {
         actions: [],
+        alertTypeId: 'siem.signals',
         createdAt: new Date().toISOString(),
         createdBy: 'gandalf',
         params: getQueryRuleParams(),

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts

@@ -16,6 +16,7 @@ import { createIndicatorMatchAlertType } from './create_indicator_match_alert_ty
 import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
 import { CountResponse } from 'kibana/server';
 import { RuleParams } from '../../schemas/rule_schemas';
+import { createMockConfig } from '../../routes/__mocks__';
 
 jest.mock('../utils/get_list_client', () => ({
   getListClient: jest.fn().mockReturnValue({
@@ -56,10 +57,9 @@ describe('Indicator Match Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      ignoreFields: [],
-      mergeStrategy: 'allFields',
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 
@@ -97,10 +97,9 @@ describe('Indicator Match Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 
@@ -136,10 +135,9 @@ describe('Indicator Match Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) => {
-  const {
-    experimentalFeatures,
-    lists,
-    logger,
-    mergeStrategy,
-    ignoreFields,
-    ruleDataClient,
-    version,
-    ruleDataService,
-  } = createOptions;
+  const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } =
+    createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
-    mergeStrategy,
-    ignoreFields,
+    config,
     ruleDataClient,
-    ruleDataService,
+    eventLogService,
   });
   return createSecurityRuleType<ThreatRuleParams, {}, PersistenceServices, {}>({
     id: INDICATOR_RULE_TYPE_ID,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts

@@ -14,6 +14,7 @@ import { createRuleTypeMocks } from '../__mocks__/rule_type';
 import { createMlAlertType } from './create_ml_alert_type';
 
 import { RuleParams } from '../../schemas/rule_schemas';
+import { createMockConfig } from '../../routes/__mocks__';
 
 jest.mock('../../signals/bulk_create_ml_signals');
 
@@ -97,11 +98,10 @@ describe('Machine Learning Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ml: mlMock,
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts

@@ -14,15 +14,13 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createMlAlertType = (createOptions: CreateRuleOptions) => {
-  const { lists, logger, mergeStrategy, ignoreFields, ml, ruleDataClient, ruleDataService } =
-    createOptions;
+  const { lists, logger, config, ml, ruleDataClient, eventLogService } = createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
-    mergeStrategy,
-    ignoreFields,
+    config,
     ruleDataClient,
-    ruleDataService,
+    eventLogService,
   });
   return createSecurityRuleType<MachineLearningRuleParams, {}, PersistenceServices, {}>({
     id: ML_RULE_TYPE_ID,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts

@@ -14,6 +14,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe
 import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
 import { createQueryAlertType } from './create_query_alert_type';
 import { createRuleTypeMocks } from '../__mocks__/rule_type';
+import { createMockConfig } from '../../routes/__mocks__';
 
 jest.mock('../utils/get_list_client', () => ({
   getListClient: jest.fn().mockReturnValue({
@@ -31,10 +32,9 @@ describe('Custom Query Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 
@@ -79,10 +79,9 @@ describe('Custom Query Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts

@@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createQueryAlertType = (createOptions: CreateRuleOptions) => {
-  const {
-    experimentalFeatures,
-    lists,
-    logger,
-    mergeStrategy,
-    ignoreFields,
-    ruleDataClient,
-    version,
-    ruleDataService,
-  } = createOptions;
+  const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } =
+    createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
-    mergeStrategy,
-    ignoreFields,
+    config,
     ruleDataClient,
-    ruleDataService,
+    eventLogService,
   });
   return createSecurityRuleType<QueryRuleParams, {}, PersistenceServices, {}>({
     id: QUERY_RULE_TYPE_ID,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts

@@ -9,6 +9,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe
 import { createThresholdAlertType } from './create_threshold_alert_type';
 import { createRuleTypeMocks } from '../__mocks__/rule_type';
 import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock';
+import { createMockConfig } from '../../routes/__mocks__';
 
 jest.mock('../../rule_execution_log/rule_execution_log_client');
 
@@ -20,10 +21,9 @@ describe('Threshold Alerts', () => {
       experimentalFeatures: allowedExperimentalValues,
       lists: dependencies.lists,
       logger: dependencies.logger,
-      mergeStrategy: 'allFields',
-      ignoreFields: [],
+      config: createMockConfig(),
       ruleDataClient: dependencies.ruleDataClient,
-      ruleDataService: dependencies.ruleDataService,
+      eventLogService: dependencies.eventLogService,
       version: '1.0.0',
     });
     dependencies.alerting.registerType(thresholdAlertTpe);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts

@@ -16,23 +16,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact
 import { CreateRuleOptions } from '../types';
 
 export const createThresholdAlertType = (createOptions: CreateRuleOptions) => {
-  const {
-    experimentalFeatures,
-    lists,
-    logger,
-    mergeStrategy,
-    ignoreFields,
-    ruleDataClient,
-    version,
-    ruleDataService,
-  } = createOptions;
+  const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } =
+    createOptions;
   const createSecurityRuleType = createSecurityRuleTypeFactory({
     lists,
     logger,
-    mergeStrategy,
-    ignoreFields,
+    config,
     ruleDataClient,
-    ruleDataService,
+    eventLogService,
   });
   return createSecurityRuleType<ThresholdRuleParams, {}, PersistenceServices, ThresholdAlertState>({
     id: THRESHOLD_RULE_TYPE_ID,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -30,12 +30,12 @@ import {
 import { BaseHit } from '../../../../common/detection_engine/types';
 import { ConfigType } from '../../../config';
 import { SetupPlugins } from '../../../plugin';
-import { IRuleDataPluginService } from '../rule_execution_log/types';
 import { RuleParams } from '../schemas/rule_schemas';
 import { BuildRuleMessage } from '../signals/rule_messages';
 import { AlertAttributes, BulkCreate, WrapHits, WrapSequences } from '../signals/types';
 import { AlertsFieldMap, RulesFieldMap } from './field_maps';
 import { ExperimentalFeatures } from '../../../../common/experimental_features';
+import { IEventLogService } from '../../../../../event_log/server';
 
 export interface SecurityAlertTypeReturnValue<TState extends AlertTypeState> {
   bulkCreateTimes: string[];
@@ -98,10 +98,9 @@ type SecurityAlertTypeWithExecutor<
 export type CreateSecurityRuleTypeFactory = (options: {
   lists: SetupPlugins['lists'];
   logger: Logger;
-  mergeStrategy: ConfigType['alertMergeStrategy'];
-  ignoreFields: ConfigType['alertIgnoreFields'];
+  config: ConfigType;
   ruleDataClient: IRuleDataClient;
-  ruleDataService: IRuleDataPluginService;
+  eventLogService: IEventLogService;
 }) => <
   TParams extends RuleParams & { index?: string[] | undefined },
   TAlertInstanceContext extends AlertInstanceContext,
@@ -127,10 +126,9 @@ export interface CreateRuleOptions {
   experimentalFeatures: ExperimentalFeatures;
   lists: SetupPlugins['lists'];
   logger: Logger;
-  mergeStrategy: ConfigType['alertMergeStrategy'];
-  ignoreFields: ConfigType['alertIgnoreFields'];
+  config: ConfigType;
   ml?: SetupPlugins['ml'];
   ruleDataClient: IRuleDataClient;
   version: string;
-  ruleDataService: IRuleDataPluginService;
+  eventLogService: IEventLogService;
 }

x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts

@@ -44,6 +44,8 @@ export const enableRule = async ({
     const currentStatusToDisable = ruleCurrentStatus[0];
     await ruleStatusClient.update({
       id: currentStatusToDisable.id,
+      ruleName: rule.name,
+      ruleType: rule.alertTypeId,
       attributes: {
         ...currentStatusToDisable.attributes,
         status: RuleExecutionStatus['going to run'],

x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts

@@ -33,6 +33,7 @@ export const sampleRuleSO = <T extends RuleParams>(params: T): SavedObject<Alert
     updated_at: '2020-03-27T22:55:59.577Z',
     attributes: {
       actions: [],
+      alertTypeId: 'siem.signals',
       enabled: true,
       name: 'rule-name',
       tags: ['some fake tag 1', 'some fake tag 2'],

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -33,6 +33,7 @@ describe('eql_executor', () => {
     updated_at: '2020-03-27T22:55:59.577Z',
     attributes: {
       actions: [],
+      alertTypeId: 'siem.signals',
       enabled: true,
       name: 'rule-name',
       tags: ['some fake tag 1', 'some fake tag 2'],

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts

@@ -30,6 +30,7 @@ describe('threshold_executor', () => {
     updated_at: '2020-03-27T22:55:59.577Z',
     attributes: {
       actions: [],
+      alertTypeId: 'siem.signals',
       enabled: true,
       name: 'rule-name',
       tags: ['some fake tag 1', 'some fake tag 2'],

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts

@@ -32,10 +32,11 @@ import { mlExecutor } from './executors/ml';
 import { getMlRuleParams, getQueryRuleParams } from '../schemas/rule_schemas.mock';
 import { ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { allowedExperimentalValues } from '../../../../common/experimental_features';
-import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks';
 import { scheduleNotificationActions } from '../notifications/schedule_notification_actions';
 import { ruleExecutionLogClientMock } from '../rule_execution_log/__mocks__/rule_execution_log_client';
 import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
+import { eventLogServiceMock } from '../../../../../event_log/server/mocks';
+import { createMockConfig } from '../routes/__mocks__';
 
 jest.mock('./utils', () => {
   const original = jest.requireActual('./utils');
@@ -124,12 +125,12 @@ describe('signal_rule_alert_type', () => {
   let alert: ReturnType<typeof signalRulesAlertType>;
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
   let alertServices: AlertServicesMock;
-  let ruleDataService: ReturnType<typeof ruleRegistryMocks.createRuleDataPluginService>;
+  let eventLogService: ReturnType<typeof eventLogServiceMock.create>;
 
   beforeEach(() => {
     alertServices = alertsMock.createAlertServices();
     logger = loggingSystemMock.createLogger();
-    ruleDataService = ruleRegistryMocks.createRuleDataPluginService();
+    eventLogService = eventLogServiceMock.create();
     (getListsClient as jest.Mock).mockReturnValue({
       listClient: getListClientMock(),
       exceptionsClient: getExceptionListClientMock(),
@@ -194,9 +195,8 @@ describe('signal_rule_alert_type', () => {
       version,
       ml: mlMock,
       lists: listMock.createSetup(),
-      mergeStrategy: 'missingFields',
-      ignoreFields: [],
-      ruleDataService,
+      config: createMockConfig(),
+      eventLogService,
     });
 
     mockRuleExecutionLogClient.logStatusChange.mockClear();
@@ -217,11 +217,18 @@ describe('signal_rule_alert_type', () => {
       payload.previousStartedAt = moment().subtract(100, 'm').toDate();
       await alert.executor(payload);
       expect(logger.warn).toHaveBeenCalled();
-      expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith(
+      expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          newStatus: RuleExecutionStatus['going to run'],
+        })
+      );
+      expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith(
+        2,
         expect.objectContaining({
           newStatus: RuleExecutionStatus.failed,
           metrics: {
-            gap: 'an hour',
+            executionGap: expect.any(Object),
           },
         })
       );

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -70,9 +70,9 @@ import { ConfigType } from '../../../config';
 import { ExperimentalFeatures } from '../../../../common/experimental_features';
 import { injectReferences, extractReferences } from './saved_object_references';
 import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client';
-import { IRuleDataPluginService } from '../rule_execution_log/types';
 import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
 import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions';
+import { IEventLogService } from '../../../../../event_log/server';
 
 export const signalRulesAlertType = ({
   logger,
@@ -81,9 +81,8 @@ export const signalRulesAlertType = ({
   version,
   ml,
   lists,
-  mergeStrategy,
-  ignoreFields,
-  ruleDataService,
+  config,
+  eventLogService,
 }: {
   logger: Logger;
   eventsTelemetry: TelemetryEventsSender | undefined;
@@ -91,10 +90,10 @@ export const signalRulesAlertType = ({
   version: string;
   ml: SetupPlugins['ml'];
   lists: SetupPlugins['lists'] | undefined;
-  mergeStrategy: ConfigType['alertMergeStrategy'];
-  ignoreFields: ConfigType['alertIgnoreFields'];
-  ruleDataService: IRuleDataPluginService;
+  config: ConfigType;
+  eventLogService: IEventLogService;
 }): SignalRuleAlertTypeDefinition => {
+  const { alertMergeStrategy: mergeStrategy, alertIgnoreFields: ignoreFields } = config;
   return {
     id: SIGNALS_ID,
     name: 'SIEM signal',
@@ -138,14 +137,16 @@ export const signalRulesAlertType = ({
       let hasError: boolean = false;
       let result = createSearchAfterReturnType();
       const ruleStatusClient = new RuleExecutionLogClient({
-        ruleDataService,
+        eventLogService,
         savedObjectsClient: services.savedObjectsClient,
+        underlyingClient: config.ruleExecutionLog.underlyingClient,
       });
 
       const savedObject = await services.savedObjectsClient.get<AlertAttributes>('alert', alertId);
       const {
         actions,
         name,
+        alertTypeId,
         schedule: { interval },
       } = savedObject.attributes;
       const refresh = actions.length ? 'wait_for' : false;
@@ -159,10 +160,16 @@ export const signalRulesAlertType = ({
       logger.debug(buildRuleMessage('[+] Starting Signal Rule execution'));
       logger.debug(buildRuleMessage(`interval: ${interval}`));
       let wroteWarningStatus = false;
-      await ruleStatusClient.logStatusChange({
-        ruleId: alertId,
-        newStatus: RuleExecutionStatus['going to run'],
+      const basicLogArguments = {
         spaceId,
+        ruleId: alertId,
+        ruleName: name,
+        ruleType: alertTypeId,
+      };
+
+      await ruleStatusClient.logStatusChange({
+        ...basicLogArguments,
+        newStatus: RuleExecutionStatus['going to run'],
       });
 
       // check if rule has permissions to access given index pattern
@@ -194,8 +201,7 @@ export const signalRulesAlertType = ({
               tryCatch(
                 () =>
                   hasReadIndexPrivileges({
-                    spaceId,
-                    ruleId: alertId,
+                    ...basicLogArguments,
                     privileges,
                     logger,
                     buildRuleMessage,
@@ -207,13 +213,11 @@ export const signalRulesAlertType = ({
               tryCatch(
                 () =>
                   hasTimestampFields({
-                    spaceId,
-                    ruleId: alertId,
+                    ...basicLogArguments,
                     wroteStatus: wroteStatus as boolean,
                     timestampField: hasTimestampOverride
                       ? (timestampOverride as string)
                       : '@timestamp',
-                    ruleName: name,
                     timestampFieldCapsResponse: timestampFieldCaps,
                     inputIndices,
                     ruleStatusClient,
@@ -247,11 +251,10 @@ export const signalRulesAlertType = ({
         logger.warn(gapMessage);
         hasError = true;
         await ruleStatusClient.logStatusChange({
-          spaceId,
-          ruleId: alertId,
+          ...basicLogArguments,
           newStatus: RuleExecutionStatus.failed,
           message: gapMessage,
-          metrics: { gap: gapString },
+          metrics: { executionGap: remainingGap },
         });
       }
       try {
@@ -383,8 +386,7 @@ export const signalRulesAlertType = ({
         if (result.warningMessages.length) {
           const warningMessage = buildRuleMessage(result.warningMessages.join());
           await ruleStatusClient.logStatusChange({
-            spaceId,
-            ruleId: alertId,
+            ...basicLogArguments,
             newStatus: RuleExecutionStatus['partial failure'],
             message: warningMessage,
           });
@@ -445,13 +447,12 @@ export const signalRulesAlertType = ({
           );
           if (!hasError && !wroteWarningStatus && !result.warning) {
             await ruleStatusClient.logStatusChange({
-              spaceId,
-              ruleId: alertId,
+              ...basicLogArguments,
               newStatus: RuleExecutionStatus.succeeded,
               message: 'succeeded',
               metrics: {
-                bulkCreateTimeDurations: result.bulkCreateTimes,
-                searchAfterTimeDurations: result.searchAfterTimes,
+                indexingDurations: result.bulkCreateTimes,
+                searchDurations: result.searchAfterTimes,
                 lastLookBackDate: result.lastLookBackDate?.toISOString(),
               },
             });
@@ -474,13 +475,12 @@ export const signalRulesAlertType = ({
           );
           logger.error(errorMessage);
           await ruleStatusClient.logStatusChange({
-            spaceId,
-            ruleId: alertId,
+            ...basicLogArguments,
             newStatus: RuleExecutionStatus.failed,
             message: errorMessage,
             metrics: {
-              bulkCreateTimeDurations: result.bulkCreateTimes,
-              searchAfterTimeDurations: result.searchAfterTimes,
+              indexingDurations: result.bulkCreateTimes,
+              searchDurations: result.searchAfterTimes,
               lastLookBackDate: result.lastLookBackDate?.toISOString(),
             },
           });
@@ -494,13 +494,12 @@ export const signalRulesAlertType = ({
 
         logger.error(message);
         await ruleStatusClient.logStatusChange({
-          spaceId,
-          ruleId: alertId,
+          ...basicLogArguments,
           newStatus: RuleExecutionStatus.failed,
           message,
           metrics: {
-            bulkCreateTimeDurations: result.bulkCreateTimes,
-            searchAfterTimeDurations: result.searchAfterTimes,
+            indexingDurations: result.bulkCreateTimes,
+            searchDurations: result.searchAfterTimes,
             lastLookBackDate: result.lastLookBackDate?.toISOString(),
           },
         });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -255,6 +255,7 @@ export interface SignalHit {
 
 export interface AlertAttributes<T extends RuleParams = RuleParams> {
   actions: RuleAlertAction[];
+  alertTypeId: string;
   enabled: boolean;
   name: string;
   tags: string[];

x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts

@@ -789,6 +789,7 @@ describe('utils', () => {
         inputIndices: ['myfa*'],
         ruleStatusClient,
         ruleId: 'ruleId',
+        ruleType: 'ruleType',
         spaceId: 'default',
         logger: mockLogger,
         buildRuleMessage,
@@ -832,6 +833,7 @@ describe('utils', () => {
         inputIndices: ['myfa*'],
         ruleStatusClient,
         ruleId: 'ruleId',
+        ruleType: 'ruleType',
         spaceId: 'default',
         logger: mockLogger,
         buildRuleMessage,
@@ -861,6 +863,7 @@ describe('utils', () => {
         inputIndices: ['logs-endpoint.alerts-*'],
         ruleStatusClient,
         ruleId: 'ruleId',
+        ruleType: 'ruleType',
         spaceId: 'default',
         logger: mockLogger,
         buildRuleMessage,
@@ -890,6 +893,7 @@ describe('utils', () => {
         inputIndices: ['logs-endpoint.alerts-*'],
         ruleStatusClient,
         ruleId: 'ruleId',
+        ruleType: 'ruleType',
         spaceId: 'default',
         logger: mockLogger,
         buildRuleMessage,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts

@@ -99,9 +99,20 @@ export const hasReadIndexPrivileges = async (args: {
   buildRuleMessage: BuildRuleMessage;
   ruleStatusClient: IRuleExecutionLogClient;
   ruleId: string;
+  ruleName: string;
+  ruleType: string;
   spaceId: string;
 }): Promise<boolean> => {
-  const { privileges, logger, buildRuleMessage, ruleStatusClient, ruleId, spaceId } = args;
+  const {
+    privileges,
+    logger,
+    buildRuleMessage,
+    ruleStatusClient,
+    ruleId,
+    ruleName,
+    ruleType,
+    spaceId,
+  } = args;
 
   const indexNames = Object.keys(privileges.index);
   const [indexesWithReadPrivileges, indexesWithNoReadPrivileges] = partition(
@@ -119,6 +130,8 @@ export const hasReadIndexPrivileges = async (args: {
     await ruleStatusClient.logStatusChange({
       message: errorString,
       ruleId,
+      ruleName,
+      ruleType,
       spaceId,
       newStatus: RuleExecutionStatus['partial failure'],
     });
@@ -136,6 +149,8 @@ export const hasReadIndexPrivileges = async (args: {
     await ruleStatusClient.logStatusChange({
       message: errorString,
       ruleId,
+      ruleName,
+      ruleType,
       spaceId,
       newStatus: RuleExecutionStatus['partial failure'],
     });
@@ -156,6 +171,7 @@ export const hasTimestampFields = async (args: {
   ruleStatusClient: IRuleExecutionLogClient;
   ruleId: string;
   spaceId: string;
+  ruleType: string;
   logger: Logger;
   buildRuleMessage: BuildRuleMessage;
 }): Promise<boolean> => {
@@ -167,6 +183,7 @@ export const hasTimestampFields = async (args: {
     inputIndices,
     ruleStatusClient,
     ruleId,
+    ruleType,
     spaceId,
     logger,
     buildRuleMessage,
@@ -184,6 +201,8 @@ export const hasTimestampFields = async (args: {
     await ruleStatusClient.logStatusChange({
       message: errorString.trimEnd(),
       ruleId,
+      ruleName,
+      ruleType,
       spaceId,
       newStatus: RuleExecutionStatus['partial failure'],
     });
@@ -210,6 +229,8 @@ export const hasTimestampFields = async (args: {
     await ruleStatusClient.logStatusChange({
       message: errorString,
       ruleId,
+      ruleName,
+      ruleType,
       spaceId,
       newStatus: RuleExecutionStatus['partial failure'],
     });

x-pack/plugins/security_solution/server/plugin.ts

@@ -109,11 +109,14 @@ import { ctiFieldMap } from './lib/detection_engine/rule_types/field_maps/cti';
 import { legacyRulesNotificationAlertType } from './lib/detection_engine/notifications/legacy_rules_notification_alert_type';
 // eslint-disable-next-line no-restricted-imports
 import { legacyIsNotificationAlertExecutor } from './lib/detection_engine/notifications/legacy_types';
+import { IEventLogClientService, IEventLogService } from '../../event_log/server';
+import { registerEventLogProvider } from './lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider';
 
 export interface SetupPlugins {
   alerting: AlertingSetup;
   data: DataPluginSetup;
   encryptedSavedObjects?: EncryptedSavedObjectsSetup;
+  eventLog: IEventLogService;
   features: FeaturesSetup;
   lists?: ListPluginSetup;
   ml?: MlSetup;
@@ -121,20 +124,21 @@ export interface SetupPlugins {
   security?: SecuritySetup;
   spaces?: SpacesSetup;
   taskManager?: TaskManagerSetupContract;
-  usageCollection?: UsageCollectionSetup;
   telemetry?: TelemetryPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export interface StartPlugins {
   alerting: AlertPluginStartContract;
+  cases?: CasesPluginStartContract;
   data: DataPluginStart;
+  eventLog: IEventLogClientService;
   fleet?: FleetStartContract;
   licensing: LicensingPluginStart;
   ruleRegistry: RuleRegistryPluginStartContract;
+  security: SecurityPluginStart;
   taskManager?: TaskManagerStartContract;
   telemetry?: TelemetryPluginStart;
-  security: SecurityPluginStart;
-  cases?: CasesPluginStartContract;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -202,6 +206,9 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
 
     this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID);
 
+    const eventLogService = plugins.eventLog;
+    registerEventLogProvider(eventLogService);
+
     const router = core.http.createRouter<SecuritySolutionRequestHandlerContext>();
     core.http.registerRouteHandlerContext<SecuritySolutionRequestHandlerContext, typeof APP_ID>(
       APP_ID,
@@ -210,8 +217,9 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
         getSpaceId: () => plugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID,
         getExecutionLogClient: () =>
           new RuleExecutionLogClient({
-            ruleDataService: plugins.ruleRegistry.ruleDataService,
             savedObjectsClient: context.core.savedObjects.client,
+            eventLogService,
+            underlyingClient: config.ruleExecutionLog.underlyingClient,
           }),
       })
     );
@@ -262,11 +270,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
         experimentalFeatures,
         lists: plugins.lists,
         logger: this.logger,
-        mergeStrategy: this.config.alertMergeStrategy,
-        ignoreFields: this.config.alertIgnoreFields,
+        config: this.config,
         ml: plugins.ml,
         ruleDataClient,
-        ruleDataService,
+        eventLogService,
         version: this.context.env.packageInfo.version,
       };
 
@@ -318,10 +325,9 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
         version: this.context.env.packageInfo.version,
         ml: plugins.ml,
         lists: plugins.lists,
-        mergeStrategy: this.config.alertMergeStrategy,
-        ignoreFields: this.config.alertIgnoreFields,
+        config: this.config,
         experimentalFeatures,
-        ruleDataService: plugins.ruleRegistry.ruleDataService,
+        eventLogService,
       });
       const ruleNotificationType = legacyRulesNotificationAlertType({
         logger: this.logger,

x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts

@@ -150,6 +150,21 @@ export default function ({ getService }: FtrProviderContext) {
         },
         kibana: {
           saved_objects: [savedObject],
+          space_ids: ['space id'],
+          alert: {
+            rule: {
+              execution: {
+                uuid: '1fef0e1a-25ba-11ec-9621-0242ac130002',
+                status: 'succeeded',
+                status_order: 10,
+                metrics: {
+                  total_indexing_duration_ms: 1000,
+                  total_search_duration_ms: 2000,
+                  execution_gap_duration_s: 3000,
+                },
+              },
+            },
+          },
           alerting: {
             instance_id: 'alert instance id',
             action_group_id: 'alert action group',