[8.8] [Security Solution] PoC of the Detection Engine health API (#157155) (#159717)

# Backport

This will backport the following PR from `main` to `8.8`:
 - https://github.com/elastic/kibana/pull/157155

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

packages/kbn-securitysolution-io-ts-types/src/iso_date_string/index.ts

@@ -13,6 +13,7 @@ import { Either } from 'fp-ts/lib/Either';
  * Types the IsoDateString as:
  *   - A string that is an ISOString
  */
+export type IsoDateString = t.TypeOf<typeof IsoDateString>;
 export const IsoDateString = new t.Type<string, string, unknown>(
   'IsoDateString',
   t.string.is,

x-pack/plugins/alerting/server/rules_client/lib/validate_rule_aggregation_fields.ts

@@ -17,7 +17,8 @@ const ALLOW_FIELDS = [
   'alert.attributes.snoozeSchedule.duration',
   'alert.attributes.alertTypeId',
   'alert.attributes.enabled',
-  'alert.attributes.params.*',
+  'alert.attributes.params.*', // TODO: https://github.com/elastic/kibana/issues/159602
+  'alert.attributes.params.immutable', // TODO: Remove after addressing https://github.com/elastic/kibana/issues/159602
 ];
 
 const ALLOW_AGG_TYPES = ['terms', 'composite', 'nested', 'filter'];

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health_schemas.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import type { HealthInterval } from '../../model/detection_engine_health/health_interval';
+import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval';
+import type { HealthTimings } from '../../model/detection_engine_health/health_metadata';
+import type {
+  ClusterHealthParameters,
+  ClusterHealthSnapshot,
+} from '../../model/detection_engine_health/cluster_health';
+
+/**
+ * Schema for the request body of the endpoint.
+ */
+export type GetClusterHealthRequestBody = t.TypeOf<typeof GetClusterHealthRequestBody>;
+export const GetClusterHealthRequestBody = t.exact(
+  t.partial({
+    interval: HealthIntervalParameters,
+    debug: t.boolean,
+  })
+);
+
+/**
+ * Validated and normalized request parameters of the endpoint.
+ */
+export interface GetClusterHealthRequest {
+  /**
+   * Time period over which health stats are requested.
+   */
+  interval: HealthInterval;
+
+  /**
+   * If true, the endpoint will return various debug information, such as
+   * aggregations sent to Elasticsearch and response received from Elasticsearch.
+   */
+  debug: boolean;
+
+  /**
+   * Timestamp at which the route handler started executing.
+   */
+  requestReceivedAt: IsoDateString;
+}
+
+/**
+ * Response body of the endpoint.
+ */
+export interface GetClusterHealthResponse {
+  // TODO: https://github.com/elastic/kibana/issues/125642 Implement the endpoint and remove the `message` property
+  message: 'Not implemented';
+
+  /**
+   * Request processing times and durations.
+   */
+  timings: HealthTimings;
+
+  /**
+   * Parameters of the health stats calculation.
+   */
+  parameters: ClusterHealthParameters;
+
+  /**
+   * Result of the health stats calculation.
+   */
+  health: ClusterHealthSnapshot;
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health_schemas.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import { NonEmptyString } from '@kbn/securitysolution-io-ts-types';
+
+import type { HealthInterval } from '../../model/detection_engine_health/health_interval';
+import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval';
+import type { HealthTimings } from '../../model/detection_engine_health/health_metadata';
+import type {
+  RuleHealthParameters,
+  RuleHealthSnapshot,
+} from '../../model/detection_engine_health/rule_health';
+
+/**
+ * Schema for the request body of the endpoint.
+ */
+export type GetRuleHealthRequestBody = t.TypeOf<typeof GetRuleHealthRequestBody>;
+export const GetRuleHealthRequestBody = t.exact(
+  t.intersection([
+    t.type({
+      rule_id: NonEmptyString,
+    }),
+    t.partial({
+      interval: HealthIntervalParameters,
+      debug: t.boolean,
+    }),
+  ])
+);
+
+/**
+ * Validated and normalized request parameters of the endpoint.
+ */
+export interface GetRuleHealthRequest {
+  /**
+   * Saved object ID of the rule to calculate health stats for.
+   */
+  ruleId: string;
+
+  /**
+   * Time period over which health stats are requested.
+   */
+  interval: HealthInterval;
+
+  /**
+   * If true, the endpoint will return various debug information, such as
+   * aggregations sent to Elasticsearch and response received from Elasticsearch.
+   */
+  debug: boolean;
+
+  /**
+   * Timestamp at which the route handler started executing.
+   */
+  requestReceivedAt: IsoDateString;
+}
+
+/**
+ * Response body of the endpoint.
+ */
+export interface GetRuleHealthResponse {
+  /**
+   * Request processing times and durations.
+   */
+  timings: HealthTimings;
+
+  /**
+   * Parameters of the health stats calculation.
+   */
+  parameters: RuleHealthParameters;
+
+  /**
+   * Result of the health stats calculation.
+   */
+  health: RuleHealthSnapshot;
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health_schemas.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import type { HealthInterval } from '../../model/detection_engine_health/health_interval';
+import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval';
+import type { HealthTimings } from '../../model/detection_engine_health/health_metadata';
+import type {
+  SpaceHealthParameters,
+  SpaceHealthSnapshot,
+} from '../../model/detection_engine_health/space_health';
+
+/**
+ * Schema for the request body of the endpoint.
+ */
+export type GetSpaceHealthRequestBody = t.TypeOf<typeof GetSpaceHealthRequestBody>;
+export const GetSpaceHealthRequestBody = t.exact(
+  t.partial({
+    interval: HealthIntervalParameters,
+    debug: t.boolean,
+  })
+);
+
+/**
+ * Validated and normalized request parameters of the endpoint.
+ */
+export interface GetSpaceHealthRequest {
+  /**
+   * Time period over which health stats are requested.
+   */
+  interval: HealthInterval;
+
+  /**
+   * If true, the endpoint will return various debug information, such as
+   * aggregations sent to Elasticsearch and response received from Elasticsearch.
+   */
+  debug: boolean;
+
+  /**
+   * Timestamp at which the route handler started executing.
+   */
+  requestReceivedAt: IsoDateString;
+}
+
+/**
+ * Response body of the endpoint.
+ */
+export interface GetSpaceHealthResponse {
+  /**
+   * Request processing times and durations.
+   */
+  timings: HealthTimings;
+
+  /**
+   * Parameters of the health stats calculation.
+   */
+  parameters: SpaceHealthParameters;
+
+  /**
+   * Result of the health stats calculation.
+   */
+  health: SpaceHealthSnapshot;
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.ts

@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import * as t from 'io-ts';
-import { PaginationResult } from '../../../schemas/common';
-import { RuleExecutionEvent } from '../../model/execution_event';
-
-/**
- * Response body of the API route.
- */
-export type GetRuleExecutionEventsResponse = t.TypeOf<typeof GetRuleExecutionEventsResponse>;
-export const GetRuleExecutionEventsResponse = t.exact(
-  t.type({
-    events: t.array(RuleExecutionEvent),
-    pagination: PaginationResult,
-  })
-);

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.ts

@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import * as t from 'io-ts';
-import { RuleExecutionResult } from '../../model/execution_result';
-
-/**
- * Response body of the API route.
- */
-export type GetRuleExecutionResultsResponse = t.TypeOf<typeof GetRuleExecutionResultsResponse>;
-export const GetRuleExecutionResultsResponse = t.exact(
-  t.type({
-    events: t.array(RuleExecutionResult),
-    total: t.number,
-  })
-);

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.mock.ts

@@ -6,7 +6,7 @@
  */
 
 import { ruleExecutionEventMock } from '../../model/execution_event.mock';
-import type { GetRuleExecutionEventsResponse } from './response_schema';
+import type { GetRuleExecutionEventsResponse } from './get_rule_execution_events_schemas';
 
 const getSomeResponse = (): GetRuleExecutionEventsResponse => {
   const events = ruleExecutionEventMock.getSomeEvents();

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.test.ts

@@ -12,7 +12,7 @@ import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils';
 import {
   GetRuleExecutionEventsRequestParams,
   GetRuleExecutionEventsRequestQuery,
-} from './request_schema';
+} from './get_rule_execution_events_schemas';
 
 describe('Request schema of Get rule execution events', () => {
   describe('GetRuleExecutionEventsRequestParams', () => {

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.ts

@@ -10,8 +10,8 @@ import * as t from 'io-ts';
 import { DefaultPerPage, DefaultPage } from '@kbn/securitysolution-io-ts-alerting-types';
 import { defaultCsvArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
 
-import { DefaultSortOrderDesc } from '../../../schemas/common';
-import { TRuleExecutionEventType } from '../../model/execution_event';
+import { DefaultSortOrderDesc, PaginationResult } from '../../../schemas/common';
+import { RuleExecutionEvent, TRuleExecutionEventType } from '../../model/execution_event';
 import { TLogLevel } from '../../model/log_level';
 
 /**
@@ -41,3 +41,14 @@ export const GetRuleExecutionEventsRequestQuery = t.exact(
     per_page: DefaultPerPage, // defaults to 20
   })
 );
+
+/**
+ * Response body of the API route.
+ */
+export type GetRuleExecutionEventsResponse = t.TypeOf<typeof GetRuleExecutionEventsResponse>;
+export const GetRuleExecutionEventsResponse = t.exact(
+  t.type({
+    events: t.array(RuleExecutionEvent),
+    pagination: PaginationResult,
+  })
+);

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.mock.ts

@@ -6,7 +6,7 @@
  */
 
 import { ruleExecutionResultMock } from '../../model/execution_result.mock';
-import type { GetRuleExecutionResultsResponse } from './response_schema';
+import type { GetRuleExecutionResultsResponse } from './get_rule_execution_results_schemas';
 
 const getSomeResponse = (): GetRuleExecutionResultsResponse => {
   const results = ruleExecutionResultMock.getSomeResults();

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.test.ts

@@ -10,7 +10,10 @@ import { left } from 'fp-ts/lib/Either';
 import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils';
 
 import { RULE_EXECUTION_STATUSES } from '../../model/execution_status';
-import { DefaultSortField, DefaultRuleExecutionStatusCsvArray } from './request_schema';
+import {
+  DefaultSortField,
+  DefaultRuleExecutionStatusCsvArray,
+} from './get_rule_execution_results_schemas';
 
 describe('Request schema of Get rule execution results', () => {
   describe('DefaultRuleExecutionStatusCsvArray', () => {

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.ts

@@ -17,7 +17,7 @@ import {
 } from '@kbn/securitysolution-io-ts-types';
 
 import { DefaultSortOrderDesc } from '../../../schemas/common';
-import { SortFieldOfRuleExecutionResult } from '../../model/execution_result';
+import { RuleExecutionResult, SortFieldOfRuleExecutionResult } from '../../model/execution_result';
 import { TRuleExecutionStatus } from '../../model/execution_status';
 
 /**
@@ -70,3 +70,14 @@ export const GetRuleExecutionResultsRequestQuery = t.exact(
     per_page: DefaultPerPage, // defaults to 20
   })
 );
+
+/**
+ * Response body of the API route.
+ */
+export type GetRuleExecutionResultsResponse = t.TypeOf<typeof GetRuleExecutionResultsResponse>;
+export const GetRuleExecutionResultsResponse = t.exact(
+  t.type({
+    events: t.array(RuleExecutionResult),
+    total: t.number,
+  })
+);

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts

@@ -7,11 +7,43 @@
 
 import { INTERNAL_DETECTION_ENGINE_URL as INTERNAL_URL } from '../../../constants';
 
+// -------------------------------------------------------------------------------------------------
+// Detection Engine health API
+
+/**
+ * Get health overview of the whole cluster. Scope: all detection rules in all Kibana spaces.
+ * See the corresponding route handler for more details.
+ */
+export const GET_CLUSTER_HEALTH_URL = `${INTERNAL_URL}/health/_cluster` as const;
+
+/**
+ * Get health overview of the current Kibana space. Scope: all detection rules in the space.
+ * See the corresponding route handler for more details.
+ */
+export const GET_SPACE_HEALTH_URL = `${INTERNAL_URL}/health/_space` as const;
+
+/**
+ * Get health overview of a rule. Scope: a given detection rule in the current Kibana space.
+ * See the corresponding route handler for more details.
+ */
+export const GET_RULE_HEALTH_URL = `${INTERNAL_URL}/health/_rule` as const;
+
+// -------------------------------------------------------------------------------------------------
+// Rule execution logs API
+
+/**
+ * Get plain individual rule execution events, such as status changes, execution metrics,
+ * log messages, etc.
+ */
 export const GET_RULE_EXECUTION_EVENTS_URL =
   `${INTERNAL_URL}/rules/{ruleId}/execution/events` as const;
 export const getRuleExecutionEventsUrl = (ruleId: string) =>
   `${INTERNAL_URL}/rules/${ruleId}/execution/events` as const;
 
+/**
+ * Get aggregated rule execution results. Each result object is built on top of all individual
+ * events logged during the corresponding rule execution.
+ */
 export const GET_RULE_EXECUTION_RESULTS_URL =
   `${INTERNAL_URL}/rules/{ruleId}/execution/results` as const;
 export const getRuleExecutionResultsUrl = (ruleId: string) =>

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/index.ts

@@ -5,12 +5,19 @@
  * 2.0.
  */
 
-export * from './api/get_rule_execution_events/request_schema';
-export * from './api/get_rule_execution_events/response_schema';
-export * from './api/get_rule_execution_results/request_schema';
-export * from './api/get_rule_execution_results/response_schema';
+export * from './api/detection_engine_health/get_cluster_health_schemas';
+export * from './api/detection_engine_health/get_rule_health_schemas';
+export * from './api/detection_engine_health/get_space_health_schemas';
+export * from './api/rule_execution_logs/get_rule_execution_events_schemas';
+export * from './api/rule_execution_logs/get_rule_execution_results_schemas';
 export * from './api/urls';
 
+export * from './model/detection_engine_health/cluster_health';
+export * from './model/detection_engine_health/health_interval';
+export * from './model/detection_engine_health/health_metadata';
+export * from './model/detection_engine_health/health_stats';
+export * from './model/detection_engine_health/rule_health';
+export * from './model/detection_engine_health/space_health';
 export * from './model/execution_event';
 export * from './model/execution_metrics';
 export * from './model/execution_result';

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/mocks.ts

@@ -5,9 +5,12 @@
  * 2.0.
  */
 
-export * from './api/get_rule_execution_events/response_schema.mock';
-export * from './api/get_rule_execution_results/response_schema.mock';
+export * from './api/rule_execution_logs/get_rule_execution_events_schemas.mock';
+export * from './api/rule_execution_logs/get_rule_execution_results_schemas.mock';
 
+export * from './model/detection_engine_health/cluster_health.mock';
+export * from './model/detection_engine_health/rule_health.mock';
+export * from './model/detection_engine_health/space_health.mock';
 export * from './model/execution_event.mock';
 export * from './model/execution_result.mock';
 export * from './model/execution_summary.mock';

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.mock.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ClusterHealthSnapshot } from './cluster_health';
+import { healthStatsMock } from './health_stats.mock';
+
+const getEmptyClusterHealthSnapshot = (): ClusterHealthSnapshot => {
+  return {
+    stats_at_the_moment: healthStatsMock.getEmptyRuleStats(),
+    stats_over_interval: {
+      message: 'Not implemented',
+    },
+    history_over_interval: {
+      buckets: [
+        {
+          timestamp: '2023-05-15T16:12:14.967Z',
+          stats: {
+            message: 'Not implemented',
+          },
+        },
+      ],
+    },
+  };
+};
+
+export const clusterHealthSnapshotMock = {
+  getEmptyClusterHealthSnapshot,
+};

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { HealthParameters, HealthSnapshot } from './health_metadata';
+import type { RuleStats, StatsHistory } from './health_stats';
+
+/**
+ * Health calculation parameters for the whole cluster.
+ */
+export type ClusterHealthParameters = HealthParameters;
+
+/**
+ * Health calculation result for the whole cluster.
+ */
+export interface ClusterHealthSnapshot extends HealthSnapshot {
+  /**
+   * Health stats at the moment of the calculation request.
+   */
+  stats_at_the_moment: ClusterHealthStatsAtTheMoment;
+
+  /**
+   * Health stats calculated over the interval specified in the health parameters.
+   */
+  stats_over_interval: ClusterHealthStatsOverInterval;
+
+  /**
+   * History of change of the same health stats during the interval.
+   */
+  history_over_interval: StatsHistory<ClusterHealthStatsOverInterval>;
+}
+
+/**
+ * Health stats at the moment of the calculation request.
+ */
+export type ClusterHealthStatsAtTheMoment = RuleStats;
+
+/**
+ * Health stats calculated over a given interval.
+ */
+export interface ClusterHealthStatsOverInterval {
+  // TODO: https://github.com/elastic/kibana/issues/125642 Implement and delete this `message`
+  message: 'Not implemented';
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_interval.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+
+/**
+ * Type of the health interval. You can specify:
+ * - a relative interval, e.g. "last_hour" = [now-1h; now] where "now" is when health request is made
+ * - a custom interval with "from" and "to" timestamps
+ */
+export enum HealthIntervalType {
+  'last_hour' = 'last_hour',
+  'last_day' = 'last_day',
+  'last_week' = 'last_week',
+  'last_month' = 'last_month',
+  'last_year' = 'last_year',
+  'custom_range' = 'custom_range',
+}
+
+/**
+ * Granularity defines how the whole health interval will be split into smaller sub-intervals.
+ * Health stats will be calculated for the whole interval + for each sub-interval.
+ * Example: if the interval is "last_day" and the granularity is "hour", stats will be calculated:
+ * - 1 time for the last 24 hours
+ * - 24 times for each hour in that interval
+ */
+export enum HealthIntervalGranularity {
+  'minute' = 'minute',
+  'hour' = 'hour',
+  'day' = 'day',
+  'week' = 'week',
+  'month' = 'month',
+}
+
+/**
+ * Time period over which we calculate health stats.
+ * This is a "raw" schema for the interval parameters that users can pass to the API.
+ */
+export type HealthIntervalParameters = t.TypeOf<typeof HealthIntervalParameters>;
+export const HealthIntervalParameters = t.union([
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.last_hour),
+      granularity: t.literal(HealthIntervalGranularity.minute),
+    })
+  ),
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.last_day),
+      granularity: t.union([
+        t.literal(HealthIntervalGranularity.minute),
+        t.literal(HealthIntervalGranularity.hour),
+      ]),
+    })
+  ),
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.last_week),
+      granularity: t.union([
+        t.literal(HealthIntervalGranularity.hour),
+        t.literal(HealthIntervalGranularity.day),
+      ]),
+    })
+  ),
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.last_month),
+      granularity: t.union([
+        t.literal(HealthIntervalGranularity.day),
+        t.literal(HealthIntervalGranularity.week),
+      ]),
+    })
+  ),
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.last_year),
+      granularity: t.union([
+        t.literal(HealthIntervalGranularity.week),
+        t.literal(HealthIntervalGranularity.month),
+      ]),
+    })
+  ),
+  t.exact(
+    t.type({
+      type: t.literal(HealthIntervalType.custom_range),
+      granularity: t.union([
+        t.literal(HealthIntervalGranularity.minute),
+        t.literal(HealthIntervalGranularity.hour),
+        t.literal(HealthIntervalGranularity.day),
+        t.literal(HealthIntervalGranularity.week),
+        t.literal(HealthIntervalGranularity.month),
+      ]),
+      from: IsoDateString,
+      to: IsoDateString,
+    })
+  ),
+]);
+
+/**
+ * Time period over which we calculate health stats.
+ * This interface represents a fully validated and normalized interval object.
+ */
+export interface HealthInterval {
+  /**
+   * Type of the interval. Defined by the user.
+   * @example 'last_week'
+   */
+  type: HealthIntervalType;
+
+  /**
+   * Granularity of the interval. Defined by the user.
+   * @example 'day'
+   */
+  granularity: HealthIntervalGranularity;
+
+  /**
+   * Start timestamp of the interval. Calculated by the app.
+   * @example '2023-05-19T14:25:19.092Z'
+   */
+  from: IsoDateString;
+
+  /**
+   * End timestamp of the interval. Calculated by the app.
+   * @example '2023-05-26T14:25:19.092Z'
+   */
+  to: IsoDateString;
+
+  /**
+   * Duration of the interval in the ISO format. Calculated by the app.
+   * @example 'PT168H'
+   */
+  duration: string;
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_metadata.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import type { HealthInterval } from './health_interval';
+
+/**
+ * Health request processing times and durations.
+ * This metadata is included in the health API responses.
+ */
+export interface HealthTimings {
+  /**
+   * Timestamp at which health calculation request was received.
+   */
+  requested_at: IsoDateString;
+
+  /**
+   * Timestamp at which health stats were calculated and returned.
+   */
+  processed_at: IsoDateString;
+
+  /**
+   * How much time it took to calculate health stats, in milliseconds.
+   */
+  processing_time_ms: number;
+}
+
+/**
+ * Base parameters of all the health API endpoints.
+ * This metadata is included in the health API responses.
+ */
+export interface HealthParameters {
+  /**
+   * Time period over which we calculate health stats.
+   */
+  interval: HealthInterval;
+}
+
+/**
+ * Base properties of a health snapshot (health calculation result at a given moment).
+ */
+export interface HealthSnapshot {
+  /**
+   * Optional debug information, such as requests and aggregations sent to Elasticsearch
+   * and responses received from Elasticsearch.
+   */
+  debug?: Record<string, unknown>;
+}

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.mock.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  AggregatedMetric,
+  RuleExecutionStats,
+  RuleStats,
+  TotalEnabledDisabled,
+} from './health_stats';
+
+const getEmptyRuleStats = (): RuleStats => {
+  return {
+    number_of_rules: {
+      all: getZeroTotalEnabledDisabled(),
+      by_origin: {
+        prebuilt: getZeroTotalEnabledDisabled(),
+        custom: getZeroTotalEnabledDisabled(),
+      },
+      by_type: {},
+      by_outcome: {},
+    },
+  };
+};
+
+const getZeroTotalEnabledDisabled = (): TotalEnabledDisabled => {
+  return {
+    total: 0,
+    enabled: 0,
+    disabled: 0,
+  };
+};
+
+const getEmptyRuleExecutionStats = (): RuleExecutionStats => {
+  return {
+    number_of_executions: {
+      total: 0,
+      by_outcome: {
+        succeeded: 0,
+        warning: 0,
+        failed: 0,
+      },
+    },
+    number_of_logged_messages: {
+      total: 0,
+      by_level: {
+        error: 0,
+        warn: 0,
+        info: 0,
+        debug: 0,
+        trace: 0,
+      },
+    },
+    number_of_detected_gaps: {
+      total: 0,
+      total_duration_s: 0,
+    },
+    schedule_delay_ms: getZeroAggregatedMetric(),
+    execution_duration_ms: getZeroAggregatedMetric(),
+    search_duration_ms: getZeroAggregatedMetric(),
+    indexing_duration_ms: getZeroAggregatedMetric(),
+    top_errors: [],
+    top_warnings: [],
+  };
+};
+
+const getZeroAggregatedMetric = (): AggregatedMetric<number> => {
+  return {
+    percentiles: {
+      '1.0': 0,
+      '5.0': 0,
+      '25.0': 0,
+      '50.0': 0,
+      '75.0': 0,
+      '95.0': 0,
+      '99.0': 0,
+    },
+  };
+};
+
+export const healthStatsMock = {
+  getEmptyRuleStats,
+  getEmptyRuleExecutionStats,
+};

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.ts

@@ -0,0 +1,269 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import type { RuleLastRunOutcomes } from '@kbn/alerting-plugin/common';
+import type { LogLevel } from '../log_level';
+
+// -------------------------------------------------------------------------------------------------
+// Stats history (date histogram)
+
+/**
+ * History of change of a set of stats over a time interval. The interval is split into discreet buckets,
+ * each bucket is a smaller sub-interval with stats calculated over this sub-interval.
+ *
+ * This model corresponds to the `date_histogram` aggregation of Elasticsearch:
+ * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html
+ */
+export interface StatsHistory<TStats> {
+  buckets: Array<StatsBucket<TStats>>;
+}
+
+/**
+ * Sub-interval with stats calculated over it.
+ */
+export interface StatsBucket<TStats> {
+  /**
+   * Start timestamp of the sub-interval.
+   */
+  timestamp: IsoDateString;
+
+  /**
+   * Set of stats.
+   */
+  stats: TStats;
+}
+
+// -------------------------------------------------------------------------------------------------
+// Rule stats
+
+// TODO: https://github.com/elastic/kibana/issues/125642 Add more stats, such as:
+// - number of Kibana instances
+// - number of Kibana spaces
+// - number of rules with exceptions
+// - number of rules with notification actions (total, normal, legacy)
+// - number of rules with response actions
+// - top X last failed status messages + rule ids for each status
+// - top X last partial failure status messages + rule ids for each status
+// - top X slowest rules by any metrics (last total execution time, search time, indexing time, etc)
+// - top X rules with the largest schedule delay (drift)
+
+/**
+ * "Static" stats calculated for a set of rules, such as number of enabled and disabled rules, etc.
+ */
+export interface RuleStats {
+  /**
+   * Various counts of different rules.
+   */
+  number_of_rules: NumberOfRules;
+}
+
+/**
+ * Various counts of different rules.
+ */
+export interface NumberOfRules {
+  /**
+   * Total number of all rules, and how many of them are enabled and disabled.
+   */
+  all: TotalEnabledDisabled;
+
+  /**
+   * Number of prebuilt and custom rules, and how many of them are enabled and disabled.
+   */
+  by_origin: Record<'prebuilt' | 'custom', TotalEnabledDisabled>;
+
+  /**
+   * Number of rules of each type, and how many of them are enabled and disabled.
+   */
+  by_type: Record<string, TotalEnabledDisabled>;
+
+  /**
+   * Number of rules by last execution outcome, and how many of them are enabled and disabled.
+   */
+  by_outcome: Record<string, TotalEnabledDisabled>;
+}
+
+/**
+ * Number of rules in a given set, and how many of them are enabled and disabled.
+ */
+export interface TotalEnabledDisabled {
+  /**
+   * Total number of rules in a set.
+   */
+  total: number;
+
+  /**
+   * Number of enabled rules in a set.
+   */
+  enabled: number;
+
+  /**
+   * Number of disabled rules in a set.
+   */
+  disabled: number;
+}
+
+// -------------------------------------------------------------------------------------------------
+// Rule execution stats
+
+// TODO: https://github.com/elastic/kibana/issues/125642 Add more stats, such as:
+// - number of detected alerts (source event "hits")
+// - number of created alerts (those we wrote to the .alerts-* index)
+// - number of times rule hit cirquit breaker, number of not created alerts because of that
+// - number of triggered actions
+// - top gaps
+
+/**
+ * "Dynamic" rule execution stats. Can be calculated either for a set of rules or for a single rule.
+ */
+export interface RuleExecutionStats {
+  /**
+   * Number of rule executions.
+   */
+  number_of_executions: NumberOfExecutions;
+
+  /**
+   * Number of events containing some message that were written to the Event Log.
+   */
+  number_of_logged_messages: NumberOfLoggedMessages;
+
+  /**
+   * Stats for detected gaps in rule execution.
+   */
+  number_of_detected_gaps: NumberOfDetectedGaps;
+
+  /**
+   * Aggregated schedule delay of a rule, in milliseconds.
+   * Also called "drift" in the Task Manager health API.
+   * This metric shows if rules start executing on time according to their schedule
+   * (in that case, it should be ideally zero, but in practice will be 3-5 seconds),
+   * or their start time gets delayed (when the cluster is overloaded it could be
+   * minutes or even hours).
+   */
+  schedule_delay_ms: AggregatedMetric<number>;
+
+  /**
+   * Aggregated total execution duration of a rule, in milliseconds.
+   */
+  execution_duration_ms: AggregatedMetric<number>;
+
+  /**
+   * Aggregated total search duration of a rule, in milliseconds.
+   * This metric shows how much time a rule spends for querying source indices.
+   */
+  search_duration_ms: AggregatedMetric<number>;
+
+  /**
+   * Aggregated total indexing duration of a rule, in milliseconds.
+   * This metric shows how much time a rule spends for writing generated alerts.
+   */
+  indexing_duration_ms: AggregatedMetric<number>;
+
+  /**
+   * N most frequent error messages logged by rule(s) to Event Log.
+   */
+  top_errors?: TopMessages;
+
+  /**
+   * N most frequent warning messages logged by rule(s) to Event Log.
+   */
+  top_warnings?: TopMessages;
+}
+
+/**
+ * Number of rule executions.
+ */
+export interface NumberOfExecutions {
+  /**
+   * Total number of rule executions.
+   */
+  total: number;
+
+  /**
+   * Number of executions by each possible execution outcome.
+   */
+  by_outcome: Record<RuleLastRunOutcomes, number>;
+}
+
+/**
+ * Number of events containing some message that were written to the Event Log.
+ */
+export interface NumberOfLoggedMessages {
+  /**
+   * Total number of message-containing events.
+   */
+  total: number;
+
+  /**
+   * Number of message-containing events by each log level.
+   */
+  by_level: Record<LogLevel, number>;
+}
+
+/**
+ * Stats for detected gaps in rule execution.
+ */
+export interface NumberOfDetectedGaps {
+  /**
+   * Total number of detected gaps.
+   */
+  total: number;
+
+  /**
+   * Sum of durations of all the detected gaps, in seconds.
+   */
+  total_duration_s: number;
+}
+
+/**
+ * When a rule runs, we calculate a bunch of rule execution metrics for a given rule run.
+ * Later, we can aggregate each metric in different ways:
+ * - for a single rule, aggregate over a time interval
+ * - for multiple rules, aggregate over a time interval
+ * - for multiple rules, aggregate over the rules at a given moment (e.g. now)
+ *
+ * For example, if the metric is "total rule execution duration", we could:
+ * - calculate average execution duration of a single rule over last week
+ * - calculate average execution duration of all rules in a space over last week
+ * - calculate average last execution duration of all rules in a space at the moment
+ *
+ * Instead of calculating only averages, we calculate a set of percentiles that can give
+ * a better picture of the metric's distribution.
+ */
+export interface AggregatedMetric<T> {
+  percentiles: Percentiles<T>;
+}
+
+/**
+ * Distribution of values of an aggregated metric represented by a set of discreet percentiles.
+ * @example
+ * {
+ *   '1.0': 27,
+ *   '5.0': 150,
+ *   '25.0': 240,
+ *   '50.0': 420,
+ *   '75.0': 700,
+ *   '95.0': 2500,
+ *   '99.0': 7800,
+ * }
+ */
+export type Percentiles<T> = Record<string, T>;
+
+/**
+ * Most frequent messages logged by rule(s) to Event Log.
+ */
+export type TopMessages = Array<{
+  /**
+   * Number of occurencies of a message.
+   */
+  count: number;
+
+  /**
+   * The message itself.
+   */
+  message: string;
+}>;

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.mock.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getRulesSchemaMock } from '../../../rule_schema/mocks';
+import { healthStatsMock } from './health_stats.mock';
+import type { RuleHealthSnapshot } from './rule_health';
+
+const getEmptyRuleHealthSnapshot = (): RuleHealthSnapshot => {
+  return {
+    stats_at_the_moment: {
+      rule: getRulesSchemaMock(),
+    },
+    stats_over_interval: healthStatsMock.getEmptyRuleExecutionStats(),
+    history_over_interval: {
+      buckets: [
+        {
+          timestamp: '2023-05-15T16:12:14.967Z',
+          stats: healthStatsMock.getEmptyRuleExecutionStats(),
+        },
+      ],
+    },
+  };
+};
+
+export const ruleHealthSnapshotMock = {
+  getEmptyRuleHealthSnapshot,
+};

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RuleResponse } from '../../../rule_schema/model/rule_schemas';
+import type { HealthParameters, HealthSnapshot } from './health_metadata';
+import type { RuleExecutionStats, StatsHistory } from './health_stats';
+
+/**
+ * Health calculation parameters for a given rule.
+ */
+export interface RuleHealthParameters extends HealthParameters {
+  /**
+   * Saved object ID of the rule.
+   */
+  rule_id: string;
+}
+
+/**
+ * Health calculation result for a given rule.
+ */
+export interface RuleHealthSnapshot extends HealthSnapshot {
+  /**
+   * Health stats at the moment of the calculation request.
+   */
+  stats_at_the_moment: RuleHealthStatsAtTheMoment;
+
+  /**
+   * Health stats calculated over the interval specified in the health parameters.
+   */
+  stats_over_interval: RuleHealthStatsOverInterval;
+
+  /**
+   * History of change of the same health stats during the interval.
+   */
+  history_over_interval: StatsHistory<RuleHealthStatsOverInterval>;
+}
+
+/**
+ * Health stats at the moment of the calculation request.
+ */
+export interface RuleHealthStatsAtTheMoment {
+  /**
+   * Rule object including its current execution summary.
+   */
+  rule: RuleResponse;
+}
+
+/**
+ * Health stats calculated over a given interval.
+ */
+export type RuleHealthStatsOverInterval = RuleExecutionStats;

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.mock.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { healthStatsMock } from './health_stats.mock';
+import type { SpaceHealthSnapshot } from './space_health';
+
+const getEmptySpaceHealthSnapshot = (): SpaceHealthSnapshot => {
+  return {
+    stats_at_the_moment: healthStatsMock.getEmptyRuleStats(),
+    stats_over_interval: healthStatsMock.getEmptyRuleExecutionStats(),
+    history_over_interval: {
+      buckets: [
+        {
+          timestamp: '2023-05-15T16:12:14.967Z',
+          stats: healthStatsMock.getEmptyRuleExecutionStats(),
+        },
+      ],
+    },
+  };
+};
+
+export const spaceHealthSnapshotMock = {
+  getEmptySpaceHealthSnapshot,
+};

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { HealthParameters, HealthSnapshot } from './health_metadata';
+import type { RuleExecutionStats, RuleStats, StatsHistory } from './health_stats';
+
+/**
+ * Health calculation parameters for the current Kibana space.
+ */
+export type SpaceHealthParameters = HealthParameters;
+
+/**
+ * Health calculation result for the current Kibana space.
+ */
+export interface SpaceHealthSnapshot extends HealthSnapshot {
+  /**
+   * Health stats at the moment of the calculation request.
+   */
+  stats_at_the_moment: SpaceHealthStatsAtTheMoment;
+
+  /**
+   * Health stats calculated over the interval specified in the health parameters.
+   */
+  stats_over_interval: SpaceHealthStatsOverInterval;
+
+  /**
+   * History of change of the same health stats during the interval.
+   */
+  history_over_interval: StatsHistory<SpaceHealthStatsOverInterval>;
+}
+
+/**
+ * Health stats at the moment of the calculation request.
+ */
+export type SpaceHealthStatsAtTheMoment = RuleStats;
+
+/**
+ * Health stats calculated over a given interval.
+ */
+export type SpaceHealthStatsOverInterval = RuleExecutionStats;

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts

@@ -23,7 +23,7 @@ import { ruleRegistryMocks } from '@kbn/rule-registry-plugin/server/mocks';
 
 import { siemMock } from '../../../../mocks';
 import { createMockConfig } from '../../../../config.mock';
-import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
+import { detectionEngineHealthClientMock, ruleExecutionLogMock } from '../../rule_monitoring/mocks';
 import { requestMock } from './request';
 import { internalFrameworkRequest } from '../../../framework';
 
@@ -58,6 +58,8 @@ export const createMockClients = () => {
 
     config: createMockConfig(),
     appClient: siemMock.createClient(),
+
+    detectionEngineHealthClient: detectionEngineHealthClientMock.create(),
     ruleExecutionLog: ruleExecutionLogMock.forRoutes.create(),
   };
 };
@@ -129,6 +131,7 @@ const createSecuritySolutionRequestContextMock = (
     }),
     getSpaceId: jest.fn(() => 'default'),
     getRuleDataService: jest.fn(() => clients.ruleDataService),
+    getDetectionEngineHealthClient: jest.fn(() => clients.detectionEngineHealthClient),
     getRuleExecutionLog: jest.fn(() => clients.ruleExecutionLog),
     getExceptionListClient: jest.fn(() => clients.lists.exceptionListClient),
     getInternalFleetServices: jest.fn(() => {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/read_rule/route.ts

@@ -44,6 +44,7 @@ export const readRuleRoute = (router: SecuritySolutionPluginRouter, logger: Logg
       try {
         const rulesClient = (await context.alerting).getRulesClient();
 
+        // TODO: https://github.com/elastic/kibana/issues/125642 Reuse fetchRuleById
         const rule = await readRules({
           id,
           rulesClient,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_request.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import type {
+  GetClusterHealthRequest,
+  GetClusterHealthRequestBody,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { validateHealthInterval } from '../health_interval';
+
+export const validateGetClusterHealthRequest = (
+  body: GetClusterHealthRequestBody
+): GetClusterHealthRequest => {
+  const now = moment();
+  const interval = validateHealthInterval(body.interval, now);
+
+  return {
+    interval,
+    debug: body.debug ?? false,
+    requestReceivedAt: now.utc().toISOString(),
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_route.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
+
+import type { GetClusterHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring';
+import {
+  GET_CLUSTER_HEALTH_URL,
+  GetClusterHealthRequestBody,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { calculateHealthTimings } from '../health_timings';
+import { validateGetClusterHealthRequest } from './get_cluster_health_request';
+
+/**
+ * Get health overview of the whole cluster. Scope: all detection rules in all Kibana spaces.
+ * Returns:
+ * - health stats at the moment of the API call
+ * - health stats over a specified period of time ("health interval")
+ * - health stats history within the same interval in the form of a histogram
+ *   (the same stats are calculated over each of the discreet sub-intervals of the whole interval)
+ */
+export const getClusterHealthRoute = (router: SecuritySolutionPluginRouter) => {
+  router.post(
+    {
+      path: GET_CLUSTER_HEALTH_URL,
+      validate: {
+        body: buildRouteValidation(GetClusterHealthRequestBody),
+      },
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const params = validateGetClusterHealthRequest(request.body);
+
+        const ctx = await context.resolve(['securitySolution']);
+        const healthClient = ctx.securitySolution.getDetectionEngineHealthClient();
+
+        const clusterHealthParameters = { interval: params.interval };
+        const clusterHealth = await healthClient.calculateClusterHealth(clusterHealthParameters);
+
+        const responseBody: GetClusterHealthResponse = {
+          // TODO: https://github.com/elastic/kibana/issues/125642 Implement the endpoint and remove the `message` property
+          message: 'Not implemented',
+          timings: calculateHealthTimings(params.requestReceivedAt),
+          parameters: clusterHealthParameters,
+          health: {
+            ...clusterHealth,
+            debug: params.debug ? clusterHealth.debug : undefined,
+          },
+        };
+
+        return response.ok({ body: responseBody });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_request.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import type {
+  GetRuleHealthRequest,
+  GetRuleHealthRequestBody,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { validateHealthInterval } from '../health_interval';
+
+export const validateGetRuleHealthRequest = (
+  body: GetRuleHealthRequestBody
+): GetRuleHealthRequest => {
+  const now = moment();
+  const interval = validateHealthInterval(body.interval, now);
+
+  return {
+    ruleId: body.rule_id,
+    interval,
+    debug: body.debug ?? false,
+    requestReceivedAt: now.utc().toISOString(),
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_route.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+
+import type { GetRuleHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring';
+import {
+  GetRuleHealthRequestBody,
+  GET_RULE_HEALTH_URL,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
+import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation';
+import { buildSiemResponse } from '../../../../routes/utils';
+import { calculateHealthTimings } from '../health_timings';
+import { validateGetRuleHealthRequest } from './get_rule_health_request';
+
+/**
+ * Get health overview of a rule. Scope: a given detection rule in the current Kibana space.
+ * Returns:
+ * - health stats at the moment of the API call (rule and its execution summary)
+ * - health stats over a specified period of time ("health interval")
+ * - health stats history within the same interval in the form of a histogram
+ *   (the same stats are calculated over each of the discreet sub-intervals of the whole interval)
+ */
+export const getRuleHealthRoute = (router: SecuritySolutionPluginRouter) => {
+  router.post(
+    {
+      path: GET_RULE_HEALTH_URL,
+      validate: {
+        body: buildRouteValidation(GetRuleHealthRequestBody),
+      },
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const params = validateGetRuleHealthRequest(request.body);
+
+        const ctx = await context.resolve(['securitySolution']);
+        const healthClient = ctx.securitySolution.getDetectionEngineHealthClient();
+
+        const ruleHealthParameters = { interval: params.interval, rule_id: params.ruleId };
+        const ruleHealth = await healthClient.calculateRuleHealth(ruleHealthParameters);
+
+        const responseBody: GetRuleHealthResponse = {
+          timings: calculateHealthTimings(params.requestReceivedAt),
+          parameters: ruleHealthParameters,
+          health: {
+            ...ruleHealth,
+            debug: params.debug ? ruleHealth.debug : undefined,
+          },
+        };
+
+        return response.ok({ body: responseBody });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_request.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import type {
+  GetSpaceHealthRequest,
+  GetSpaceHealthRequestBody,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { validateHealthInterval } from '../health_interval';
+
+export const validateGetSpaceHealthRequest = (
+  body: GetSpaceHealthRequestBody
+): GetSpaceHealthRequest => {
+  const now = moment();
+  const interval = validateHealthInterval(body.interval, now);
+
+  return {
+    interval,
+    debug: body.debug ?? false,
+    requestReceivedAt: now.utc().toISOString(),
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_route.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
+
+import type { GetSpaceHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring';
+import {
+  GET_SPACE_HEALTH_URL,
+  GetSpaceHealthRequestBody,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { calculateHealthTimings } from '../health_timings';
+import { validateGetSpaceHealthRequest } from './get_space_health_request';
+
+/**
+ * Get health overview of the current Kibana space. Scope: all detection rules in the space.
+ * Returns:
+ * - health stats at the moment of the API call
+ * - health stats over a specified period of time ("health interval")
+ * - health stats history within the same interval in the form of a histogram
+ *   (the same stats are calculated over each of the discreet sub-intervals of the whole interval)
+ */
+export const getSpaceHealthRoute = (router: SecuritySolutionPluginRouter) => {
+  router.post(
+    {
+      path: GET_SPACE_HEALTH_URL,
+      validate: {
+        body: buildRouteValidation(GetSpaceHealthRequestBody),
+      },
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const params = validateGetSpaceHealthRequest(request.body);
+
+        const ctx = await context.resolve(['securitySolution']);
+        const healthClient = ctx.securitySolution.getDetectionEngineHealthClient();
+
+        const spaceHealthParameters = { interval: params.interval };
+        const spaceHealth = await healthClient.calculateSpaceHealth(spaceHealthParameters);
+
+        const responseBody: GetSpaceHealthResponse = {
+          timings: calculateHealthTimings(params.requestReceivedAt),
+          parameters: spaceHealthParameters,
+          health: {
+            ...spaceHealth,
+            debug: params.debug ? spaceHealth.debug : undefined,
+          },
+        };
+
+        return response.ok({ body: responseBody });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_interval.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+import type {
+  HealthInterval,
+  HealthIntervalParameters,
+} from '../../../../../../common/detection_engine/rule_monitoring';
+import {
+  HealthIntervalGranularity,
+  HealthIntervalType,
+} from '../../../../../../common/detection_engine/rule_monitoring';
+import { assertUnreachable } from '../../../../../../common/utility_types';
+
+const DEFAULT_INTERVAL_PARAMETERS: HealthIntervalParameters = {
+  type: HealthIntervalType.last_day,
+  granularity: HealthIntervalGranularity.hour,
+};
+
+export const validateHealthInterval = (
+  params: HealthIntervalParameters | undefined,
+  now: moment.Moment
+): HealthInterval => {
+  const parameters = params ?? DEFAULT_INTERVAL_PARAMETERS;
+
+  const from = getFrom(parameters, now);
+  const to = getTo(parameters, now);
+  const duration = moment.duration(to.diff(from));
+
+  // TODO: https://github.com/elastic/kibana/issues/125642 Validate that:
+  //   - to > from
+  //   - granularity is not too big, e.g. < duration (could be invalid when custom_range)
+  //   - granularity is not too small (could be invalid when custom_range)
+
+  return {
+    type: parameters.type,
+    granularity: parameters.granularity,
+    from: from.utc().toISOString(),
+    to: to.utc().toISOString(),
+    duration: duration.toISOString(),
+  };
+};
+
+const getFrom = (params: HealthIntervalParameters, now: moment.Moment): moment.Moment => {
+  const { type } = params;
+
+  // NOTE: it's important to clone `now` with `moment(now)` because moment objects are mutable.
+  // If you call .subtract() or other methods on the original `now`, you will change it which
+  // might cause bugs depending on how you use it in your calculations later.
+
+  if (type === HealthIntervalType.custom_range) {
+    return moment(params.from);
+  }
+  if (type === HealthIntervalType.last_hour) {
+    return moment(now).subtract(1, 'hour');
+  }
+  if (type === HealthIntervalType.last_day) {
+    return moment(now).subtract(1, 'day');
+  }
+  if (type === HealthIntervalType.last_week) {
+    return moment(now).subtract(1, 'week');
+  }
+  if (type === HealthIntervalType.last_month) {
+    return moment(now).subtract(1, 'month');
+  }
+  if (type === HealthIntervalType.last_year) {
+    return moment(now).subtract(1, 'year');
+  }
+
+  return assertUnreachable(type, 'Unhandled health interval type');
+};
+
+const getTo = (params: HealthIntervalParameters, now: moment.Moment): moment.Moment => {
+  const { type } = params;
+
+  if (type === HealthIntervalType.custom_range) {
+    return moment(params.to);
+  }
+
+  // NOTE: it's important to clone `now` with `moment(now)` because moment objects are mutable. If you
+  // return the original now from this method and then call .subtract() or other methods on it, it will
+  // change the original now which might cause bugs depending on how you use it in your calculations later.
+
+  return moment(now);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_timings.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import type { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import type { HealthTimings } from '../../../../../../common/detection_engine/rule_monitoring';
+
+export const calculateHealthTimings = (requestReceivedAt: IsoDateString): HealthTimings => {
+  const requestedAt = moment(requestReceivedAt);
+  const processedAt = moment().utc();
+  const processingTime = moment.duration(processedAt.diff(requestReceivedAt));
+
+  return {
+    requested_at: requestedAt.toISOString(),
+    processed_at: processedAt.toISOString(),
+    processing_time_ms: processingTime.asMilliseconds(),
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts

@@ -6,10 +6,19 @@
  */
 
 import type { SecuritySolutionPluginRouter } from '../../../../types';
-import { getRuleExecutionEventsRoute } from './get_rule_execution_events/route';
-import { getRuleExecutionResultsRoute } from './get_rule_execution_results/route';
+import { getClusterHealthRoute } from './detection_engine_health/get_cluster_health/get_cluster_health_route';
+import { getRuleHealthRoute } from './detection_engine_health/get_rule_health/get_rule_health_route';
+import { getSpaceHealthRoute } from './detection_engine_health/get_space_health/get_space_health_route';
+import { getRuleExecutionEventsRoute } from './rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route';
+import { getRuleExecutionResultsRoute } from './rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route';
 
 export const registerRuleMonitoringRoutes = (router: SecuritySolutionPluginRouter) => {
+  // Detection Engine health API
+  getClusterHealthRoute(router);
+  getSpaceHealthRoute(router);
+  getRuleHealthRoute(router);
+
+  // Rule execution logs API
   getRuleExecutionEventsRoute(router);
   getRuleExecutionResultsRoute(router);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.test.ts

@@ -5,16 +5,16 @@
  * 2.0.
  */
 
-import { serverMock, requestContextMock, requestMock } from '../../../routes/__mocks__';
+import { serverMock, requestContextMock, requestMock } from '../../../../routes/__mocks__';
 
 import {
   GET_RULE_EXECUTION_EVENTS_URL,
   LogLevel,
   RuleExecutionEventType,
-} from '../../../../../../common/detection_engine/rule_monitoring';
-import { getRuleExecutionEventsResponseMock } from '../../../../../../common/detection_engine/rule_monitoring/mocks';
-import type { GetExecutionEventsArgs } from '../../logic/rule_execution_log';
-import { getRuleExecutionEventsRoute } from './route';
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import { getRuleExecutionEventsResponseMock } from '../../../../../../../common/detection_engine/rule_monitoring/mocks';
+import type { GetExecutionEventsArgs } from '../../../logic/rule_execution_log';
+import { getRuleExecutionEventsRoute } from './get_rule_execution_events_route';
 
 describe('getRuleExecutionEventsRoute', () => {
   let server: ReturnType<typeof serverMock.create>;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts

@@ -6,16 +6,16 @@
  */
 
 import { transformError } from '@kbn/securitysolution-es-utils';
-import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation';
-import { buildSiemResponse } from '../../../routes/utils';
-import type { SecuritySolutionPluginRouter } from '../../../../../types';
+import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
 
-import type { GetRuleExecutionEventsResponse } from '../../../../../../common/detection_engine/rule_monitoring';
+import type { GetRuleExecutionEventsResponse } from '../../../../../../../common/detection_engine/rule_monitoring';
 import {
   GET_RULE_EXECUTION_EVENTS_URL,
   GetRuleExecutionEventsRequestParams,
   GetRuleExecutionEventsRequestQuery,
-} from '../../../../../../common/detection_engine/rule_monitoring';
+} from '../../../../../../../common/detection_engine/rule_monitoring';
 
 /**
  * Returns execution events of a given rule (e.g. status changes) from Event Log.

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.test.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { serverMock, requestContextMock, requestMock } from '../../../routes/__mocks__';
+import { serverMock, requestContextMock, requestMock } from '../../../../routes/__mocks__';
 
-import { GET_RULE_EXECUTION_RESULTS_URL } from '../../../../../../common/detection_engine/rule_monitoring';
-import { getRuleExecutionResultsResponseMock } from '../../../../../../common/detection_engine/rule_monitoring/mocks';
-import { getRuleExecutionResultsRoute } from './route';
+import { GET_RULE_EXECUTION_RESULTS_URL } from '../../../../../../../common/detection_engine/rule_monitoring';
+import { getRuleExecutionResultsResponseMock } from '../../../../../../../common/detection_engine/rule_monitoring/mocks';
+import { getRuleExecutionResultsRoute } from './get_rule_execution_results_route';
 
 describe('getRuleExecutionResultsRoute', () => {
   let server: ReturnType<typeof serverMock.create>;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.ts

@@ -6,16 +6,16 @@
  */
 
 import { transformError } from '@kbn/securitysolution-es-utils';
-import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation';
-import { buildSiemResponse } from '../../../routes/utils';
-import type { SecuritySolutionPluginRouter } from '../../../../../types';
+import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
 
-import type { GetRuleExecutionResultsResponse } from '../../../../../../common/detection_engine/rule_monitoring';
+import type { GetRuleExecutionResultsResponse } from '../../../../../../../common/detection_engine/rule_monitoring';
 import {
   GET_RULE_EXECUTION_RESULTS_URL,
   GetRuleExecutionResultsRequestParams,
   GetRuleExecutionResultsRequestQuery,
-} from '../../../../../../common/detection_engine/rule_monitoring';
+} from '../../../../../../../common/detection_engine/rule_monitoring';
 
 /**
  * Returns execution results of a given rule (aggregated by execution UUID) from Event Log.

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/index.ts

@@ -6,4 +6,9 @@
  */
 
 export * from './api/register_routes';
+export { RULE_EXECUTION_LOG_PROVIDER } from './logic/event_log/event_log_constants';
+export * from './logic/detection_engine_health';
 export * from './logic/rule_execution_log';
+export * from './logic/service_interface';
+export * from './logic/service';
+export { truncateList } from './logic/utils/normalization';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  clusterHealthSnapshotMock,
+  ruleHealthSnapshotMock,
+  spaceHealthSnapshotMock,
+} from '../../../../../../../common/detection_engine/rule_monitoring/mocks';
+
+import type { IDetectionEngineHealthClient } from '../detection_engine_health_client_interface';
+
+type CalculateRuleHealth = IDetectionEngineHealthClient['calculateRuleHealth'];
+type CalculateSpaceHealth = IDetectionEngineHealthClient['calculateSpaceHealth'];
+type CalculateClusterHealth = IDetectionEngineHealthClient['calculateClusterHealth'];
+
+export const detectionEngineHealthClientMock = {
+  create: (): jest.Mocked<IDetectionEngineHealthClient> => ({
+    calculateRuleHealth: jest
+      .fn<ReturnType<CalculateRuleHealth>, Parameters<CalculateRuleHealth>>()
+      .mockResolvedValue(ruleHealthSnapshotMock.getEmptyRuleHealthSnapshot()),
+
+    calculateSpaceHealth: jest
+      .fn<ReturnType<CalculateSpaceHealth>, Parameters<CalculateSpaceHealth>>()
+      .mockResolvedValue(spaceHealthSnapshotMock.getEmptySpaceHealthSnapshot()),
+
+    calculateClusterHealth: jest
+      .fn<ReturnType<CalculateClusterHealth>, Parameters<CalculateClusterHealth>>()
+      .mockResolvedValue(clusterHealthSnapshotMock.getEmptyClusterHealthSnapshot()),
+  }),
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { withSecuritySpan } from '../../../../../utils/with_security_span';
+import type { ExtMeta } from '../utils/console_logging';
+
+import type {
+  ClusterHealthParameters,
+  ClusterHealthSnapshot,
+  RuleHealthParameters,
+  RuleHealthSnapshot,
+  SpaceHealthParameters,
+  SpaceHealthSnapshot,
+} from '../../../../../../common/detection_engine/rule_monitoring';
+
+import type { IEventLogHealthClient } from './event_log/event_log_health_client';
+import type { IRuleObjectsHealthClient } from './rule_objects/rule_objects_health_client';
+import type { IDetectionEngineHealthClient } from './detection_engine_health_client_interface';
+
+export const createDetectionEngineHealthClient = (
+  ruleObjectsHealthClient: IRuleObjectsHealthClient,
+  eventLogHealthClient: IEventLogHealthClient,
+  logger: Logger,
+  currentSpaceId: string
+): IDetectionEngineHealthClient => {
+  return {
+    calculateRuleHealth: (args: RuleHealthParameters): Promise<RuleHealthSnapshot> => {
+      return withSecuritySpan('IDetectionEngineHealthClient.calculateRuleHealth', async () => {
+        const ruleId = args.rule_id;
+        try {
+          // We call these two sequentially, because if the rule doesn't exist we need to throw 404
+          // from ruleObjectsHealthClient before we calculate expensive stats in eventLogHealthClient.
+          const statsBasedOnRuleObjects = await ruleObjectsHealthClient.calculateRuleHealth(args);
+          const statsBasedOnEventLog = await eventLogHealthClient.calculateRuleHealth(args);
+
+          return {
+            stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment,
+            stats_over_interval: statsBasedOnEventLog.stats_over_interval,
+            history_over_interval: statsBasedOnEventLog.history_over_interval,
+            debug: {
+              ...statsBasedOnRuleObjects.debug,
+              ...statsBasedOnEventLog.debug,
+            },
+          };
+        } catch (e) {
+          const logMessage = 'Error calculating rule health';
+          const logReason = e instanceof Error ? e.message : String(e);
+          const logSuffix = `[rule id ${ruleId}]`;
+          const logMeta: ExtMeta = {
+            rule: { id: ruleId },
+          };
+
+          logger.error(`${logMessage}: ${logReason} ${logSuffix}`, logMeta);
+          throw e;
+        }
+      });
+    },
+
+    calculateSpaceHealth: (args: SpaceHealthParameters): Promise<SpaceHealthSnapshot> => {
+      return withSecuritySpan('IDetectionEngineHealthClient.calculateSpaceHealth', async () => {
+        try {
+          const [statsBasedOnRuleObjects, statsBasedOnEventLog] = await Promise.all([
+            ruleObjectsHealthClient.calculateSpaceHealth(args),
+            eventLogHealthClient.calculateSpaceHealth(args),
+          ]);
+
+          return {
+            stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment,
+            stats_over_interval: statsBasedOnEventLog.stats_over_interval,
+            history_over_interval: statsBasedOnEventLog.history_over_interval,
+            debug: {
+              ...statsBasedOnRuleObjects.debug,
+              ...statsBasedOnEventLog.debug,
+            },
+          };
+        } catch (e) {
+          const logMessage = 'Error calculating space health';
+          const logReason = e instanceof Error ? e.message : String(e);
+          const logSuffix = `[space id ${currentSpaceId}]`;
+          const logMeta: ExtMeta = {
+            kibana: { spaceId: currentSpaceId },
+          };
+
+          logger.error(`${logMessage}: ${logReason} ${logSuffix}`, logMeta);
+          throw e;
+        }
+      });
+    },
+
+    calculateClusterHealth: (args: ClusterHealthParameters): Promise<ClusterHealthSnapshot> => {
+      return withSecuritySpan('IDetectionEngineHealthClient.calculateClusterHealth', async () => {
+        try {
+          const [statsBasedOnRuleObjects, statsBasedOnEventLog] = await Promise.all([
+            ruleObjectsHealthClient.calculateClusterHealth(args),
+            eventLogHealthClient.calculateClusterHealth(args),
+          ]);
+
+          return {
+            stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment,
+            stats_over_interval: statsBasedOnEventLog.stats_over_interval,
+            history_over_interval: statsBasedOnEventLog.history_over_interval,
+            debug: {
+              ...statsBasedOnRuleObjects.debug,
+              ...statsBasedOnEventLog.debug,
+            },
+          };
+        } catch (e) {
+          const logMessage = 'Error calculating cluster health';
+          const logReason = e instanceof Error ? e.message : String(e);
+
+          logger.error(`${logMessage}: ${logReason}`);
+          throw e;
+        }
+      });
+    },
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client_interface.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  ClusterHealthParameters,
+  ClusterHealthSnapshot,
+  RuleHealthParameters,
+  RuleHealthSnapshot,
+  SpaceHealthParameters,
+  SpaceHealthSnapshot,
+} from '../../../../../../common/detection_engine/rule_monitoring';
+
+/**
+ * Calculates health of the Detection Engine overall and detection rules individually.
+ */
+export interface IDetectionEngineHealthClient {
+  /**
+   * Calculates health stats for a given rule.
+   */
+  calculateRuleHealth(args: RuleHealthParameters): Promise<RuleHealthSnapshot>;
+
+  /**
+   * Calculates health stats for all rules in the current Kibana space.
+   */
+  calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealthSnapshot>;
+
+  /**
+   * Calculates health stats for the whole cluster.
+   */
+  calculateClusterHealth(args: ClusterHealthParameters): Promise<ClusterHealthSnapshot>;
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/health_stats_for_rule.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { AggregateEventsBySavedObjectResult } from '@kbn/event-log-plugin/server';
+
+import type {
+  HealthIntervalGranularity,
+  RuleHealthSnapshot,
+  RuleHealthStatsOverInterval,
+  StatsHistory,
+} from '../../../../../../../../common/detection_engine/rule_monitoring';
+import type { RawData } from '../../../utils/normalization';
+
+import * as f from '../../../event_log/event_log_fields';
+import {
+  getRuleExecutionStatsAggregation,
+  normalizeRuleExecutionStatsAggregationResult,
+} from './rule_execution_stats';
+
+export const getRuleHealthAggregation = (
+  granularity: HealthIntervalGranularity
+): Record<string, estypes.AggregationsAggregationContainer> => {
+  // Let's say we want to calculate rule execution statistics over some date interval, where:
+  //   - the whole interval is one week (7 days)
+  //   - the interval's granularity is one day
+  // This means we will be calculating the same rule execution stats:
+  //   - One time over the whole week.
+  //   - Seven times over a day, per each day in the week.
+  return {
+    // And so this function creates several aggs that will be calculated for the whole interval.
+    ...getRuleExecutionStatsAggregation('whole-interval'),
+    // And this one creates a histogram, where for each bucket we will calculate the same aggs.
+    // The histogram's "calendar_interval" is equal to the granularity parameter.
+    ...getRuleExecutionStatsHistoryAggregation(granularity),
+  };
+};
+
+const getRuleExecutionStatsHistoryAggregation = (
+  granularity: HealthIntervalGranularity
+): Record<string, estypes.AggregationsAggregationContainer> => {
+  return {
+    statsHistory: {
+      date_histogram: {
+        field: f.TIMESTAMP,
+        calendar_interval: granularity,
+      },
+      aggs: getRuleExecutionStatsAggregation('histogram'),
+    },
+  };
+};
+
+export const normalizeRuleHealthAggregationResult = (
+  result: AggregateEventsBySavedObjectResult,
+  requestAggs: Record<string, estypes.AggregationsAggregationContainer>
+): Omit<RuleHealthSnapshot, 'stats_at_the_moment'> => {
+  const aggregations = result.aggregations ?? {};
+  return {
+    stats_over_interval: normalizeRuleExecutionStatsAggregationResult(
+      aggregations,
+      'whole-interval'
+    ),
+    history_over_interval: normalizeHistoryOverInterval(aggregations),
+    debug: {
+      eventLog: {
+        request: { aggs: requestAggs },
+        response: { aggregations },
+      },
+    },
+  };
+};
+
+const normalizeHistoryOverInterval = (
+  aggregations: Record<string, RawData>
+): StatsHistory<RuleHealthStatsOverInterval> => {
+  const statsHistory = aggregations.statsHistory || {};
+
+  return {
+    buckets: statsHistory.buckets.map((rawBucket: RawData) => {
+      const timestamp: string = String(rawBucket.key_as_string);
+      const stats = normalizeRuleExecutionStatsAggregationResult(rawBucket, 'histogram');
+      return { timestamp, stats };
+    }),
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts

@@ -0,0 +1,290 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mapValues } from 'lodash';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import type {
+  AggregatedMetric,
+  NumberOfDetectedGaps,
+  NumberOfExecutions,
+  NumberOfLoggedMessages,
+  RuleExecutionStats,
+  TopMessages,
+} from '../../../../../../../../common/detection_engine/rule_monitoring';
+import {
+  RuleExecutionEventType,
+  RuleExecutionStatus,
+  LogLevel,
+} from '../../../../../../../../common/detection_engine/rule_monitoring';
+
+import { DEFAULT_PERCENTILES } from '../../../utils/es_aggregations';
+import type { RawData } from '../../../utils/normalization';
+import * as f from '../../../event_log/event_log_fields';
+
+export type RuleExecutionStatsAggregationLevel = 'whole-interval' | 'histogram';
+
+export const getRuleExecutionStatsAggregation = (
+  aggregationContext: RuleExecutionStatsAggregationLevel
+): Record<string, estypes.AggregationsAggregationContainer> => {
+  return {
+    totalExecutions: {
+      cardinality: {
+        field: f.RULE_EXECUTION_UUID,
+      },
+    },
+    executeEvents: {
+      filter: {
+        term: { [f.EVENT_ACTION]: 'execute' },
+      },
+      aggs: {
+        executionDurationMs: {
+          percentiles: {
+            field: f.RULE_EXECUTION_TOTAL_DURATION_MS,
+            missing: 0,
+            percents: DEFAULT_PERCENTILES,
+          },
+        },
+        scheduleDelayNs: {
+          percentiles: {
+            field: f.RULE_EXECUTION_SCHEDULE_DELAY_NS,
+            missing: 0,
+            percents: DEFAULT_PERCENTILES,
+          },
+        },
+      },
+    },
+    statusChangeEvents: {
+      filter: {
+        bool: {
+          filter: [
+            {
+              term: {
+                [f.EVENT_ACTION]: RuleExecutionEventType['status-change'],
+              },
+            },
+          ],
+          must_not: [
+            {
+              terms: {
+                [f.RULE_EXECUTION_STATUS]: [
+                  RuleExecutionStatus.running,
+                  RuleExecutionStatus['going to run'],
+                ],
+              },
+            },
+          ],
+        },
+      },
+      aggs: {
+        executionsByStatus: {
+          terms: {
+            field: f.RULE_EXECUTION_STATUS,
+          },
+        },
+      },
+    },
+    executionMetricsEvents: {
+      filter: {
+        term: { [f.EVENT_ACTION]: RuleExecutionEventType['execution-metrics'] },
+      },
+      aggs: {
+        gaps: {
+          filter: {
+            exists: {
+              field: f.RULE_EXECUTION_GAP_DURATION_S,
+            },
+          },
+          aggs: {
+            totalGapDurationS: {
+              sum: {
+                field: f.RULE_EXECUTION_GAP_DURATION_S,
+              },
+            },
+          },
+        },
+        searchDurationMs: {
+          percentiles: {
+            field: f.RULE_EXECUTION_SEARCH_DURATION_MS,
+            missing: 0,
+            percents: DEFAULT_PERCENTILES,
+          },
+        },
+        indexingDurationMs: {
+          percentiles: {
+            field: f.RULE_EXECUTION_INDEXING_DURATION_MS,
+            missing: 0,
+            percents: DEFAULT_PERCENTILES,
+          },
+        },
+      },
+    },
+    messageContainingEvents: {
+      filter: {
+        terms: {
+          [f.EVENT_ACTION]: [
+            RuleExecutionEventType['status-change'],
+            RuleExecutionEventType.message,
+          ],
+        },
+      },
+      aggs: {
+        messagesByLogLevel: {
+          terms: {
+            field: f.LOG_LEVEL,
+          },
+        },
+        ...(aggregationContext === 'whole-interval'
+          ? {
+              errors: {
+                filter: {
+                  term: { [f.LOG_LEVEL]: LogLevel.error },
+                },
+                aggs: {
+                  topErrors: {
+                    categorize_text: {
+                      field: 'message',
+                      size: 5,
+                      similarity_threshold: 99,
+                    },
+                  },
+                },
+              },
+              warnings: {
+                filter: {
+                  term: { [f.LOG_LEVEL]: LogLevel.warn },
+                },
+                aggs: {
+                  topWarnings: {
+                    categorize_text: {
+                      field: 'message',
+                      size: 5,
+                      similarity_threshold: 99,
+                    },
+                  },
+                },
+              },
+            }
+          : {}),
+      },
+    },
+  };
+};
+
+export const normalizeRuleExecutionStatsAggregationResult = (
+  aggregations: Record<string, RawData>,
+  aggregationLevel: RuleExecutionStatsAggregationLevel
+): RuleExecutionStats => {
+  const totalExecutions = aggregations.totalExecutions || {};
+  const executeEvents = aggregations.executeEvents || {};
+  const statusChangeEvents = aggregations.statusChangeEvents || {};
+  const executionMetricsEvents = aggregations.executionMetricsEvents || {};
+  const messageContainingEvents = aggregations.messageContainingEvents || {};
+
+  const executionDurationMs = executeEvents.executionDurationMs || {};
+  const scheduleDelayNs = executeEvents.scheduleDelayNs || {};
+  const executionsByStatus = statusChangeEvents.executionsByStatus || {};
+  const gaps = executionMetricsEvents.gaps || {};
+  const searchDurationMs = executionMetricsEvents.searchDurationMs || {};
+  const indexingDurationMs = executionMetricsEvents.indexingDurationMs || {};
+
+  return {
+    number_of_executions: normalizeNumberOfExecutions(totalExecutions, executionsByStatus),
+    number_of_logged_messages: normalizeNumberOfLoggedMessages(messageContainingEvents),
+    number_of_detected_gaps: normalizeNumberOfDetectedGaps(gaps),
+    schedule_delay_ms: normalizeAggregatedMetric(scheduleDelayNs, (val) => val / 1_000_000),
+    execution_duration_ms: normalizeAggregatedMetric(executionDurationMs),
+    search_duration_ms: normalizeAggregatedMetric(searchDurationMs),
+    indexing_duration_ms: normalizeAggregatedMetric(indexingDurationMs),
+    top_errors:
+      aggregationLevel === 'whole-interval'
+        ? normalizeTopErrors(messageContainingEvents)
+        : undefined,
+    top_warnings:
+      aggregationLevel === 'whole-interval'
+        ? normalizeTopWarnings(messageContainingEvents)
+        : undefined,
+  };
+};
+
+const normalizeNumberOfExecutions = (
+  totalExecutions: RawData,
+  executionsByStatus: RawData
+): NumberOfExecutions => {
+  const getStatusCount = (status: RuleExecutionStatus): number => {
+    const bucket = executionsByStatus.buckets.find((b: RawData) => b.key === status);
+    return Number(bucket?.doc_count || 0);
+  };
+
+  return {
+    total: Number(totalExecutions.value || 0),
+    by_outcome: {
+      succeeded: getStatusCount(RuleExecutionStatus.succeeded),
+      warning: getStatusCount(RuleExecutionStatus['partial failure']),
+      failed: getStatusCount(RuleExecutionStatus.failed),
+    },
+  };
+};
+
+const normalizeNumberOfLoggedMessages = (
+  messageContainingEvents: RawData
+): NumberOfLoggedMessages => {
+  const messagesByLogLevel = messageContainingEvents.messagesByLogLevel || {};
+
+  const getMessageCount = (level: LogLevel): number => {
+    const bucket = messagesByLogLevel.buckets.find((b: RawData) => b.key === level);
+    return Number(bucket?.doc_count || 0);
+  };
+
+  return {
+    total: Number(messageContainingEvents.doc_count || 0),
+    by_level: {
+      error: getMessageCount(LogLevel.error),
+      warn: getMessageCount(LogLevel.warn),
+      info: getMessageCount(LogLevel.info),
+      debug: getMessageCount(LogLevel.debug),
+      trace: getMessageCount(LogLevel.trace),
+    },
+  };
+};
+
+const normalizeNumberOfDetectedGaps = (gaps: RawData): NumberOfDetectedGaps => {
+  return {
+    total: Number(gaps.doc_count || 0),
+    total_duration_s: Number(gaps.totalGapDurationS?.value || 0),
+  };
+};
+
+const normalizeAggregatedMetric = (
+  percentilesAggregate: RawData,
+  modifier: (value: number) => number = (v) => v
+): AggregatedMetric<number> => {
+  const rawPercentiles = percentilesAggregate.values || {};
+  return {
+    percentiles: mapValues(rawPercentiles, (rawValue) => modifier(Number(rawValue || 0))),
+  };
+};
+
+const normalizeTopErrors = (messageContainingEvents: RawData): TopMessages => {
+  const topErrors = messageContainingEvents.errors?.topErrors || {};
+  return normalizeTopMessages(topErrors);
+};
+
+const normalizeTopWarnings = (messageContainingEvents: RawData): TopMessages => {
+  const topWarnings = messageContainingEvents.warnings?.topWarnings || {};
+  return normalizeTopMessages(topWarnings);
+};
+
+const normalizeTopMessages = (categorizeTextAggregate: RawData): TopMessages => {
+  const buckets = (categorizeTextAggregate || {}).buckets || [];
+  return buckets.map((b: RawData) => {
+    return {
+      count: Number(b?.doc_count || 0),
+      message: String(b?.key || ''),
+    };
+  });
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { KueryNode } from '@kbn/es-query';
+import type { IEventLogClient } from '@kbn/event-log-plugin/server';
+import type {
+  ClusterHealthParameters,
+  ClusterHealthSnapshot,
+  RuleHealthParameters,
+  RuleHealthSnapshot,
+  SpaceHealthParameters,
+  SpaceHealthSnapshot,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+
+import * as f from '../../event_log/event_log_fields';
+import {
+  ALERTING_PROVIDER,
+  RULE_EXECUTION_LOG_PROVIDER,
+  RULE_SAVED_OBJECT_TYPE,
+} from '../../event_log/event_log_constants';
+import { kqlOr } from '../../utils/kql';
+import {
+  getRuleHealthAggregation,
+  normalizeRuleHealthAggregationResult,
+} from './aggregations/health_stats_for_rule';
+
+/**
+ * Client for calculating health stats based on events in .kibana-event-log-* index.
+ */
+export interface IEventLogHealthClient {
+  calculateRuleHealth(args: RuleHealthParameters): Promise<RuleHealth>;
+  calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealth>;
+  calculateClusterHealth(args: ClusterHealthParameters): Promise<ClusterHealth>;
+}
+
+type RuleHealth = Omit<RuleHealthSnapshot, 'stats_at_the_moment'>;
+type SpaceHealth = Omit<SpaceHealthSnapshot, 'stats_at_the_moment'>;
+type ClusterHealth = Omit<ClusterHealthSnapshot, 'stats_at_the_moment'>;
+
+export const createEventLogHealthClient = (eventLog: IEventLogClient): IEventLogHealthClient => {
+  return {
+    async calculateRuleHealth(args: RuleHealthParameters): Promise<RuleHealth> {
+      const { rule_id: ruleId, interval } = args;
+      const soType = RULE_SAVED_OBJECT_TYPE;
+      const soIds = [ruleId];
+      const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER];
+
+      const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`;
+      const aggs = getRuleHealthAggregation(interval.granularity);
+
+      const result = await eventLog.aggregateEventsBySavedObjectIds(soType, soIds, {
+        start: interval.from,
+        end: interval.to,
+        filter: kqlFilter,
+        aggs,
+      });
+
+      return normalizeRuleHealthAggregationResult(result, aggs);
+    },
+
+    async calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealth> {
+      const { interval } = args;
+      const soType = RULE_SAVED_OBJECT_TYPE;
+      const authFilter = {} as KueryNode;
+      const namespaces = undefined; // means current Kibana space
+      const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER];
+
+      const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`;
+      const aggs = getRuleHealthAggregation(interval.granularity);
+
+      // TODO: https://github.com/elastic/kibana/issues/125642 Check with ResponseOps that this is correct usage of this method
+      const result = await eventLog.aggregateEventsWithAuthFilter(
+        soType,
+        authFilter,
+        {
+          start: interval.from,
+          end: interval.to,
+          filter: kqlFilter,
+          aggs,
+        },
+        namespaces
+      );
+
+      return normalizeRuleHealthAggregationResult(result, aggs);
+    },
+
+    async calculateClusterHealth(args: ClusterHealthParameters): Promise<ClusterHealth> {
+      // TODO: https://github.com/elastic/kibana/issues/125642 Implement
+      return {
+        stats_over_interval: {
+          message: 'Not implemented',
+        },
+        history_over_interval: {
+          buckets: [],
+        },
+        debug: {
+          eventLog: {
+            request: {},
+            response: {},
+          },
+        },
+      };
+    },
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './detection_engine_health_client_interface';
+export * from './detection_engine_health_client';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/health_stats_for_space.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { SpaceHealthStatsAtTheMoment } from '../../../../../../../../common/detection_engine/rule_monitoring';
+import { getRuleStatsAggregation, normalizeRuleStatsAggregation } from './rule_stats';
+
+export const getSpaceHealthAggregation = (): Record<
+  string,
+  estypes.AggregationsAggregationContainer
+> => {
+  return getRuleStatsAggregation();
+};
+
+export const normalizeSpaceHealthAggregationResult = (
+  aggregations: Record<string, unknown>
+): SpaceHealthStatsAtTheMoment => {
+  return normalizeRuleStatsAggregation(aggregations);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/rule_stats.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  RuleStats,
+  TotalEnabledDisabled,
+} from '../../../../../../../../common/detection_engine/rule_monitoring';
+import type { RawData } from '../../../utils/normalization';
+
+export const getRuleStatsAggregation = (): Record<
+  string,
+  estypes.AggregationsAggregationContainer
+> => {
+  const rulesByEnabled: estypes.AggregationsAggregationContainer = {
+    terms: {
+      field: 'alert.attributes.enabled',
+    },
+  };
+
+  return {
+    rulesByEnabled,
+    rulesByOrigin: {
+      terms: {
+        field: 'alert.attributes.params.immutable',
+      },
+      aggs: {
+        rulesByEnabled,
+      },
+    },
+    rulesByType: {
+      terms: {
+        field: 'alert.attributes.alertTypeId',
+      },
+      aggs: {
+        rulesByEnabled,
+      },
+    },
+    rulesByOutcome: {
+      terms: {
+        field: 'alert.attributes.lastRun.outcome',
+      },
+      aggs: {
+        rulesByEnabled,
+      },
+    },
+  };
+};
+
+export const normalizeRuleStatsAggregation = (aggregations: Record<string, RawData>): RuleStats => {
+  const rulesByEnabled = aggregations.rulesByEnabled || {};
+  const rulesByOrigin = aggregations.rulesByOrigin || {};
+  const rulesByType = aggregations.rulesByType || {};
+  const rulesByOutcome = aggregations.rulesByOutcome || {};
+
+  return {
+    number_of_rules: {
+      all: normalizeByEnabled(rulesByEnabled),
+      by_origin: normalizeByOrigin(rulesByOrigin),
+      by_type: normalizeByAnyKeyword(rulesByType),
+      by_outcome: normalizeByAnyKeyword(rulesByOutcome),
+    },
+  };
+};
+
+const normalizeByEnabled = (rulesByEnabled: RawData): TotalEnabledDisabled => {
+  const getEnabled = (value: 'true' | 'false'): number => {
+    const bucket = rulesByEnabled?.buckets?.find((b: RawData) => b.key_as_string === value);
+    return Number(bucket?.doc_count || 0);
+  };
+
+  const enabled = getEnabled('true');
+  const disabled = getEnabled('false');
+
+  return {
+    total: enabled + disabled,
+    enabled,
+    disabled,
+  };
+};
+
+const normalizeByOrigin = (
+  rulesByOrigin: RawData
+): Record<'prebuilt' | 'custom', TotalEnabledDisabled> => {
+  const getOrigin = (value: 'true' | 'false'): TotalEnabledDisabled => {
+    const bucket = rulesByOrigin?.buckets?.find((b: RawData) => b.key === value);
+    return normalizeByEnabled(bucket?.rulesByEnabled);
+  };
+
+  const prebuilt = getOrigin('true');
+  const custom = getOrigin('false');
+
+  return { prebuilt, custom };
+};
+
+const normalizeByAnyKeyword = (
+  rulesByAnyKeyword: RawData
+): Record<string, TotalEnabledDisabled> => {
+  const kvPairs = rulesByAnyKeyword?.buckets?.map((b: RawData) => {
+    const bucketKey = b.key;
+    const rulesByEnabled = b?.rulesByEnabled || {};
+    return {
+      [bucketKey]: normalizeByEnabled(rulesByEnabled),
+    };
+  });
+
+  return Object.assign({}, ...kvPairs);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/fetch_rule_by_id.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import Boom from '@hapi/boom';
+import type { RulesClient } from '@kbn/alerting-plugin/server';
+import type {
+  RuleObjectId,
+  RuleResponse,
+} from '../../../../../../../common/detection_engine/rule_schema';
+import { readRules } from '../../../../rule_management/logic/crud/read_rules';
+import { transform } from '../../../../rule_management/utils/utils';
+
+// TODO: https://github.com/elastic/kibana/issues/125642 Move to rule_management into a RuleManagementClient
+
+export const fetchRuleById = async (
+  rulesClient: RulesClient,
+  id: RuleObjectId
+): Promise<RuleResponse> => {
+  const rawRule = await readRules({
+    id,
+    rulesClient,
+    ruleId: undefined,
+  });
+
+  if (rawRule == null) {
+    throw Boom.notFound(`Rule not found, id: "${id}" `);
+  }
+
+  const normalizedRule = transform(rawRule);
+
+  if (normalizedRule == null) {
+    throw Boom.internal('Internal error normalizing rule object');
+  }
+
+  return normalizedRule;
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RulesClientApi } from '@kbn/alerting-plugin/server/types';
+import type {
+  ClusterHealthParameters,
+  ClusterHealthSnapshot,
+  RuleHealthParameters,
+  RuleHealthSnapshot,
+  SpaceHealthParameters,
+  SpaceHealthSnapshot,
+} from '../../../../../../../common/detection_engine/rule_monitoring';
+import {
+  getSpaceHealthAggregation,
+  normalizeSpaceHealthAggregationResult,
+} from './aggregations/health_stats_for_space';
+import { fetchRuleById } from './fetch_rule_by_id';
+
+/**
+ * Client for calculating health stats based on rule saved objects.
+ */
+export interface IRuleObjectsHealthClient {
+  calculateRuleHealth(args: RuleHealthParameters): Promise<RuleHealth>;
+  calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealth>;
+  calculateClusterHealth(args: ClusterHealthParameters): Promise<ClusterHealth>;
+}
+
+type RuleHealth = Pick<RuleHealthSnapshot, 'stats_at_the_moment' | 'debug'>;
+type SpaceHealth = Pick<SpaceHealthSnapshot, 'stats_at_the_moment' | 'debug'>;
+type ClusterHealth = Pick<ClusterHealthSnapshot, 'stats_at_the_moment' | 'debug'>;
+
+export const createRuleObjectsHealthClient = (
+  rulesClient: RulesClientApi
+): IRuleObjectsHealthClient => {
+  return {
+    async calculateRuleHealth(args: RuleHealthParameters): Promise<RuleHealth> {
+      const rule = await fetchRuleById(rulesClient, args.rule_id);
+      return {
+        stats_at_the_moment: { rule },
+        debug: {},
+      };
+    },
+
+    async calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealth> {
+      const aggs = getSpaceHealthAggregation();
+      const aggregations = await rulesClient.aggregate({ aggs });
+
+      return {
+        stats_at_the_moment: normalizeSpaceHealthAggregationResult(aggregations),
+        debug: {
+          rulesClient: {
+            request: { aggs },
+            response: { aggregations },
+          },
+        },
+      };
+    },
+
+    async calculateClusterHealth(args: ClusterHealthParameters): Promise<ClusterHealth> {
+      // TODO: https://github.com/elastic/kibana/issues/125642 Implement
+      return {
+        stats_at_the_moment: {
+          number_of_rules: {
+            all: {
+              total: 0,
+              enabled: 0,
+              disabled: 0,
+            },
+            by_origin: {
+              prebuilt: {
+                total: 0,
+                enabled: 0,
+                disabled: 0,
+              },
+              custom: {
+                total: 0,
+                enabled: 0,
+                disabled: 0,
+              },
+            },
+            by_type: {},
+            by_outcome: {},
+          },
+        },
+        debug: {
+          rulesClient: {
+            request: {},
+            response: {},
+          },
+        },
+      };
+    },
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_constants.ts

@@ -8,3 +8,5 @@
 export const RULE_SAVED_OBJECT_TYPE = 'alert';
 
 export const RULE_EXECUTION_LOG_PROVIDER = 'securitySolution.ruleExecution';
+
+export const ALERTING_PROVIDER = 'alerting';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// -------------------------------------------------------------------------------------------------
+// ECS fields
+
+export const TIMESTAMP = `@timestamp` as const;
+
+export const EVENT_PROVIDER = 'event.provider' as const;
+export const EVENT_ACTION = 'event.action' as const;
+export const EVENT_SEQUENCE = 'event.sequence' as const;
+
+export const LOG_LEVEL = 'log.level' as const;
+
+// -------------------------------------------------------------------------------------------------
+// Custom fields of Alerting Framework and Security Solution
+
+const RULE_EXECUTION = 'kibana.alert.rule.execution' as const;
+const RULE_EXECUTION_METRICS = `${RULE_EXECUTION}.metrics` as const;
+
+export const RULE_EXECUTION_UUID = `${RULE_EXECUTION}.uuid` as const;
+
+export const RULE_EXECUTION_OUTCOME = 'kibana.alerting.outcome' as const;
+
+export const RULE_EXECUTION_STATUS = `${RULE_EXECUTION}.status` as const;
+
+export const RULE_EXECUTION_TOTAL_DURATION_MS =
+  `${RULE_EXECUTION_METRICS}.total_run_duration_ms` as const;
+
+export const RULE_EXECUTION_SEARCH_DURATION_MS =
+  `${RULE_EXECUTION_METRICS}.total_search_duration_ms` as const;
+
+export const RULE_EXECUTION_INDEXING_DURATION_MS =
+  `${RULE_EXECUTION_METRICS}.total_indexing_duration_ms` as const;
+
+export const RULE_EXECUTION_GAP_DURATION_S =
+  `${RULE_EXECUTION_METRICS}.execution_gap_duration_s` as const;
+
+export const RULE_EXECUTION_SCHEDULE_DELAY_NS = 'kibana.task.schedule_delay' as const;
+
+export const NUMBER_OF_ALERTS_GENERATED = `${RULE_EXECUTION_METRICS}.alert_counts.new` as const;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/register_event_log_provider.ts

@@ -6,8 +6,8 @@
  */
 
 import type { IEventLogService } from '@kbn/event-log-plugin/server';
-import { RuleExecutionEventType } from '../../../../../../../common/detection_engine/rule_monitoring';
-import { RULE_EXECUTION_LOG_PROVIDER } from './constants';
+import { RuleExecutionEventType } from '../../../../../../common/detection_engine/rule_monitoring';
+import { RULE_EXECUTION_LOG_PROVIDER } from './event_log_constants';
 
 export const registerEventLogProvider = (eventLogService: IEventLogService) => {
   eventLogService.registerProviderActions(

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/client.ts

@@ -27,8 +27,8 @@ import {
 
 import { assertUnreachable } from '../../../../../../../common/utility_types';
 import { withSecuritySpan } from '../../../../../../utils/with_security_span';
-import { truncateValue } from '../utils/normalization';
-import type { ExtMeta } from '../utils/console_logging';
+import { truncateValue } from '../../utils/normalization';
+import type { ExtMeta } from '../../utils/console_logging';
 import { getCorrelationIds } from './correlation_ids';
 
 import type { IEventLogWriter } from '../event_log/event_log_writer';
@@ -38,7 +38,7 @@ import type {
   StatusChangeArgs,
 } from './client_interface';
 
-export const createClientForExecutors = (
+export const createRuleExecutionLogClientForExecutors = (
   settings: RuleExecutionSettings,
   eventLog: IEventLogWriter,
   logger: Logger,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/correlation_ids.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ExtMeta } from '../utils/console_logging';
+import type { ExtMeta } from '../../utils/console_logging';
 import type { RuleExecutionStatus } from '../../../../../../../common/detection_engine/rule_monitoring';
 import type { RuleExecutionContext } from './client_interface';
 
@@ -75,7 +75,7 @@ const createBuilder = (state: BuilderState): ICorrelationIds => {
         },
       };
 
-      if (status != null && logMeta.rule.execution != null) {
+      if (status != null && logMeta.rule?.execution != null) {
         logMeta.rule.execution.status = status;
       }
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client.ts

@@ -6,22 +6,22 @@
  */
 
 import type { Logger } from '@kbn/core/server';
-import { withSecuritySpan } from '../../../../../../utils/with_security_span';
-import type { ExtMeta } from '../utils/console_logging';
 
 import type {
   GetRuleExecutionEventsResponse,
   GetRuleExecutionResultsResponse,
 } from '../../../../../../../common/detection_engine/rule_monitoring';
 
+import { withSecuritySpan } from '../../../../../../utils/with_security_span';
 import type { IEventLogReader } from '../event_log/event_log_reader';
+import type { ExtMeta } from '../../utils/console_logging';
 import type {
   GetExecutionEventsArgs,
   GetExecutionResultsArgs,
   IRuleExecutionLogForRoutes,
 } from './client_interface';
 
-export const createClientForRoutes = (
+export const createRuleExecutionLogClientForRoutes = (
   eventLog: IEventLogReader,
   logger: Logger
 ): IRuleExecutionLogForRoutes => {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client_interface.ts

@@ -5,7 +5,6 @@
  * 2.0.
  */
 
-import type { SortOrder } from '../../../../../../../common/detection_engine/schemas/common';
 import type {
   GetRuleExecutionEventsResponse,
   GetRuleExecutionResultsResponse,
@@ -14,6 +13,8 @@ import type {
   RuleExecutionStatus,
   SortFieldOfRuleExecutionResult,
 } from '../../../../../../../common/detection_engine/rule_monitoring';
+import type { RuleObjectId } from '../../../../../../../common/detection_engine/rule_schema';
+import type { SortOrder } from '../../../../../../../common/detection_engine/schemas/common';
 
 /**
  * Used from route handlers to fetch and manage various information about the rule execution:
@@ -36,7 +37,7 @@ export interface IRuleExecutionLogForRoutes {
 
 export interface GetExecutionEventsArgs {
   /** Saved object id of the rule (`rule.id`). */
-  ruleId: string;
+  ruleId: RuleObjectId;
 
   /** Include events of the specified types. If empty, all types of events will be included. */
   eventTypes: RuleExecutionEventType[];
@@ -56,7 +57,7 @@ export interface GetExecutionEventsArgs {
 
 export interface GetExecutionResultsArgs {
   /** Saved object id of the rule (`rule.id`). */
-  ruleId: string;
+  ruleId: RuleObjectId;
 
   /** Start of daterange to filter to. */
   start: string;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.test.ts

@@ -13,7 +13,7 @@
  */
 
 import { MAX_EXECUTION_EVENTS_DISPLAYED } from '@kbn/securitysolution-rules';
-import { RuleExecutionStatus } from '../../../../../../../../common/detection_engine/rule_monitoring';
+import { RuleExecutionStatus } from '../../../../../../../../../common/detection_engine/rule_monitoring';
 
 import {
   formatExecutionEventResponse,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.ts

@@ -14,14 +14,16 @@ import type { AggregateEventsBySavedObjectResult } from '@kbn/event-log-plugin/s
 import type {
   RuleExecutionResult,
   GetRuleExecutionResultsResponse,
-} from '../../../../../../../../common/detection_engine/rule_monitoring';
-import { RuleExecutionStatus } from '../../../../../../../../common/detection_engine/rule_monitoring';
+} from '../../../../../../../../../common/detection_engine/rule_monitoring';
+import { RuleExecutionStatus } from '../../../../../../../../../common/detection_engine/rule_monitoring';
 import type {
   ExecutionEventAggregationOptions,
   ExecutionUuidAggResult,
   ExecutionUuidAggBucket,
 } from './types';
-import { EXECUTION_UUID_FIELD } from './types';
+import * as f from '../../../../event_log/event_log_fields';
+
+// TODO: https://github.com/elastic/kibana/issues/125642 Move the fields from this file to `event_log_fields.ts`
 
 // Base ECS fields
 const ACTION_FIELD = 'event.action';
@@ -104,13 +106,13 @@ export const getExecutionEventAggregation = ({
     // Total unique executions for given root filters
     totalExecutions: {
       cardinality: {
-        field: EXECUTION_UUID_FIELD,
+        field: f.RULE_EXECUTION_UUID,
       },
     },
     executionUuid: {
       // Bucket by execution UUID
       terms: {
-        field: EXECUTION_UUID_FIELD,
+        field: f.RULE_EXECUTION_UUID,
         size: maxExecutions,
         order: formatSortForTermsSort(sort),
       },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/types.ts

@@ -7,9 +7,6 @@
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
-// Shared constants, consider moving to packages
-export const EXECUTION_UUID_FIELD = 'kibana.alert.rule.execution.uuid';
-
 type AlertCounts = estypes.AggregationsMultiBucketAggregateBase & {
   buckets: {
     activeAlerts: estypes.AggregationsSingleBucketAggregateBase;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_reader.ts

@@ -9,14 +9,10 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { IEventLogClient, IValidatedEvent } from '@kbn/event-log-plugin/server';
 import { MAX_EXECUTION_EVENTS_DISPLAYED } from '@kbn/securitysolution-rules';
 
-import { assertUnreachable } from '../../../../../../../common/utility_types';
-import { invariant } from '../../../../../../../common/utils/invariant';
-import { withSecuritySpan } from '../../../../../../utils/with_security_span';
-
 import type {
-  RuleExecutionEvent,
   GetRuleExecutionEventsResponse,
   GetRuleExecutionResultsResponse,
+  RuleExecutionEvent,
 } from '../../../../../../../common/detection_engine/rule_monitoring';
 import {
   LogLevel,
@@ -24,19 +20,28 @@ import {
   RuleExecutionEventType,
   ruleExecutionEventTypeFromString,
 } from '../../../../../../../common/detection_engine/rule_monitoring';
+
+import { assertUnreachable } from '../../../../../../../common/utility_types';
+import { invariant } from '../../../../../../../common/utils/invariant';
+import { withSecuritySpan } from '../../../../../../utils/with_security_span';
+import { kqlAnd, kqlOr } from '../../utils/kql';
+
 import type {
   GetExecutionEventsArgs,
   GetExecutionResultsArgs,
 } from '../client_for_routes/client_interface';
-
-import { RULE_SAVED_OBJECT_TYPE, RULE_EXECUTION_LOG_PROVIDER } from './constants';
 import {
   formatExecutionEventResponse,
   getExecutionEventAggregation,
   mapRuleExecutionStatusToPlatformStatus,
-} from './get_execution_event_aggregation';
-import type { ExecutionUuidAggResult } from './get_execution_event_aggregation/types';
-import { EXECUTION_UUID_FIELD } from './get_execution_event_aggregation/types';
+} from './aggregations/execution_results';
+import type { ExecutionUuidAggResult } from './aggregations/execution_results/types';
+
+import * as f from '../../event_log/event_log_fields';
+import {
+  RULE_EXECUTION_LOG_PROVIDER,
+  RULE_SAVED_OBJECT_TYPE,
+} from '../../event_log/event_log_constants';
 
 export interface IEventLogReader {
   getExecutionEvents(args: GetExecutionEventsArgs): Promise<GetRuleExecutionEventsResponse>;
@@ -54,17 +59,17 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader
 
       // TODO: include Framework events
       const kqlFilter = kqlAnd([
-        `event.provider:${RULE_EXECUTION_LOG_PROVIDER}`,
-        eventTypes.length > 0 ? `event.action:(${kqlOr(eventTypes)})` : '',
-        logLevels.length > 0 ? `log.level:(${kqlOr(logLevels)})` : '',
+        `${f.EVENT_PROVIDER}:${RULE_EXECUTION_LOG_PROVIDER}`,
+        eventTypes.length > 0 ? `${f.EVENT_ACTION}:(${kqlOr(eventTypes)})` : '',
+        logLevels.length > 0 ? `${f.LOG_LEVEL}:(${kqlOr(logLevels)})` : '',
       ]);
 
       const findResult = await withSecuritySpan('findEventsBySavedObjectIds', () => {
         return eventLog.findEventsBySavedObjectIds(soType, soIds, {
           filter: kqlFilter,
           sort: [
-            { sort_field: '@timestamp', sort_order: sortOrder },
-            { sort_field: 'event.sequence', sort_order: sortOrder },
+            { sort_field: f.TIMESTAMP, sort_order: sortOrder },
+            { sort_field: f.EVENT_SEQUENCE, sort_order: sortOrder },
           ],
           page,
           per_page: perPage,
@@ -102,25 +107,23 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader
           start,
           end,
           // Also query for `event.outcome` to catch executions that only contain platform events
-          filter: `kibana.alert.rule.execution.status:(${statusFilters.join(
-            ' OR '
-          )}) ${outcomeFilter}`,
+          filter: `${f.RULE_EXECUTION_STATUS}:(${statusFilters.join(' OR ')}) ${outcomeFilter}`,
           aggs: {
             totalExecutions: {
               cardinality: {
-                field: EXECUTION_UUID_FIELD,
+                field: f.RULE_EXECUTION_UUID,
               },
             },
             filteredExecutionUUIDs: {
               terms: {
-                field: EXECUTION_UUID_FIELD,
+                field: f.RULE_EXECUTION_UUID,
                 order: { executeStartTime: 'desc' },
                 size: MAX_EXECUTION_EVENTS_DISPLAYED,
               },
               aggs: {
                 executeStartTime: {
                   min: {
-                    field: '@timestamp',
+                    field: f.TIMESTAMP,
                   },
                 },
               },
@@ -144,7 +147,7 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader
 
       // Now query for aggregate events, and pass any ID's as filters as determined from the above status/queryText results
       const idsFilter = statusIds.length
-        ? `kibana.alert.rule.execution.uuid:(${statusIds.join(' OR ')})`
+        ? `${f.RULE_EXECUTION_UUID}:(${statusIds.join(' OR ')})`
         : '';
       const results = await eventLog.aggregateEventsBySavedObjectIds(soType, soIds, {
         start,
@@ -163,14 +166,6 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader
   };
 };
 
-const kqlAnd = <T>(items: T[]): string => {
-  return items.filter(Boolean).map(String).join(' and ');
-};
-
-const kqlOr = <T>(items: T[]): string => {
-  return items.filter(Boolean).map(String).join(' or ');
-};
-
 const normalizeEvent = (rawEvent: IValidatedEvent): RuleExecutionEvent => {
   invariant(rawEvent, 'Event not found');
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_writer.ts

@@ -19,7 +19,10 @@ import {
   RuleExecutionEventType,
   ruleExecutionStatusToNumber,
 } from '../../../../../../../common/detection_engine/rule_monitoring';
-import { RULE_SAVED_OBJECT_TYPE, RULE_EXECUTION_LOG_PROVIDER } from './constants';
+import {
+  RULE_SAVED_OBJECT_TYPE,
+  RULE_EXECUTION_LOG_PROVIDER,
+} from '../../event_log/event_log_constants';
 
 export interface IEventLogWriter {
   logMessage(args: MessageArgs): void;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/index.ts

@@ -7,9 +7,5 @@
 
 export * from './client_for_executors/client_interface';
 export * from './client_for_routes/client_interface';
-export * from './service_interface';
-export * from './service';
 
-export { RULE_EXECUTION_LOG_PROVIDER } from './event_log/constants';
 export { createRuleExecutionSummary } from './create_rule_execution_summary';
-export * from './utils/normalization';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service.ts

@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { Logger } from '@kbn/core/server';
-import { invariant } from '../../../../../../common/utils/invariant';
-import type { ConfigType } from '../../../../../config';
-import { withSecuritySpan } from '../../../../../utils/with_security_span';
-import type {
-  SecuritySolutionPluginCoreSetupDependencies,
-  SecuritySolutionPluginSetupDependencies,
-} from '../../../../../plugin_contract';
-
-import type { IRuleExecutionLogForRoutes } from './client_for_routes/client_interface';
-import { createClientForRoutes } from './client_for_routes/client';
-import type { IRuleExecutionLogForExecutors } from './client_for_executors/client_interface';
-import { createClientForExecutors } from './client_for_executors/client';
-
-import { registerEventLogProvider } from './event_log/register_event_log_provider';
-import { createEventLogReader } from './event_log/event_log_reader';
-import { createEventLogWriter } from './event_log/event_log_writer';
-import { fetchRuleExecutionSettings } from './execution_settings/fetch_rule_execution_settings';
-import type {
-  ClientForExecutorsParams,
-  ClientForRoutesParams,
-  IRuleExecutionLogService,
-} from './service_interface';
-
-export const createRuleExecutionLogService = (
-  config: ConfigType,
-  logger: Logger,
-  core: SecuritySolutionPluginCoreSetupDependencies,
-  plugins: SecuritySolutionPluginSetupDependencies
-): IRuleExecutionLogService => {
-  return {
-    registerEventLogProvider: () => {
-      registerEventLogProvider(plugins.eventLog);
-    },
-
-    createClientForRoutes: (params: ClientForRoutesParams): IRuleExecutionLogForRoutes => {
-      const { eventLogClient } = params;
-
-      const eventLogReader = createEventLogReader(eventLogClient);
-
-      return createClientForRoutes(eventLogReader, logger);
-    },
-
-    createClientForExecutors: (
-      params: ClientForExecutorsParams
-    ): Promise<IRuleExecutionLogForExecutors> => {
-      return withSecuritySpan('IRuleExecutionLogService.createClientForExecutors', async () => {
-        const { savedObjectsClient, context, ruleMonitoringService, ruleResultService } = params;
-
-        invariant(ruleMonitoringService, 'ruleMonitoringService required for detection rules');
-        invariant(ruleResultService, 'ruleResultService required for detection rules');
-
-        const childLogger = logger.get('ruleExecution');
-
-        const ruleExecutionSettings = await fetchRuleExecutionSettings(
-          config,
-          childLogger,
-          core,
-          savedObjectsClient
-        );
-
-        const eventLogWriter = createEventLogWriter(plugins.eventLog);
-
-        return createClientForExecutors(
-          ruleExecutionSettings,
-          eventLogWriter,
-          childLogger,
-          context,
-          ruleMonitoringService,
-          ruleResultService
-        );
-      });
-    },
-  };
-};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service_interface.ts

@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { SavedObjectsClientContract } from '@kbn/core/server';
-import type { IEventLogClient } from '@kbn/event-log-plugin/server';
-
-import type {
-  PublicRuleResultService,
-  PublicRuleMonitoringService,
-} from '@kbn/alerting-plugin/server/types';
-import type { IRuleExecutionLogForRoutes } from './client_for_routes/client_interface';
-import type {
-  IRuleExecutionLogForExecutors,
-  RuleExecutionContext,
-} from './client_for_executors/client_interface';
-
-export interface IRuleExecutionLogService {
-  registerEventLogProvider(): void;
-
-  createClientForRoutes(params: ClientForRoutesParams): IRuleExecutionLogForRoutes;
-
-  createClientForExecutors(
-    params: ClientForExecutorsParams
-  ): Promise<IRuleExecutionLogForExecutors>;
-}
-
-export interface ClientForRoutesParams {
-  savedObjectsClient: SavedObjectsClientContract;
-  eventLogClient: IEventLogClient;
-}
-
-export interface ClientForExecutorsParams {
-  savedObjectsClient: SavedObjectsClientContract;
-  ruleMonitoringService?: PublicRuleMonitoringService;
-  ruleResultService?: PublicRuleResultService;
-  context: RuleExecutionContext;
-}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { invariant } from '../../../../../common/utils/invariant';
+import type { ConfigType } from '../../../../config';
+import { withSecuritySpan } from '../../../../utils/with_security_span';
+import type {
+  SecuritySolutionPluginCoreSetupDependencies,
+  SecuritySolutionPluginSetupDependencies,
+} from '../../../../plugin_contract';
+
+import type { IDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client_interface';
+import type { IRuleExecutionLogForRoutes } from './rule_execution_log/client_for_routes/client_interface';
+import { createRuleExecutionLogClientForRoutes } from './rule_execution_log/client_for_routes/client';
+import type { IRuleExecutionLogForExecutors } from './rule_execution_log/client_for_executors/client_interface';
+import { createRuleExecutionLogClientForExecutors } from './rule_execution_log/client_for_executors/client';
+
+import { registerEventLogProvider } from './event_log/register_event_log_provider';
+import { createDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client';
+import { createEventLogHealthClient } from './detection_engine_health/event_log/event_log_health_client';
+import { createRuleObjectsHealthClient } from './detection_engine_health/rule_objects/rule_objects_health_client';
+import { createEventLogReader } from './rule_execution_log/event_log/event_log_reader';
+import { createEventLogWriter } from './rule_execution_log/event_log/event_log_writer';
+import { fetchRuleExecutionSettings } from './rule_execution_log/execution_settings/fetch_rule_execution_settings';
+import type {
+  RuleExecutionLogClientForExecutorsParams,
+  RuleExecutionLogClientForRoutesParams,
+  IRuleMonitoringService,
+  DetectionEngineHealthClientParams,
+} from './service_interface';
+
+export const createRuleMonitoringService = (
+  config: ConfigType,
+  logger: Logger,
+  core: SecuritySolutionPluginCoreSetupDependencies,
+  plugins: SecuritySolutionPluginSetupDependencies
+): IRuleMonitoringService => {
+  return {
+    registerEventLogProvider: () => {
+      registerEventLogProvider(plugins.eventLog);
+    },
+
+    createDetectionEngineHealthClient: (
+      params: DetectionEngineHealthClientParams
+    ): IDetectionEngineHealthClient => {
+      const { rulesClient, eventLogClient, currentSpaceId } = params;
+      const ruleObjectsHealthClient = createRuleObjectsHealthClient(rulesClient);
+      const eventLogHealthClient = createEventLogHealthClient(eventLogClient);
+      return createDetectionEngineHealthClient(
+        ruleObjectsHealthClient,
+        eventLogHealthClient,
+        logger,
+        currentSpaceId
+      );
+    },
+
+    createRuleExecutionLogClientForRoutes: (
+      params: RuleExecutionLogClientForRoutesParams
+    ): IRuleExecutionLogForRoutes => {
+      const { eventLogClient } = params;
+      const eventLogReader = createEventLogReader(eventLogClient);
+      return createRuleExecutionLogClientForRoutes(eventLogReader, logger);
+    },
+
+    createRuleExecutionLogClientForExecutors: (
+      params: RuleExecutionLogClientForExecutorsParams
+    ): Promise<IRuleExecutionLogForExecutors> => {
+      return withSecuritySpan(
+        'IRuleMonitoringService.createRuleExecutionLogClientForExecutors',
+        async () => {
+          const { savedObjectsClient, context, ruleMonitoringService, ruleResultService } = params;
+
+          invariant(ruleMonitoringService, 'ruleMonitoringService required for detection rules');
+          invariant(ruleResultService, 'ruleResultService required for detection rules');
+
+          const childLogger = logger.get('ruleExecution');
+
+          const ruleExecutionSettings = await fetchRuleExecutionSettings(
+            config,
+            childLogger,
+            core,
+            savedObjectsClient
+          );
+
+          const eventLogWriter = createEventLogWriter(plugins.eventLog);
+
+          return createRuleExecutionLogClientForExecutors(
+            ruleExecutionSettings,
+            eventLogWriter,
+            childLogger,
+            context,
+            ruleMonitoringService,
+            ruleResultService
+          );
+        }
+      );
+    },
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service_interface.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsClientContract } from '@kbn/core/server';
+import type { IEventLogClient } from '@kbn/event-log-plugin/server';
+import type {
+  PublicRuleResultService,
+  PublicRuleMonitoringService,
+  RulesClientApi,
+} from '@kbn/alerting-plugin/server/types';
+
+import type { IDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client_interface';
+import type { IRuleExecutionLogForRoutes } from './rule_execution_log/client_for_routes/client_interface';
+import type {
+  IRuleExecutionLogForExecutors,
+  RuleExecutionContext,
+} from './rule_execution_log/client_for_executors/client_interface';
+
+export interface IRuleMonitoringService {
+  registerEventLogProvider(): void;
+
+  createDetectionEngineHealthClient(
+    params: DetectionEngineHealthClientParams
+  ): IDetectionEngineHealthClient;
+
+  createRuleExecutionLogClientForRoutes(
+    params: RuleExecutionLogClientForRoutesParams
+  ): IRuleExecutionLogForRoutes;
+
+  createRuleExecutionLogClientForExecutors(
+    params: RuleExecutionLogClientForExecutorsParams
+  ): Promise<IRuleExecutionLogForExecutors>;
+}
+
+export interface DetectionEngineHealthClientParams {
+  savedObjectsClient: SavedObjectsClientContract;
+  rulesClient: RulesClientApi;
+  eventLogClient: IEventLogClient;
+  currentSpaceId: string;
+}
+
+export interface RuleExecutionLogClientForRoutesParams {
+  savedObjectsClient: SavedObjectsClientContract;
+  eventLogClient: IEventLogClient;
+}
+
+export interface RuleExecutionLogClientForExecutorsParams {
+  savedObjectsClient: SavedObjectsClientContract;
+  ruleMonitoringService?: PublicRuleMonitoringService;
+  ruleResultService?: PublicRuleResultService;
+  context: RuleExecutionContext;
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/console_logging.ts

@@ -6,13 +6,13 @@
  */
 
 import type { LogMeta } from '@kbn/core/server';
-import type { RuleExecutionStatus } from '../../../../../../../common/detection_engine/rule_monitoring';
+import type { RuleExecutionStatus } from '../../../../../../common/detection_engine/rule_monitoring';
 
 /**
  * Extended metadata that rule execution logger can attach to every console log record.
  */
 export interface ExtMeta extends LogMeta {
-  rule: ExtRule;
+  rule?: ExtRule;
   kibana?: ExtKibana;
 }
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/es_aggregations.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DEFAULT_PERCENTILES: number[] = [50, 95, 99, 99.9];

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/kql.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const kqlAnd = <T>(items: T[]): string => {
+  return items.filter(Boolean).map(String).join(' and ');
+};
+
+export const kqlOr = <T>(items: T[]): string => {
+  return items.filter(Boolean).map(String).join(' or ');
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/normalization.ts

@@ -7,6 +7,12 @@
 
 import { take, toString, truncate, uniq } from 'lodash';
 
+/**
+ * Useful for normalizing responses from Elasticsearch.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type RawData = any;
+
 // When we write rule execution status updates to saved objects or to event log,
 // we can write warning/failure messages as well. In some cases those messages
 // are built from N errors collected during the "big loop" of Detection Engine,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/mocks.ts

@@ -5,4 +5,5 @@
  * 2.0.
  */
 
+export * from './logic/detection_engine_health/__mocks__';
 export * from './logic/rule_execution_log/__mocks__';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/preview_rule_execution_logger.ts

@@ -6,13 +6,13 @@
  */
 
 import type {
-  IRuleExecutionLogService,
+  IRuleMonitoringService,
   RuleExecutionContext,
   StatusChangeArgs,
 } from '../../../rule_monitoring';
 
 export interface IPreviewRuleExecutionLogger {
-  factory: IRuleExecutionLogService['createClientForExecutors'];
+  factory: IRuleMonitoringService['createRuleExecutionLogClientForExecutors'];
 }
 
 export const createPreviewRuleExecutionLogger = (

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -40,7 +40,7 @@ import type { SetupPlugins } from '../../../plugin';
 import type { CompleteRule, RuleParams } from '../rule_schema';
 import type { ExperimentalFeatures } from '../../../../common/experimental_features';
 import type { ITelemetryEventsSender } from '../../telemetry/sender';
-import type { IRuleExecutionLogForExecutors, IRuleExecutionLogService } from '../rule_monitoring';
+import type { IRuleExecutionLogForExecutors, IRuleMonitoringService } from '../rule_monitoring';
 import type { RefreshTypes } from '../types';
 
 import type { Status } from '../../../../common/detection_engine/schemas/common/schemas';
@@ -133,7 +133,7 @@ export interface CreateSecurityRuleTypeWrapperProps {
   config: ConfigType;
   publicBaseUrl: string | undefined;
   ruleDataClient: IRuleDataClient;
-  ruleExecutionLoggerFactory: IRuleExecutionLogService['createClientForExecutors'];
+  ruleExecutionLoggerFactory: IRuleMonitoringService['createRuleExecutionLogClientForExecutors'];
   version: string;
   isPreview?: boolean;
 }

x-pack/plugins/security_solution/server/plugin.ts

@@ -71,7 +71,7 @@ import { TelemetryReceiver } from './lib/telemetry/receiver';
 import { licenseService } from './lib/license';
 import { PolicyWatcher } from './endpoint/lib/policy/license_watch';
 import previewPolicy from './lib/detection_engine/routes/index/preview_policy.json';
-import { createRuleExecutionLogService } from './lib/detection_engine/rule_monitoring';
+import { createRuleMonitoringService } from './lib/detection_engine/rule_monitoring';
 import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features';
 import { EndpointMetadataService } from './endpoint/services/metadata';
 import type {
@@ -161,11 +161,13 @@ export class Plugin implements ISecuritySolutionPlugin {
 
     initSavedObjects(core.savedObjects);
     initUiSettings(core.uiSettings, experimentalFeatures);
+
     if (experimentalFeatures.assistantEnabled ?? false) {
       plugins.actions.registerSubActionConnectorType(getGenerativeAiConnectorType());
     }
-    const ruleExecutionLogService = createRuleExecutionLogService(config, logger, core, plugins);
-    ruleExecutionLogService.registerEventLogProvider();
+
+    const ruleMonitoringService = createRuleMonitoringService(config, logger, core, plugins);
+    ruleMonitoringService.registerEventLogProvider();
 
     const requestContextFactory = new RequestContextFactory({
       config,
@@ -173,7 +175,7 @@ export class Plugin implements ISecuritySolutionPlugin {
       core,
       plugins,
       endpointAppContextService: this.endpointAppContextService,
-      ruleExecutionLogService,
+      ruleMonitoringService,
       kibanaVersion: pluginContext.env.packageInfo.version,
       kibanaBranch: pluginContext.env.packageInfo.branch,
     });
@@ -243,7 +245,7 @@ export class Plugin implements ISecuritySolutionPlugin {
       config: this.config,
       publicBaseUrl: core.http.basePath.publicBaseUrl,
       ruleDataClient,
-      ruleExecutionLoggerFactory: ruleExecutionLogService.createClientForExecutors,
+      ruleExecutionLoggerFactory: ruleMonitoringService.createRuleExecutionLogClientForExecutors,
       version: pluginContext.env.packageInfo.version,
     };
 

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -12,7 +12,7 @@ import type { Logger, KibanaRequest, RequestHandlerContext } from '@kbn/core/ser
 import { DEFAULT_SPACE_ID } from '../common/constants';
 import { AppClientFactory } from './client';
 import type { ConfigType } from './config';
-import type { IRuleExecutionLogService } from './lib/detection_engine/rule_monitoring';
+import type { IRuleMonitoringService } from './lib/detection_engine/rule_monitoring';
 import { buildFrameworkRequest } from './lib/timeline/utils/common';
 import type {
   SecuritySolutionPluginCoreSetupDependencies,
@@ -39,7 +39,7 @@ interface ConstructorOptions {
   core: SecuritySolutionPluginCoreSetupDependencies;
   plugins: SecuritySolutionPluginSetupDependencies;
   endpointAppContextService: EndpointAppContextService;
-  ruleExecutionLogService: IRuleExecutionLogService;
+  ruleMonitoringService: IRuleMonitoringService;
   kibanaVersion: string;
   kibanaBranch: string;
 }
@@ -56,13 +56,16 @@ export class RequestContextFactory implements IRequestContextFactory {
     request: KibanaRequest
   ): Promise<SecuritySolutionApiRequestHandlerContext> {
     const { options, appClientFactory } = this;
-    const { config, core, plugins, endpointAppContextService, ruleExecutionLogService } = options;
+    const { config, core, plugins, endpointAppContextService, ruleMonitoringService } = options;
     const { lists, ruleRegistry, security } = plugins;
 
     const [, startPlugins] = await core.getStartServices();
     const frameworkRequest = await buildFrameworkRequest(context, security, request);
     const coreContext = await context.core;
 
+    const getSpaceId = (): string =>
+      startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID;
+
     appClientFactory.setup({
       getSpaceId: startPlugins.spaces?.spacesService?.getSpaceId,
       config,
@@ -92,14 +95,23 @@ export class RequestContextFactory implements IRequestContextFactory {
 
       getAppClient: () => appClientFactory.create(request),
 
-      getSpaceId: () => startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID,
+      getSpaceId,
 
       getRuleDataService: () => ruleRegistry.ruleDataService,
 
       getRacClient: startPlugins.ruleRegistry.getRacClientWithRequest,
 
+      getDetectionEngineHealthClient: memoize(() =>
+        ruleMonitoringService.createDetectionEngineHealthClient({
+          savedObjectsClient: coreContext.savedObjects.client,
+          rulesClient: startPlugins.alerting.getRulesClientWithRequest(request),
+          eventLogClient: startPlugins.eventLog.getClient(request),
+          currentSpaceId: getSpaceId(),
+        })
+      ),
+
       getRuleExecutionLog: memoize(() =>
-        ruleExecutionLogService.createClientForRoutes({
+        ruleMonitoringService.createRuleExecutionLogClientForRoutes({
           savedObjectsClient: coreContext.savedObjects.client,
           eventLogClient: startPlugins.eventLog.getClient(request),
         })

x-pack/plugins/security_solution/server/types.ts

@@ -21,7 +21,10 @@ import type { IRuleDataService, AlertsClient } from '@kbn/rule-registry-plugin/s
 import type { Immutable } from '../common/endpoint/types';
 import { AppClient } from './client';
 import type { ConfigType } from './config';
-import type { IRuleExecutionLogForRoutes } from './lib/detection_engine/rule_monitoring';
+import type {
+  IDetectionEngineHealthClient,
+  IRuleExecutionLogForRoutes,
+} from './lib/detection_engine/rule_monitoring';
 import type { FrameworkRequest } from './lib/framework';
 import type { EndpointAuthz } from '../common/endpoint/types/authz';
 import type { EndpointInternalFleetServicesInterface } from './endpoint/services/fleet';
@@ -36,6 +39,7 @@ export interface SecuritySolutionApiRequestHandlerContext {
   getAppClient: () => AppClient;
   getSpaceId: () => string;
   getRuleDataService: () => IRuleDataService;
+  getDetectionEngineHealthClient: () => IDetectionEngineHealthClient;
   getRuleExecutionLog: () => IRuleExecutionLogForRoutes;
   getRacClient: (req: KibanaRequest) => Promise<AlertsClient>;
   getExceptionListClient: () => ExceptionListClient | null;