[Metric threshold] Save the ECS group by fields at the AAD root level (#188976)

Related to #183220

## Summary

This PR extracts `getEcsGroups` to a package to save ECS groups in the
Alert As Data (AAD) document for the metric threshold rule.

### 🧪 How to test
- Create a metric threshold rule with multiple groups (both ECS and
non-ECS fields)
- Check the related AAD document; you should be able to see the ECS
fields at the root level and not see non-ECS fields there
- Check the same information for the recovered alerts
- Rules without group by should work as before

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -617,6 +617,7 @@ x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/ob
 x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant
 x-pack/plugins/observability_solution/observability_ai_assistant @elastic/obs-ai-assistant
 x-pack/packages/observability/alert_details @elastic/obs-ux-management-team
+x-pack/packages/observability/alerting_rule_utils @elastic/obs-ux-management-team
 x-pack/packages/observability/alerting_test_data @elastic/obs-ux-management-team
 x-pack/test/cases_api_integration/common/plugins/observability @elastic/response-ops
 x-pack/packages/observability/get_padded_alert_time_range_util @elastic/obs-ux-management-team

package.json

@@ -648,6 +648,7 @@
     "@kbn/observability-ai-assistant-management-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant_management",
     "@kbn/observability-ai-assistant-plugin": "link:x-pack/plugins/observability_solution/observability_ai_assistant",
     "@kbn/observability-alert-details": "link:x-pack/packages/observability/alert_details",
+    "@kbn/observability-alerting-rule-utils": "link:x-pack/packages/observability/alerting_rule_utils",
     "@kbn/observability-alerting-test-data": "link:x-pack/packages/observability/alerting_test_data",
     "@kbn/observability-fixtures-plugin": "link:x-pack/test/cases_api_integration/common/plugins/observability",
     "@kbn/observability-get-padded-alert-time-range-util": "link:x-pack/packages/observability/get_padded_alert_time_range_util",

tsconfig.base.json

@@ -1228,6 +1228,8 @@
       "@kbn/observability-ai-assistant-plugin/*": ["x-pack/plugins/observability_solution/observability_ai_assistant/*"],
       "@kbn/observability-alert-details": ["x-pack/packages/observability/alert_details"],
       "@kbn/observability-alert-details/*": ["x-pack/packages/observability/alert_details/*"],
+      "@kbn/observability-alerting-rule-utils": ["x-pack/packages/observability/alerting_rule_utils"],
+      "@kbn/observability-alerting-rule-utils/*": ["x-pack/packages/observability/alerting_rule_utils/*"],
       "@kbn/observability-alerting-test-data": ["x-pack/packages/observability/alerting_test_data"],
       "@kbn/observability-alerting-test-data/*": ["x-pack/packages/observability/alerting_test_data/*"],
       "@kbn/observability-fixtures-plugin": ["x-pack/test/cases_api_integration/common/plugins/observability"],

x-pack/packages/observability/alerting_rule_utils/README.md

@@ -0,0 +1,5 @@
+# @kbn/alerting-rule-utils
+
+Utilities shared between observability alerting rules
+
+- getEcsGroups: By passing the group by fields to this function, it will return fields that exist in ECS mapping with keyword type

x-pack/packages/observability/alerting_rule_utils/index.ts

@@ -5,7 +5,5 @@
  * 2.0.
  */
 
-export interface Group {
-  field: string;
-  value: string;
-}
+export { getEcsGroups } from './src/get_ecs_groups';
+export type { Group } from './src/get_ecs_groups';

x-pack/packages/observability/alerting_rule_utils/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../..',
+  roots: ['<rootDir>/x-pack/packages/observability/alerting_rule_utils'],
+};

x-pack/packages/observability/alerting_rule_utils/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/observability-alerting-rule-utils",
+  "owner": "@elastic/obs-ux-management-team"
+}

x-pack/packages/observability/alerting_rule_utils/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/observability-alerting-rule-utils",
+  "description": "Utils shared between observability alerting rules",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/observability/alerting_rule_utils/src/get_ecs_groups.ts

@@ -6,7 +6,11 @@
  */
 
 import { ecsFieldMap } from '@kbn/alerts-as-data-utils';
-import { Group } from '../../../../../common/typings';
+
+export interface Group {
+  field: string;
+  value: string;
+}
 
 export const getEcsGroups = (groups: Group[] = []): Record<string, string> => {
   const ecsGroups = groups.filter((group) => {

x-pack/packages/observability/alerting_rule_utils/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/alerts-as-data-utils"
+  ]
+}

x-pack/plugins/observability_solution/infra/server/lib/alerting/common/utils.ts

@@ -19,6 +19,7 @@ import {
 import { ES_FIELD_TYPES } from '@kbn/field-types';
 import { set } from '@kbn/safer-lodash-set';
 import { Alert } from '@kbn/alerts-as-data-utils';
+import { type Group } from '@kbn/observability-alerting-rule-utils';
 import { ParsedExperimentalFields } from '@kbn/rule-registry-plugin/common/parse_experimental_fields';
 import {
   getInventoryViewInAppUrl,
@@ -28,7 +29,6 @@ import {
   AlertExecutionDetails,
   InventoryMetricConditions,
 } from '../../../../common/alerting/metrics/types';
-import { Group } from '../../../../common/alerting/types';
 
 const ALERT_CONTEXT_CONTAINER = 'container';
 const ALERT_CONTEXT_ORCHESTRATOR = 'orchestrator';

x-pack/plugins/observability_solution/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts

@@ -6,10 +6,12 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { Group } from '@kbn/observability-alerting-rule-utils';
 import {
   ALERT_REASON,
   ALERT_EVALUATION_VALUES,
   ALERT_EVALUATION_THRESHOLD,
+  ALERT_GROUP,
 } from '@kbn/rule-data-utils';
 import { first, get } from 'lodash';
 import {
@@ -64,11 +66,12 @@ export type InventoryMetricThresholdAlertContext = AlertContext; // no specific
 
 export type InventoryMetricThresholdAlert = Omit<
   ObservabilityMetricsAlert,
-  'kibana.alert.evaluation.values' | 'kibana.alert.evaluation.threshold'
+  'kibana.alert.evaluation.values' | 'kibana.alert.evaluation.threshold' | 'kibana.alert.group'
 > & {
   // Defining a custom type for this because the schema generation script doesn't allow explicit null values
   [ALERT_EVALUATION_VALUES]?: Array<number | null>;
   [ALERT_EVALUATION_THRESHOLD]?: Array<number | null>;
+  [ALERT_GROUP]?: Group[];
 };
 
 export const createInventoryMetricThresholdExecutor =

x-pack/plugins/observability_solution/infra/server/lib/alerting/log_threshold/log_threshold_executor.ts

@@ -31,6 +31,7 @@ import {
   PublicAlertsClient,
   RecoveredAlertData,
 } from '@kbn/alerting-plugin/server/alerts_client/types';
+import { type Group } from '@kbn/observability-alerting-rule-utils';
 
 import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
 import { decodeOrThrow } from '@kbn/io-ts-utils';
@@ -77,7 +78,6 @@ import {
   LogThresholdRuleTypeParams,
   positiveComparators,
 } from '../../../../common/alerting/logs/log_threshold/query_helpers';
-import { Group } from '../../../../common/alerting/types';
 
 export type LogThresholdActionGroups = ActionGroupIdsOf<typeof FIRED_ACTIONS>;
 export type LogThresholdRuleTypeState = RuleTypeState; // no specific state used

x-pack/plugins/observability_solution/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts

@@ -30,7 +30,7 @@ import {
   ALERT_REASON,
   ALERT_GROUP,
 } from '@kbn/rule-data-utils';
-import { Group } from '../../../../common/alerting/types';
+import { type Group } from '@kbn/observability-alerting-rule-utils';
 
 jest.mock('./lib/evaluate_rule', () => ({ evaluateRule: jest.fn() }));
 
@@ -959,6 +959,7 @@ describe('The metric threshold rule type', () => {
         tags: ['host-01_tag1', 'host-01_tag2', 'ruleTag1', 'ruleTag2'],
         groupByKeys: { host: { name: alertIdA } },
         group: [{ field: 'host.name', value: alertIdA }],
+        ecsGroups: { 'host.name': alertIdA },
       });
       testAlertReported(2, {
         id: alertIdB,
@@ -971,6 +972,7 @@ describe('The metric threshold rule type', () => {
         tags: ['host-02_tag1', 'host-02_tag2', 'ruleTag1', 'ruleTag2'],
         groupByKeys: { host: { name: alertIdB } },
         group: [{ field: 'host.name', value: alertIdB }],
+        ecsGroups: { 'host.name': alertIdB },
       });
     });
   });
@@ -2333,6 +2335,7 @@ describe('The metric threshold rule type', () => {
       conditions,
       reason,
       tags,
+      ecsGroups,
     }: {
       id: string;
       actionGroup: string;
@@ -2348,6 +2351,7 @@ describe('The metric threshold rule type', () => {
       reason: string;
       tags?: string[];
       group?: Group[];
+      ecsGroups?: Record<string, string>;
     }
   ) {
     expect(services.alertsClient.report).toHaveBeenNthCalledWith(index, {
@@ -2416,6 +2420,7 @@ describe('The metric threshold rule type', () => {
           : {}),
         [ALERT_REASON]: reason,
         ...(tags ? { tags } : {}),
+        ...(ecsGroups ? ecsGroups : {}),
       },
     });
   }

x-pack/plugins/observability_solution/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts

@@ -23,6 +23,7 @@ import { AlertsClientError, RuleExecutorOptions, RuleTypeState } from '@kbn/aler
 import { TimeUnitChar, getAlertUrl } from '@kbn/observability-plugin/common';
 import { ObservabilityMetricsAlert } from '@kbn/alerts-as-data-utils';
 import { COMPARATORS } from '@kbn/alerting-comparators';
+import { getEcsGroups, type Group } from '@kbn/observability-alerting-rule-utils';
 import { convertToBuiltInComparators } from '@kbn/observability-plugin/common/utils/convert_legacy_outside_comparator';
 import { getOriginalActionGroup } from '../../../utils/get_original_action_group';
 import { AlertStates } from '../../../../common/alerting/metrics';
@@ -52,15 +53,15 @@ import { getEvaluationValues, getThresholds } from '../common/get_values';
 import { EvaluatedRuleParams, evaluateRule, Evaluation } from './lib/evaluate_rule';
 import { MissingGroupsRecord } from './lib/check_missing_group';
 import { convertStringsToMissingGroupsRecord } from './lib/convert_strings_to_missing_groups_record';
-import { Group } from '../../../../common/alerting/types';
 
 export type MetricThresholdAlert = Omit<
   ObservabilityMetricsAlert,
-  'kibana.alert.evaluation.values'
+  'kibana.alert.evaluation.values' | 'kibana.alert.evaluation.threshold' | 'kibana.alert.group'
 > & {
   // Defining a custom type for this because the schema generation script doesn't allow explicit null values
   [ALERT_EVALUATION_VALUES]?: Array<number | null>;
   [ALERT_EVALUATION_THRESHOLD]?: Array<number | null>;
+  [ALERT_GROUP]?: Group[];
 };
 
 export type MetricThresholdRuleParams = Record<string, any>;
@@ -94,7 +95,7 @@ type MetricThresholdAlertReporter = (params: {
   context: MetricThresholdAlertContext;
   additionalContext?: AdditionalContext | null;
   evaluationValues?: Array<number | null>;
-  groups?: object[];
+  groups?: Group[];
   thresholds?: Array<number | null>;
 }) => void;
 
@@ -149,7 +150,6 @@ export const createMetricThresholdExecutor =
         id,
         actionGroup,
       });
-      const groupsPayload = typeof groups !== 'undefined' ? { [ALERT_GROUP]: groups } : {};
 
       alertsClient.setAlertData({
         id,
@@ -157,8 +157,9 @@ export const createMetricThresholdExecutor =
           [ALERT_REASON]: reason,
           [ALERT_EVALUATION_VALUES]: evaluationValues,
           [ALERT_EVALUATION_THRESHOLD]: thresholds,
-          ...groupsPayload,
+          [ALERT_GROUP]: groups,
           ...flattenAdditionalContext(additionalContext),
+          ...getEcsGroups(groups),
         },
         context: {
           ...contextWithoutAlertDetailsUrl,

x-pack/plugins/observability_solution/infra/tsconfig.json

@@ -102,6 +102,7 @@
     "@kbn/router-utils",
     "@kbn/react-kibana-context-render",
     "@kbn/react-kibana-context-theme",
+    "@kbn/observability-alerting-rule-utils",
     "@kbn/presentation-publishing",
     "@kbn/presentation-containers",
     "@kbn/deeplinks-observability",

x-pack/plugins/observability_solution/observability/server/lib/rules/custom_threshold/custom_threshold_executor.test.ts

@@ -1635,25 +1635,6 @@ describe('The custom threshold alert type', () => {
         });
         await execute(COMPARATORS.GREATER_THAN, [0.9]);
         const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
-        expect(services.alertsClient.setAlertData).toBeCalledTimes(1);
-        expect(services.alertsClient.setAlertData).toBeCalledWith({
-          context: {
-            alertDetailsUrl: 'http://localhost:5601/app/observability/alerts/mockedUuid',
-            viewInAppUrl: 'mockedViewInApp',
-            group: [
-              {
-                field: 'host.name',
-                value: 'host-0',
-              },
-            ],
-            host: {
-              name: 'host-0',
-            },
-            timestamp: expect.stringMatching(ISO_DATE_REGEX),
-          },
-          id: 'host-0',
-          'host.name': 'host-0',
-        });
         expect(getViewInAppUrl).toBeCalledTimes(1);
         expect(getViewInAppUrl).toBeCalledWith({
           dataViewId: 'c34a7c79-a88b-4b4a-ad19-72f6d24104e4',

x-pack/plugins/observability_solution/observability/server/lib/rules/custom_threshold/custom_threshold_executor.ts

@@ -17,11 +17,11 @@ import { LocatorPublic } from '@kbn/share-plugin/common';
 import { RecoveredActionGroup } from '@kbn/alerting-plugin/common';
 import { IBasePath, Logger } from '@kbn/core/server';
 import { AlertsClientError, RuleExecutorOptions } from '@kbn/alerting-plugin/server';
+import { getEcsGroups } from '@kbn/observability-alerting-rule-utils';
 import { getEvaluationValues, getThreshold } from './lib/get_values';
 import { AlertsLocatorParams, getAlertDetailsUrl } from '../../../../common';
 import { getViewInAppUrl } from '../../../../common/custom_threshold_rule/get_view_in_app_url';
 import { ObservabilityConfig } from '../../..';
-import { getEcsGroups } from './lib/get_ecs_groups';
 import { FIRED_ACTIONS_ID, NO_DATA_ACTIONS_ID, UNGROUPED_FACTORY_KEY } from './constants';
 import {
   AlertStates,
@@ -323,7 +323,6 @@ export const createCustomThresholdExecutor = ({
       alertsClient.setAlertData({
         id: recoveredAlertId,
         context,
-        ...getEcsGroups(group),
       });
     }
 

x-pack/plugins/observability_solution/observability/server/lib/rules/custom_threshold/utils.ts

@@ -167,7 +167,7 @@ export const hasAdditionalContext = (
 ): boolean => {
   return groupBy
     ? Array.isArray(groupBy)
-      ? groupBy.every((group) => validGroups.includes(group))
+      ? groupBy.some((group) => validGroups.includes(group))
       : validGroups.includes(groupBy)
     : false;
 };

x-pack/plugins/observability_solution/observability/tsconfig.json

@@ -108,6 +108,7 @@
     "@kbn/event-annotation-components",
     "@kbn/slo-schema",
     "@kbn/license-management-plugin",
+    "@kbn/observability-alerting-rule-utils",
   ],
   "exclude": [
     "target/**/*"

x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts

@@ -136,7 +136,7 @@ export default function ({ getService }: FtrProviderContext) {
               },
               index: DATA_VIEW_ID,
             },
-            groupBy: ['host.name', 'container.id'],
+            groupBy: ['host.name', 'container.id', 'event.dataset', '_index'],
           },
           actions: [
             {
@@ -209,10 +209,9 @@ export default function ({ getService }: FtrProviderContext) {
           'custom_threshold.fired'
         );
         expect(resp.hits.hits[0]._source).property('tags').contain('observability');
-        expect(resp.hits.hits[0]._source).property(
-          'kibana.alert.instance.id',
-          'host-0,container-0'
-        );
+        expect(resp.hits.hits[0]._source)
+          .property('kibana.alert.instance.id')
+          .contain('host-0,container-0,system.cpu,kbn-data-forge-fake_hosts.fake_hosts');
         expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open');
         expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
         expect(resp.hits.hits[0]._source).property('event.action', 'open');
@@ -223,20 +222,23 @@ export default function ({ getService }: FtrProviderContext) {
           .eql(['00-00-5E-00-53-23', '00-00-5E-00-53-24']);
         expect(resp.hits.hits[0]._source).property('container.id', 'container-0');
         expect(resp.hits.hits[0]._source).property('container.name', 'container-name');
+        expect(resp.hits.hits[0]._source).property('event.dataset', 'system.cpu');
         expect(resp.hits.hits[0]._source).not.property('container.cpu');
 
-        expect(resp.hits.hits[0]._source)
-          .property('kibana.alert.group')
-          .eql([
-            {
-              field: 'host.name',
-              value: 'host-0',
-            },
-            {
-              field: 'container.id',
-              value: 'container-0',
-            },
-          ]);
+        const alertGroups = (resp.hits.hits[0]._source as any)?.['kibana.alert.group'];
+        expect(alertGroups[0]).eql({
+          field: 'host.name',
+          value: 'host-0',
+        });
+        expect(alertGroups[1]).eql({
+          field: 'container.id',
+          value: 'container-0',
+        });
+        expect(alertGroups[2]).eql({
+          field: 'event.dataset',
+          value: 'system.cpu',
+        });
+        expect(alertGroups[3].value).contain('kbn-data-forge-fake_hosts.fake_hosts');
         expect(resp.hits.hits[0]._source).property('kibana.alert.evaluation.threshold').eql([0.2]);
         expect(resp.hits.hits[0]._source)
           .property('kibana.alert.rule.parameters')
@@ -253,7 +255,7 @@ export default function ({ getService }: FtrProviderContext) {
             alertOnNoData: true,
             alertOnGroupDisappear: true,
             searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
-            groupBy: ['host.name', 'container.id'],
+            groupBy: ['host.name', 'container.id', 'event.dataset', '_index'],
           });
       });
 
@@ -269,15 +271,15 @@ export default function ({ getService }: FtrProviderContext) {
         expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
           `https://localhost:5601/app/observability/alerts/${alertId}`
         );
-        expect(resp.hits.hits[0]._source?.reason).eql(
-          `Average system.cpu.total.norm.pct is 80%, above or equal the threshold of 20%. (duration: 1 min, data view: ${DATA_VIEW}, group: host-0,container-0)`
+        expect(resp.hits.hits[0]._source?.reason).contain(
+          `Average system.cpu.total.norm.pct is 80%, above or equal the threshold of 20%. (duration: 1 min, data view: ${DATA_VIEW}, group: host-0,container-0,system.cpu,kbn-data-forge-fake_hosts.fake_hosts`
         );
         expect(resp.hits.hits[0]._source?.value).eql('80%');
         expect(resp.hits.hits[0]._source?.host).eql(
           '{"name":"host-0","mac":["00-00-5E-00-53-23","00-00-5E-00-53-24"]}'
         );
-        expect(resp.hits.hits[0]._source?.group).eql(
-          '{"field":"host.name","value":"host-0"},{"field":"container.id","value":"container-0"}'
+        expect(resp.hits.hits[0]._source?.group).contain(
+          '{"field":"host.name","value":"host-0"},{"field":"container.id","value":"container-0"},{"field":"event.dataset","value":"system.cpu"},{"field":"_index","value":"kbn-data-forge-fake_hosts.fake_hosts'
         );
       });
     });

yarn.lock

@@ -5721,6 +5721,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/observability-alerting-rule-utils@link:x-pack/packages/observability/alerting_rule_utils":
+  version "0.0.0"
+  uid ""
+
 "@kbn/observability-alerting-test-data@link:x-pack/packages/observability/alerting_test_data":
   version "0.0.0"
   uid ""