[ES Query] Save ECS keyword group by fields in AAD document (#191103)

Related to https://github.com/elastic/kibana/issues/183220 ## Summary This PR saves ECS keyword group by fields in AAD document for ES query rule. |Rule|Before|After| |---|---|---| |![image](00a57945-1590-4286-8132-072e19d5866f)| |![image](41ce2028-54f0-4ca4-8d63-0d371819865f)| ### How to test - Create some data with ECS fields - For example, you can use synthtrace command: `node scripts/synthtrace simple_trace.ts --local --live` - Create an ES Query rule grouped by ECS and non-ECS fields - In the generated alert, you should be able to see the ECS group by field but not the no-ECS ones
2025-04-24 17:59:23 -04:00 · 2024-08-23 23:02:37 +02:00 · 2024-08-23 23:02:37 +02:00 · 8ba84ecf00
commit 8ba84ecf00
parent 0770e947e2
7 changed files with 328 additions and 2 deletions
--- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts
@ -377,16 +377,19 @@ describe('es_query executor', () => {
          results: [
            {
              group: 'host-1',
+              groups: [{ field: 'host.name', value: 'host-1' }],
              count: 291,
              hits: [],
            },
            {
              group: 'host-2',
+              groups: [{ field: 'host.name', value: 'host-2' }],
              count: 477,
              hits: [],
            },
            {
              group: 'host-3',
+              groups: [{ field: 'host.name', value: 'host-3' }],
              count: 999,
              hits: [],
            },
@ -429,6 +432,7 @@ describe('es_query executor', () => {
          latestTimestamp: undefined,
        },
        payload: {
+          'host.name': 'host-1',
          'kibana.alert.evaluation.conditions':
            'Number of matching documents for group "host-1" is greater than or equal to 200',
          'kibana.alert.evaluation.threshold': 200,
@ -460,6 +464,7 @@ describe('es_query executor', () => {
          latestTimestamp: undefined,
        },
        payload: {
+          'host.name': 'host-2',
          'kibana.alert.evaluation.conditions':
            'Number of matching documents for group "host-2" is greater than or equal to 200',
          'kibana.alert.evaluation.threshold': 200,
@ -491,6 +496,7 @@ describe('es_query executor', () => {
          latestTimestamp: undefined,
        },
        payload: {
+          'host.name': 'host-3',
          'kibana.alert.evaluation.conditions':
            'Number of matching documents for group "host-3" is greater than or equal to 200',
          'kibana.alert.evaluation.threshold': 200,
--- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts
@ -4,9 +4,11 @@
 * 2.0; you may not use this file except in compliance with the Elastic License
 * 2.0.
 */
+
 import { sha256 } from 'js-sha256';
 import { i18n } from '@kbn/i18n';
 import { CoreSetup, Logger } from '@kbn/core/server';
+import { getEcsGroups } from '@kbn/observability-alerting-rule-utils';
 import { isGroupAggregation, UngroupedGroupId } from '@kbn/triggers-actions-ui-plugin/common';
 import {
  ALERT_EVALUATION_THRESHOLD,
@ -178,6 +180,7 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
    });

    const id = alertId === UngroupedGroupId && !isGroupAgg ? ConditionMetAlertInstanceId : alertId;
+    const ecsGroups = getEcsGroups(result.groups);

    alertsClient.report({
      id,
@ -191,6 +194,7 @@ export async function executor(core: CoreSetup, options: ExecutorOptions<EsQuery
        [ALERT_EVALUATION_CONDITIONS]: actionContext.conditions,
        [ALERT_EVALUATION_VALUE]: `${actionContext.value}`,
        [ALERT_EVALUATION_THRESHOLD]: params.threshold?.length === 1 ? params.threshold[0] : null,
+        ...ecsGroups,
        ...actionContext.sourceFields,
      },
    });
--- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts
@ -106,6 +106,7 @@ export async function fetchSearchSourceQuery({
      isGroupAgg,
      esResult: searchResult,
      sourceFieldsParams: params.sourceFields,
+      termField: params.termField,
    }),
    index: [index.name],
    query: searchRequestBody,
--- a/x-pack/plugins/stack_alerts/tsconfig.json
+++ b/x-pack/plugins/stack_alerts/tsconfig.json
@ -54,7 +54,8 @@
    "@kbn/alerting-comparators",
    "@kbn/task-manager-plugin",
    "@kbn/core-logging-server-mocks",
-    "@kbn/core-saved-objects-server"
+    "@kbn/core-saved-objects-server",
+    "@kbn/observability-alerting-rule-utils"
  ],
  "exclude": [
    "target/**/*",
--- a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts
+++ b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts
@ -175,35 +175,66 @@ describe('parseAggregationResults', () => {
            },
          },
        },
+        termField: 'event',
      })
    ).toEqual({
      results: [
        {
          group: 'execute',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+          ],
          count: 120,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'execute-start',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+          ],
          count: 120,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'active-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'active-instance',
+            },
+          ],
          count: 100,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'execute-action',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-action',
+            },
+          ],
          count: 100,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'new-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'new-instance',
+            },
+          ],
          count: 100,
          hits: [],
          sourceFields: {},
@ -302,35 +333,66 @@ describe('parseAggregationResults', () => {
            },
          },
        },
+        termField: 'event',
      })
    ).toEqual({
      results: [
        {
          group: 'execute',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+          ],
          count: 120,
          hits: [sampleHit],
          sourceFields: {},
        },
        {
          group: 'execute-start',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+          ],
          count: 120,
          hits: [sampleHit],
          sourceFields: {},
        },
        {
          group: 'active-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'active-instance',
+            },
+          ],
          count: 100,
          hits: [sampleHit],
          sourceFields: {},
        },
        {
          group: 'execute-action',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-action',
+            },
+          ],
          count: 100,
          hits: [sampleHit],
          sourceFields: {},
        },
        {
          group: 'new-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'new-instance',
+            },
+          ],
          count: 100,
          hits: [sampleHit],
          sourceFields: {},
@ -425,11 +487,18 @@ describe('parseAggregationResults', () => {
            },
          },
        },
+        termField: 'event',
      })
    ).toEqual({
      results: [
        {
          group: 'execute-action',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-action',
+            },
+          ],
          count: 120,
          hits: [],
          value: null,
@ -437,6 +506,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'execute-start',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+          ],
          count: 139,
          hits: [],
          value: null,
@ -444,6 +519,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'starting',
+          groups: [
+            {
+              field: 'event',
+              value: 'starting',
+            },
+          ],
          count: 1,
          hits: [],
          value: null,
@ -451,6 +532,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'recovered-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'recovered-instance',
+            },
+          ],
          count: 120,
          hits: [],
          value: 12837500000,
@ -458,6 +545,160 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'execute',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+          ],
+          count: 139,
+          hits: [],
+          value: 137647482.0143885,
+          sourceFields: {},
+        },
+      ],
+      truncated: false,
+    });
+  });
+
+  it('correctly parses results for aggregate metric over top N multiple termFields', () => {
+    expect(
+      parseAggregationResults({
+        isCountAgg: false,
+        isGroupAgg: true,
+        esResult: {
+          took: 238,
+          timed_out: false,
+          _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+          hits: { total: 643, max_score: null, hits: [] },
+          aggregations: {
+            groupAgg: {
+              doc_count_error_upper_bound: 0,
+              sum_other_doc_count: 240,
+              buckets: [
+                {
+                  key: ['execute-action', 'action1'],
+                  doc_count: 120,
+                  metricAgg: {
+                    value: null,
+                  },
+                },
+                {
+                  key: ['execute-start', 'action2'],
+                  doc_count: 139,
+                  metricAgg: {
+                    value: null,
+                  },
+                },
+                {
+                  key: ['starting', 'action3'],
+                  doc_count: 1,
+                  metricAgg: {
+                    value: null,
+                  },
+                },
+                {
+                  key: ['recovered-instance', 'action4'],
+                  doc_count: 120,
+                  metricAgg: {
+                    value: 12837500000,
+                  },
+                },
+                {
+                  key: ['execute', 'action5'],
+                  doc_count: 139,
+                  metricAgg: {
+                    value: 137647482.0143885,
+                  },
+                },
+              ],
+            },
+          },
+        },
+        termField: ['event', 'action'],
+      })
+    ).toEqual({
+      results: [
+        {
+          group: 'execute-action,action1',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-action',
+            },
+            {
+              field: 'action',
+              value: 'action1',
+            },
+          ],
+          count: 120,
+          hits: [],
+          value: null,
+          sourceFields: {},
+        },
+        {
+          group: 'execute-start,action2',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+            {
+              field: 'action',
+              value: 'action2',
+            },
+          ],
+          count: 139,
+          hits: [],
+          value: null,
+          sourceFields: {},
+        },
+        {
+          group: 'starting,action3',
+          groups: [
+            {
+              field: 'event',
+              value: 'starting',
+            },
+            {
+              field: 'action',
+              value: 'action3',
+            },
+          ],
+          count: 1,
+          hits: [],
+          value: null,
+          sourceFields: {},
+        },
+        {
+          group: 'recovered-instance,action4',
+          groups: [
+            {
+              field: 'event',
+              value: 'recovered-instance',
+            },
+            {
+              field: 'action',
+              value: 'action4',
+            },
+          ],
+          count: 120,
+          hits: [],
+          value: 12837500000,
+          sourceFields: {},
+        },
+        {
+          group: 'execute,action5',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+            {
+              field: 'action',
+              value: 'action5',
+            },
+          ],
          count: 139,
          hits: [],
          value: 137647482.0143885,
@ -572,11 +813,18 @@ describe('parseAggregationResults', () => {
            },
          },
        },
+        termField: ['event'],
      })
    ).toEqual({
      results: [
        {
          group: 'execute-action',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-action',
+            },
+          ],
          count: 120,
          hits: [sampleHit],
          value: null,
@ -584,6 +832,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'execute-start',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+          ],
          count: 139,
          hits: [sampleHit],
          value: null,
@ -591,6 +845,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'starting',
+          groups: [
+            {
+              field: 'event',
+              value: 'starting',
+            },
+          ],
          count: 1,
          hits: [sampleHit],
          value: null,
@ -598,6 +858,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'recovered-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'recovered-instance',
+            },
+          ],
          count: 120,
          hits: [sampleHit],
          value: 12837500000,
@ -605,6 +871,12 @@ describe('parseAggregationResults', () => {
        },
        {
          group: 'execute',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+          ],
          count: 139,
          hits: [sampleHit],
          value: 137647482.0143885,
@ -658,23 +930,42 @@ describe('parseAggregationResults', () => {
          },
        },
        resultLimit: 3,
+        termField: ['event'],
      })
    ).toEqual({
      results: [
        {
          group: 'execute',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute',
+            },
+          ],
          count: 120,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'execute-start',
+          groups: [
+            {
+              field: 'event',
+              value: 'execute-start',
+            },
+          ],
          count: 120,
          hits: [],
          sourceFields: {},
        },
        {
          group: 'active-instance',
+          groups: [
+            {
+              field: 'event',
+              value: 'active-instance',
+            },
+          ],
          count: 100,
          hits: [],
          sourceFields: {},
@ -776,6 +1067,7 @@ describe('parseAggregationResults', () => {
          },
        },
        resultLimit: 1000,
+        termField: ['host.name'],
        sourceFieldsParams: [
          { label: 'host.hostname', searchPath: 'host.hostname.keyword' },
          { label: 'host.id', searchPath: 'host.id.keyword' },
@ -786,6 +1078,12 @@ describe('parseAggregationResults', () => {
      results: [
        {
          group: 'host-1',
+          groups: [
+            {
+              field: 'host.name',
+              value: 'host-1',
+            },
+          ],
          hits: [
            sampleSourceFieldsHit,
            sampleSourceFieldsHit,
--- a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts
+++ b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts
@ -11,6 +11,7 @@ import {
  SearchHitsMetadata,
  AggregationsSingleMetricAggregateBase,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { Group } from '@kbn/observability-alerting-rule-utils';

 export const UngroupedGroupId = 'all documents';
 export interface ParsedAggregationGroup {
@ -18,6 +19,7 @@ export interface ParsedAggregationGroup {
  count: number;
  hits: Array<SearchHit<unknown>>;
  sourceFields: string[];
+  groups?: Group[];
  value?: number;
 }

@ -33,6 +35,7 @@ interface ParseAggregationResultsOpts {
  resultLimit?: number;
  sourceFieldsParams?: Array<{ label: string; searchPath: string }>;
  generateSourceFieldsFromHits?: boolean;
+  termField?: string | string[];
 }
 export const parseAggregationResults = ({
  isCountAgg,
@ -41,6 +44,7 @@ export const parseAggregationResults = ({
  resultLimit,
  sourceFieldsParams = [],
  generateSourceFieldsFromHits = false,
+  termField,
 }: ParseAggregationResultsOpts): ParsedAggregationResults => {
  const aggregations = esResult?.aggregations || {};

@ -83,6 +87,16 @@ export const parseAggregationResults = ({
    if (resultLimit && results.results.length === resultLimit) break;

    const groupName: string = `${groupBucket?.key}`;
+    const groups =
+      termField && groupBucket?.key
+        ? [termField].flat().reduce<Group[]>((resultGroups, groupByItem, groupIndex) => {
+            resultGroups.push({
+              field: groupByItem,
+              value: [groupBucket.key].flat()[groupIndex],
+            });
+            return resultGroups;
+          }, [])
+        : undefined;
    const sourceFields: { [key: string]: string[] } = {};

    sourceFieldsParams.forEach((field) => {
@ -105,6 +119,7 @@ export const parseAggregationResults = ({

    const groupResult: any = {
      group: groupName,
+      groups,
      count: groupBucket?.doc_count,
      hits: groupBucket?.topHitsAgg?.hits?.hits ?? [],
      ...(!isCountAgg ? { value: groupBucket?.metricAgg?.value } : {}),
--- a/x-pack/plugins/triggers_actions_ui/tsconfig.json
+++ b/x-pack/plugins/triggers_actions_ui/tsconfig.json
@ -69,7 +69,8 @@
    "@kbn/alerting-comparators",
    "@kbn/alerting-types",
    "@kbn/visualization-utils",
-    "@kbn/core-ui-settings-browser"
+    "@kbn/core-ui-settings-browser",
+    "@kbn/observability-alerting-rule-utils"
  ],
  "exclude": ["target/**/*"]
 }