backport

x-pack/solutions/security/packages/kbn-securitysolution-utils/src/esql/get_mv_expand_fields.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getMvExpandFields } from './get_mv_expand_fields';
+
+describe('getMvExpandFields', () => {
+  it('returns empty array if MV_EXPAND not used', () => {
+    expect(getMvExpandFields('from auditbeat*')).toEqual([]);
+  });
+  it('returns single item array if MV_EXPAND used once', () => {
+    expect(getMvExpandFields('from auditbeat* | mv_expand agent.name')).toEqual(['agent.name']);
+  });
+  it('returns array of fields if MV_EXPAND used twice', () => {
+    expect(
+      getMvExpandFields('from auditbeat* | mv_expand agent.name | mv_expand host.name')
+    ).toEqual(['agent.name', 'host.name']);
+  });
+});

x-pack/solutions/security/packages/kbn-securitysolution-utils/src/esql/get_mv_expand_fields.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { parse } from '@kbn/esql-ast';
+import { isColumnItem } from '@kbn/esql-validation-autocomplete';
+
+export const getMvExpandFields = (query: string): string[] => {
+  const { root } = parse(query);
+
+  const mvExpandCommands = root.commands.filter((command) => command.name === 'mv_expand');
+
+  return mvExpandCommands.reduce<string[]>((acc, command) => {
+    const argument = command.args[0];
+
+    if (isColumnItem(argument) && argument.name) {
+      acc.push(argument.name);
+    }
+
+    return acc;
+  }, []);
+};

x-pack/solutions/security/packages/kbn-securitysolution-utils/src/esql/index.ts

@@ -8,3 +8,4 @@
 export * from './compute_if_esql_query_aggregating';
 export * from './get_index_list_from_esql_query';
 export * from './parse_esql_query';
+export * from './get_mv_expand_fields';

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts

@@ -12,6 +12,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type * as estypes from '@elastic/elasticsearch/lib/api/types';
+import { cloneDeep } from 'lodash';
 
 import {
   computeIsESQLQueryAggregating,
@@ -24,7 +25,7 @@ import { wrapEsqlAlerts } from './wrap_esql_alerts';
 import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts';
 import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory';
 import { createEnrichEventsFunction } from '../utils/enrichments';
-import { rowToDocument, mergeEsqlResultInSource } from './utils';
+import { rowToDocument, mergeEsqlResultInSource, getMvExpandUsage } from './utils';
 import { fetchSourceDocuments } from './fetch_source_documents';
 import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters';
 import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
@@ -147,7 +148,6 @@ export const esqlExecutor = async ({
           // slicing already processed results in previous iterations
           .slice(size - tuple.maxSignals)
           .map((row) => rowToDocument(response.columns, row));
-
         const index = getIndexListFromEsqlQuery(completeRule.ruleParams.query);
 
         const sourceDocuments = await fetchSourceDocuments({
@@ -163,6 +163,11 @@ export const esqlExecutor = async ({
           licensing,
         });
 
+        const { expandedFieldsInResponse: expandedFields, hasMvExpand } = getMvExpandUsage(
+          response.columns,
+          completeRule.ruleParams.query
+        );
+
         const wrapHits = (events: Array<estypes.SearchHit<SignalSource>>) =>
           wrapEsqlAlerts({
             events,
@@ -175,14 +180,18 @@ export const esqlExecutor = async ({
             publicBaseUrl,
             tuple,
             intendedTimestamp,
+            expandedFields,
           });
 
         const syntheticHits: Array<estypes.SearchHit<SignalSource>> = results.map((document) => {
           const { _id, _version, _index, ...esqlResult } = document;
 
           const sourceDocument = _id ? sourceDocuments[_id] : undefined;
+          // when mv_expand command present we must clone source, since the reference will be used multiple times
+          const source = hasMvExpand ? cloneDeep(sourceDocument?._source) : sourceDocument?._source;
+
           return {
-            _source: mergeEsqlResultInSource(sourceDocument?._source, esqlResult),
+            _source: mergeEsqlResultInSource(source, esqlResult),
             fields: sourceDocument?.fields,
             _id: _id ?? '',
             _index: _index || sourceDocument?._index || '',
@@ -205,6 +214,7 @@ export const esqlExecutor = async ({
               secondaryTimestamp,
               tuple,
               intendedTimestamp,
+              expandedFields,
             });
 
           const bulkCreateResult = await bulkCreateSuppressedAlertsInMemory({

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts

@@ -16,6 +16,9 @@ const mockEvent: estypes.SearchHit<SignalSource> = {
   _id: 'test_id',
   _version: 2,
   _index: 'test_index',
+  _source: {
+    'agent.name': 'test-0',
+  },
 };
 
 const mockRule = {
@@ -142,5 +145,21 @@ describe('generateAlertId', () => {
       modifiedIdParams.completeRule.ruleParams.query = 'from packetbeat*';
       expect(id).toBe(generateAlertId(modifiedIdParams));
     });
+
+    it('creates id dependant on expandedFields fields in source event', () => {
+      modifiedIdParams.expandedFields = ['agent.name'];
+      expect(id).not.toBe(generateAlertId(modifiedIdParams));
+    });
+
+    it('creates id not dependant on expandedFields fields, if they are not in event source', () => {
+      modifiedIdParams.expandedFields = ['agent.type'];
+      expect(id).toBe(generateAlertId(modifiedIdParams));
+    });
+
+    // when expanded fields empty, it means expanded field was dropped from ES|QL response, so we need to hash the whole event source object to properly deduplicate alerts
+    it('creates id not dependant on empty expandedFields fields', () => {
+      modifiedIdParams.expandedFields = [];
+      expect(id).not.toBe(generateAlertId(modifiedIdParams));
+    });
   });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts

@@ -9,8 +9,10 @@ import objectHash from 'object-hash';
 import type { Moment } from 'moment';
 import type * as estypes from '@elastic/elasticsearch/lib/api/types';
 
+import { robustGet } from '../../utils/source_fields_merging/utils/robust_field_access';
 import type { CompleteRule, EsqlRuleParams } from '../../../rule_schema';
 import type { SignalSource } from '../../types';
+
 /**
  * Generates id for ES|QL alert.
  * Id is generated as hash of event properties and rule/space config identifiers.
@@ -23,6 +25,7 @@ export const generateAlertId = ({
   tuple,
   isRuleAggregating,
   index,
+  expandedFields,
 }: {
   isRuleAggregating: boolean;
   event: estypes.SearchHit<SignalSource>;
@@ -34,15 +37,49 @@ export const generateAlertId = ({
     maxSignals: number;
   };
   index: number;
+  expandedFields?: string[];
 }) => {
   const ruleRunId = tuple.from.toISOString() + tuple.to.toISOString();
 
-  return !isRuleAggregating && event._id
-    ? objectHash([event._id, event._version, event._index, `${spaceId}:${completeRule.alertId}`])
-    : objectHash([
-        ruleRunId,
-        completeRule.ruleParams.query,
-        `${spaceId}:${completeRule.alertId}`,
-        index,
-      ]);
+  if (!isRuleAggregating && event._id) {
+    const idFields = [
+      event._id,
+      event._version,
+      event._index,
+      `${spaceId}:${completeRule.alertId}`,
+      ...retrieveExpandedValues({
+        event,
+        fields: expandedFields,
+      }),
+    ];
+    return objectHash(idFields);
+  } else {
+    return objectHash([
+      ruleRunId,
+      completeRule.ruleParams.query,
+      `${spaceId}:${completeRule.alertId}`,
+      index,
+    ]);
+  }
+};
+
+/**
+ * returns array of values from source event for requested list of fields
+ * undefined values are dropped
+ */
+const retrieveExpandedValues = ({
+  event,
+  fields,
+}: {
+  event: estypes.SearchHit<SignalSource>;
+  fields?: string[];
+}) => {
+  if (!fields || !event._source) {
+    return [];
+  }
+
+  const values = fields.map((field) =>
+    event._source ? robustGet({ key: field, document: event._source }) : undefined
+  );
+  return fields.length === 0 ? [event] : values.filter(Boolean);
 };

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/get_mv_expand_usage.test.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getMvExpandUsage } from './get_mv_expand_usage';
+
+describe('getMvExpandUsage', () => {
+  it('returns hasMvExpand false if mv_expand not present', () => {
+    expect(getMvExpandUsage([], 'from auditbeat*')).toEqual({
+      hasMvExpand: false,
+    });
+  });
+
+  it('returns hasMvExpand true if mv_expand present', () => {
+    expect(getMvExpandUsage([], 'from auditbeat* | mv_expand agent.name')).toHaveProperty(
+      'hasMvExpand',
+      true
+    );
+  });
+
+  it('returns all expended fields if they present in response columns', () => {
+    const columns = [
+      { name: 'agent.name', type: 'keyword' as const },
+      { name: 'host.name', type: 'keyword' as const },
+    ];
+    expect(
+      getMvExpandUsage(columns, 'from auditbeat* | mv_expand agent.name |  mv_expand host.name')
+    ).toHaveProperty('expandedFieldsInResponse', ['agent.name', 'host.name']);
+  });
+
+  it('returns empty expended fields if at least one is missing in response columns', () => {
+    const columns = [{ name: 'agent.name', type: 'keyword' as const }];
+    expect(
+      getMvExpandUsage(columns, 'from auditbeat* | mv_expand agent.name |  mv_expand host.name')
+    ).toHaveProperty('expandedFieldsInResponse', []);
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/get_mv_expand_usage.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getMvExpandFields } from '@kbn/securitysolution-utils';
+import type { EsqlResultColumn } from '../esql_request';
+
+export const getMvExpandUsage = (columns: EsqlResultColumn[], query: string) => {
+  const expandedFieldsFromQuery = getMvExpandFields(query);
+  if (expandedFieldsFromQuery.length === 0) {
+    return {
+      hasMvExpand: false,
+    };
+  }
+
+  const columnNamesSet = columns.reduce<Set<string>>((acc, column) => {
+    acc.add(column.name);
+    return acc;
+  }, new Set());
+  const hasExpandedFieldsMissed = expandedFieldsFromQuery.some(
+    (field) => !columnNamesSet.has(field)
+  );
+  const expandedFieldsInResponse = hasExpandedFieldsMissed ? [] : expandedFieldsFromQuery;
+
+  return {
+    hasMvExpand: expandedFieldsFromQuery.length > 0,
+    expandedFieldsInResponse,
+  };
+};

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts

@@ -8,3 +8,4 @@
 export * from './row_to_document';
 export * from './generate_alert_id';
 export * from './merge_esql_result_in_source';
+export * from './get_mv_expand_usage';

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts

@@ -93,4 +93,28 @@ describe('wrapSuppressedEsqlAlerts', () => {
       })
     );
   });
+
+  test('should filter our events with identical ids', () => {
+    const doc1 = sampleDocNoSortIdWithTimestamp(docId);
+    const doc2 = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d72');
+    const doc3 = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d73');
+    const alerts = wrapEsqlAlerts({
+      events: [doc1, doc1, doc2, doc2, doc3],
+      isRuleAggregating: false,
+      spaceId: 'default',
+      mergeStrategy: 'missingFields',
+      completeRule,
+      alertTimestampOverride: undefined,
+      ruleExecutionLogger,
+      publicBaseUrl,
+      tuple: {
+        to: moment('2010-10-20 04:43:12'),
+        from: moment('2010-10-20 04:43:12'),
+        maxSignals: 100,
+      },
+      intendedTimestamp: undefined,
+    });
+
+    expect(alerts).toHaveLength(3);
+  });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts

@@ -7,6 +7,7 @@
 
 import type { Moment } from 'moment';
 import type * as estypes from '@elastic/elasticsearch/lib/api/types';
+import { uniqBy } from 'lodash';
 
 import type {
   BaseFieldsLatest,
@@ -31,6 +32,7 @@ export const wrapEsqlAlerts = ({
   tuple,
   isRuleAggregating,
   intendedTimestamp,
+  expandedFields,
 }: {
   isRuleAggregating: boolean;
   events: Array<estypes.SearchHit<SignalSource>>;
@@ -46,6 +48,7 @@ export const wrapEsqlAlerts = ({
     maxSignals: number;
   };
   intendedTimestamp: Date | undefined;
+  expandedFields?: string[];
 }): Array<WrappedFieldsLatest<BaseFieldsLatest>> => {
   const wrapped = events.map<WrappedFieldsLatest<BaseFieldsLatest>>((event, i) => {
     const id = generateAlertId({
@@ -55,6 +58,7 @@ export const wrapEsqlAlerts = ({
       tuple,
       isRuleAggregating,
       index: i,
+      expandedFields,
     });
 
     const baseAlert: BaseFieldsLatest = transformHitToAlert({
@@ -77,11 +81,9 @@ export const wrapEsqlAlerts = ({
     return {
       _id: id,
       _index: event._index ?? '',
-      _source: {
-        ...baseAlert,
-      },
+      _source: baseAlert,
     };
   });
 
-  return wrapped;
+  return uniqBy(wrapped, (alert) => alert._id);
 };

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts

@@ -151,4 +151,28 @@ describe('wrapSuppressedEsqlAlerts', () => {
       })
     );
   });
+
+  test('should filter our events with identical ids', () => {
+    const doc1 = sampleDocNoSortIdWithTimestamp(docId);
+    const doc2 = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d72');
+    const alerts = wrapSuppressedEsqlAlerts({
+      events: [doc1, doc1, doc2],
+      isRuleAggregating: false,
+      spaceId: 'default',
+      mergeStrategy: 'missingFields',
+      completeRule,
+      alertTimestampOverride: undefined,
+      ruleExecutionLogger,
+      publicBaseUrl,
+      primaryTimestamp: '@timestamp',
+      tuple: {
+        to: moment('2010-10-20 04:43:12'),
+        from: moment('2010-10-20 04:43:12'),
+        maxSignals: 100,
+      },
+      intendedTimestamp: undefined,
+    });
+
+    expect(alerts).toHaveLength(2);
+  });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { uniqBy } from 'lodash';
 import objectHash from 'object-hash';
 import type { Moment } from 'moment';
 import type * as estypes from '@elastic/elasticsearch/lib/api/types';
@@ -37,6 +38,7 @@ export const wrapSuppressedEsqlAlerts = ({
   primaryTimestamp,
   secondaryTimestamp,
   intendedTimestamp,
+  expandedFields,
 }: {
   isRuleAggregating: boolean;
   events: Array<estypes.SearchHit<SignalSource>>;
@@ -54,6 +56,7 @@ export const wrapSuppressedEsqlAlerts = ({
   primaryTimestamp: string;
   secondaryTimestamp?: string;
   intendedTimestamp: Date | undefined;
+  expandedFields?: string[];
 }): Array<WrappedFieldsLatest<BaseFieldsLatest & SuppressionFieldsLatest>> => {
   const wrapped = events.map<WrappedFieldsLatest<BaseFieldsLatest & SuppressionFieldsLatest>>(
     (event, i) => {
@@ -71,6 +74,7 @@ export const wrapSuppressedEsqlAlerts = ({
         tuple,
         isRuleAggregating,
         index: i,
+        expandedFields,
       });
 
       const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]);
@@ -110,5 +114,5 @@ export const wrapSuppressedEsqlAlerts = ({
     }
   );
 
-  return wrapped;
+  return uniqBy(wrapped, (alert) => alert._id);
 };

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql.ts

@@ -409,6 +409,275 @@ export default ({ getService }: FtrProviderContext) => {
         expect(previewAlerts[0]._source).not.toHaveProperty(['agent.type']);
         expect(previewAlerts[0]._source).not.toHaveProperty('agent.type');
       });
+
+      describe('mv_expand command', () => {
+        it('should generate alert per expanded row', async () => {
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const documents = [
+            { agent: { name: 'test-1', type: 'auditbeat' } },
+            { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+          ];
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | mv_expand agent.name`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocuments({
+            documents,
+            interval,
+            id,
+          });
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+          expect(previewAlerts.length).toBe(3);
+          expect(previewAlerts.map((_) => _._source?.['agent.name'])).toEqual(
+            expect.arrayContaining(['part-0', 'part-1', 'test-1'])
+          );
+        });
+
+        it('should generate alert per expanded row when expanded field renamed', async () => {
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const documents = [
+            { agent: { name: 'test-1', type: 'auditbeat' } },
+            { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+          ];
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(
+              id
+            )} | mv_expand agent.name | rename agent.name as new_field`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocuments({
+            documents,
+            interval,
+            id,
+          });
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+          expect(previewAlerts.length).toBe(3);
+        });
+
+        //  When expanded field dropped, ES|QL response rows will be identical.
+        //  In this case, identical duplicated alerts won't be created
+        it('should NOT generate alert per expanded row when expanded field dropped', async () => {
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const documents = [
+            { agent: { name: 'test-1', type: 'auditbeat' } },
+            { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+          ];
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(
+              id
+            )} | mv_expand agent.name | drop agent.name`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocuments({
+            documents,
+            interval,
+            id,
+          });
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+          expect(previewAlerts.length).toBe(2);
+        });
+
+        it('should generate alert per expanded row when mv_expand used multiple times', async () => {
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const documents = [
+            { agent: { name: 'test-1', type: 'auditbeat' } },
+            { agent: { name: 'test-1', type: 'auditbeat' }, 'host.name': ['host-0', 'host-1'] },
+            {
+              agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+              'host.name': ['host-2', 'host-3'],
+            },
+          ];
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(
+              id
+            )} | mv_expand agent.name | mv_expand host.name`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocuments({
+            documents,
+            interval,
+            id,
+          });
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+          expect(previewAlerts.length).toBe(7);
+          expect(previewAlerts.map((_) => _._source?.['agent.name'])).toEqual(
+            expect.arrayContaining(['part-0', 'part-1', 'test-1'])
+          );
+          expect(previewAlerts.map((_) => _._source?.['host.name'])).toEqual(
+            expect.arrayContaining([undefined, 'host-0', 'host-1', 'host-2', 'host-3'])
+          );
+        });
+
+        it('should deduplicate alerts generated from expanded rows', async () => {
+          const id = uuidv4();
+          // document will fall into 2 rule execution windows
+          const doc1 = {
+            id,
+            '@timestamp': '2020-10-28T05:55:00.000Z',
+            agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | mv_expand agent.name`,
+            from: 'now-45m',
+            interval: '30m',
+          };
+
+          await indexListOfDocuments([doc1]);
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+            invocationCount: 2,
+          });
+
+          const previewAlerts = await getPreviewAlerts({
+            es,
+            previewId,
+            size: 10,
+          });
+
+          expect(previewAlerts.length).toBe(2);
+        });
+
+        it('should deduplicate alerts generated form expanded rows when expanded field renamed', async () => {
+          const id = uuidv4();
+          // document will fall into 2 rule execution windows
+          const doc1 = {
+            id,
+            '@timestamp': '2020-10-28T05:55:00.000Z',
+            agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(
+              id
+            )} | mv_expand agent.name | rename agent.name as new_field`,
+            from: 'now-45m',
+            interval: '30m',
+          };
+
+          await indexListOfDocuments([doc1]);
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+            invocationCount: 2,
+          });
+
+          const previewAlerts = await getPreviewAlerts({
+            es,
+            previewId,
+            size: 10,
+          });
+
+          expect(previewAlerts.length).toBe(2);
+        });
+
+        it('should deduplicate alert when expanded field dropped', async () => {
+          const id = uuidv4();
+          // document will fall into 2 rule execution windows
+          const doc1 = {
+            id,
+            '@timestamp': '2020-10-28T05:55:00.000Z',
+            agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_compliant metadata _id ${internalIdPipe(
+              id
+            )} | mv_expand agent.name | drop agent.name`,
+            from: 'now-45m',
+            interval: '30m',
+          };
+
+          await indexListOfDocuments([doc1]);
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+            invocationCount: 2,
+          });
+
+          const previewAlerts = await getPreviewAlerts({
+            es,
+            previewId,
+            size: 10,
+          });
+
+          expect(previewAlerts.length).toBe(1);
+        });
+      });
     });
 
     describe('aggregating query rules', () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_suppression.ts

@@ -2105,5 +2105,221 @@ export default ({ getService }: FtrProviderContext) => {
         expect(previewAlerts[0]?._source?.['host.asset.criticality']).toBe('extreme_impact');
       });
     });
+
+    describe('mv_expand command', () => {
+      it('should suppress alerts generated from expanded rows', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const documents = [
+          { agent: { name: 'test-1', type: 'auditbeat' } },
+          { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+        ];
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | mv_expand agent.name`,
+          from: 'now-1h',
+          interval: '1h',
+          alert_suppression: {
+            group_by: ['agent.type'],
+            missing_fields_strategy: 'suppress',
+          },
+        };
+
+        await indexEnhancedDocuments({
+          documents,
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty([ALERT_SUPPRESSION_DOCS_COUNT], 2);
+      });
+
+      it('should suppress alerts generated from expanded rows when expanded field renamed', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const documents = [
+          { agent: { name: 'test-1', type: 'auditbeat' } },
+          { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+        ];
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
+            id
+          )} | mv_expand agent.name | rename agent.name as new_field`,
+          from: 'now-1h',
+          interval: '1h',
+          alert_suppression: {
+            group_by: ['agent.type'],
+            missing_fields_strategy: 'suppress',
+          },
+        };
+
+        await indexEnhancedDocuments({
+          documents,
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty([ALERT_SUPPRESSION_DOCS_COUNT], 2);
+      });
+
+      it('should NOT generate alerts per expanded row when expanded field dropped', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const documents = [
+          { agent: { name: 'test-1', type: 'auditbeat' } },
+          { agent: { name: ['part-0', 'part-1'], type: 'auditbeat' } },
+        ];
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
+            id
+          )} | mv_expand agent.name | drop agent.name`,
+          from: 'now-1h',
+          interval: '1h',
+          alert_suppression: {
+            group_by: ['agent.type'],
+            missing_fields_strategy: 'suppress',
+          },
+        };
+
+        await indexEnhancedDocuments({
+          documents,
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty([ALERT_SUPPRESSION_DOCS_COUNT], 1);
+      });
+
+      it('should suppress alerts from expanded rows on interval', async () => {
+        const id = uuidv4();
+        const doc1 = {
+          id,
+          '@timestamp': '2020-10-28T05:45:00.000Z',
+          agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+        };
+
+        const doc2 = {
+          id,
+          '@timestamp': '2020-10-28T06:25:00.000Z',
+          agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+        };
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          // only _id and agent.name is projected at the end of query pipeline
+          query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | mv_expand agent.name`,
+          from: 'now-35m',
+          interval: '30m',
+          alert_suppression: {
+            group_by: ['agent.type'],
+            missing_fields_strategy: 'suppress',
+            duration: {
+              value: 300,
+              unit: 'm',
+            },
+          },
+        };
+
+        await indexListOfDocuments([doc1, doc2]);
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          invocationCount: 2,
+        });
+
+        const previewAlerts = await getPreviewAlerts({
+          es,
+          previewId,
+          size: 10,
+        });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty([ALERT_SUPPRESSION_DOCS_COUNT], 3);
+      });
+
+      it('should suppress alerts on interval when expanded field renamed', async () => {
+        const id = uuidv4();
+        const doc1 = {
+          id,
+          '@timestamp': '2020-10-28T05:45:00.000Z',
+          agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+        };
+
+        const doc2 = {
+          id,
+          '@timestamp': '2020-10-28T06:25:00.000Z',
+          agent: { name: ['part-0', 'part-1'], type: 'auditbeat' },
+        };
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          // only _id and agent.name is projected at the end of query pipeline
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
+            id
+          )} | mv_expand agent.name  | rename agent.name as new_field`,
+          from: 'now-35m',
+          interval: '30m',
+          alert_suppression: {
+            group_by: ['agent.type'],
+            missing_fields_strategy: 'suppress',
+            duration: {
+              value: 300,
+              unit: 'm',
+            },
+          },
+        };
+
+        await indexListOfDocuments([doc1, doc2]);
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          invocationCount: 2,
+        });
+
+        const previewAlerts = await getPreviewAlerts({
+          es,
+          previewId,
+          size: 10,
+        });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty([ALERT_SUPPRESSION_DOCS_COUNT], 3);
+      });
+    });
   });
 };