[8.15] [Security Solution][Detections][BUG] ES|QL rule execution error when source document has a non-ECS compliant sub-field with data under event field (#187384) (#187549) (#187661)

# Backport This will backport the following commits from `main` to `8.15`: - [[Security Solution][Detections][BUG] ES|QL rule execution error when source document has a non-ECS compliant sub-field with data under event field (#187384) (#187549)](https://github.com/elastic/kibana/pull/187549)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Ievgen Sorokopud <ievgen.sorokopud@elastic.co> Co-authored-by: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com>
2025-04-23 09:19:04 -04:00 · 2024-07-08 11:30:23 +02:00 · 2024-07-08 11:30:23 +02:00 · 2004c4c933
commit 2004c4c933
parent 7e0626c9de
5 changed files with 112 additions and 1 deletions
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/mocks/es_results.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/mocks/es_results.ts
@ -359,6 +359,24 @@ export const sampleDocNoSortIdWithTimestamp = (
  };
 };

+export const sampleDocWithNonEcsCompliantFields = (
+  someUuid: string = sampleIdGuid,
+  nonEcsFields: Record<string, string>
+): SignalSourceHit & {
+  _id: Required<SignalSourceHit>['_id'];
+  _source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
+} => {
+  const doc = sampleDocNoSortId(someUuid);
+  return {
+    ...doc,
+    _source: {
+      ...doc._source,
+      ...nonEcsFields,
+      '@timestamp': new Date().toISOString(),
+    },
+  };
+};
+
 export const sampleAlertDocNoSortIdWithTimestamp = (
  someUuid: string = sampleIdGuid,
  ip?: string
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts
@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { sampleDocWithNonEcsCompliantFields } from '../../__mocks__/es_results';
+import { buildBulkBody } from './build_bulk_body';
+import { getCompleteRuleMock, getEsqlRuleParams } from '../../../rule_schema/mocks';
+import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks';
+
+const SPACE_ID = 'space';
+const publicBaseUrl = 'testKibanaBasePath.com';
+const alertUuid = 'test-uuid';
+const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71';
+const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
+
+describe('buildBulkBody', () => {
+  test('should strip non-ECS compliant sub-fields of `event.action` field', () => {
+    const doc = sampleDocWithNonEcsCompliantFields(docId, {
+      'event.action': 'process',
+      'event.action.keyword': 'process',
+    });
+    const completeRule = getCompleteRuleMock(getEsqlRuleParams());
+    const buildReasonMessageStub = jest.fn();
+    const alert = buildBulkBody(
+      SPACE_ID,
+      completeRule,
+      doc,
+      'missingFields',
+      [],
+      true,
+      buildReasonMessageStub,
+      [],
+      undefined,
+      ruleExecutionLogger,
+      alertUuid,
+      publicBaseUrl
+    );
+
+    expect(alert['kibana.alert.original_event.action']).toEqual('process');
+    expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined();
+  });
+});
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts
@ -104,6 +104,7 @@ export const buildBulkBody = (
    mergedDoc,
  });

+  const thresholdResult = mergedDoc._source?.threshold_result;
  if (isSourceDoc(mergedDoc)) {
    return {
      ...validatedSource,
@ -121,7 +122,11 @@ export const buildBulkBody = (
      ),
      ...additionalAlertFields({
        ...mergedDoc,
-        _source: { ...mergedDoc._source, ...validatedEventFields },
+        _source: {
+          ...validatedSource,
+          ...validatedEventFields,
+          threshold_result: thresholdResult,
+        },
      }),
    };
  }
--- a/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json
+++ b/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json
@ -42,6 +42,15 @@
            "properties": {
              "created": {
                "type": "keyword"
+              },
+              "action": {
+                "type": "text",
+                "fields": {
+                  "keyword": {
+                    "type": "keyword",
+                    "ignore_above": 256
+                  }
+                }
              }
            }
          },
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts
@ -1103,6 +1103,40 @@ export default ({ getService }: FtrProviderContext) => {
            'random non-ecs field'
          );
        });
+
+        it('creates alert if `event.action` ECS field has non-ECS sub-field', async () => {
+          // The issue was found by customer and reported in
+          // https://github.com/elastic/sdh-security-team/issues/1015
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const doc1 = {
+            'event.action': 'process',
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_non_compliant metadata _id ${internalIdPipe(id)}`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocumentsToNonEcs({
+            documents: [doc1],
+            interval,
+            id,
+          });
+
+          const { logs } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          expect(logs[0].errors.length).toEqual(0);
+        });
      });
    });
  });