[Security Solution][RAC] Flatten alert fields (#107581)

* incremental changes

* No more type errors

* Type guards

* Begin adding tests

* Flatten

* Reduce scope of branch

* Remove extraneous argument to filter_duplicate_signals

packages/kbn-rule-data-utils/src/technical_field_names.ts

@@ -74,6 +74,12 @@ const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const;
 const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const;
 const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const;
 
+const namespaces = {
+  KIBANA_NAMESPACE,
+  ALERT_NAMESPACE,
+  ALERT_RULE_NAMESPACE,
+};
+
 const fields = {
   CONSUMERS,
   ECS_VERSION,
@@ -142,6 +148,8 @@ export {
   ALERT_EVALUATION_THRESHOLD,
   ALERT_EVALUATION_VALUE,
   ALERT_ID,
+  ALERT_NAMESPACE,
+  ALERT_RULE_NAMESPACE,
   ALERT_OWNER,
   ALERT_PRODUCER,
   ALERT_REASON,
@@ -185,6 +193,7 @@ export {
   ECS_VERSION,
   EVENT_ACTION,
   EVENT_KIND,
+  KIBANA_NAMESPACE,
   RULE_CATEGORY,
   RULE_CONSUMERS,
   RULE_ID,
@@ -196,4 +205,4 @@ export {
   VERSION,
 };
 
-export type TechnicalRuleDataFieldName = ValuesType<typeof fields>;
+export type TechnicalRuleDataFieldName = ValuesType<typeof fields & typeof namespaces>;

x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.test.tsx

@@ -11,10 +11,12 @@ import {
   ALERT_EVALUATION_VALUE,
   ALERT_ID,
   ALERT_PRODUCER,
+  ALERT_OWNER,
   ALERT_SEVERITY_LEVEL,
   ALERT_START,
   ALERT_STATUS,
   ALERT_UUID,
+  SPACE_IDS,
 } from '@kbn/rule-data-utils';
 import { ValuesType } from 'utility-types';
 import { EuiTheme } from '../../../../../../../../src/plugins/kibana_react/common';
@@ -32,6 +34,7 @@ const theme = ({
   eui: { euiColorDanger, euiColorWarning },
 } as unknown) as EuiTheme;
 const alert: Alert = {
+  [SPACE_IDS]: ['space-id'],
   'rule.id': ['apm.transaction_duration'],
   [ALERT_EVALUATION_VALUE]: [2057657.39],
   'service.name': ['frontend-rum'],
@@ -42,6 +45,7 @@ const alert: Alert = {
   'transaction.type': ['page-load'],
   [ALERT_PRODUCER]: ['apm'],
   [ALERT_UUID]: ['af2ae371-df79-4fca-b0eb-a2dbd9478180'],
+  [ALERT_OWNER]: ['apm'],
   'rule.uuid': ['82e0ee40-c2f4-11eb-9a42-a9da66a1722f'],
   'event.action': ['active'],
   '@timestamp': ['2021-06-01T16:16:05.183Z'],

x-pack/plugins/rule_registry/README.md

@@ -124,7 +124,6 @@ The following fields are defined in the technical field component template and s
 - `rule.uuid`: the saved objects id of the rule.
 - `rule.name`: the name of the rule (as specified by the user).
 - `rule.category`: the name of the rule type (as defined by the rule type producer)
-- `kibana.alert.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`.
 - `kibana.alert.owner`: the feature which produced the alert. Usually a Kibana feature id like `apm`, `siem`...
 - `kibana.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`.
 - `kibana.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again.

x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts

@@ -2148,7 +2148,7 @@ export const ecsFieldMap = {
   'rule.id': {
     type: 'keyword',
     array: false,
-    required: false,
+    required: true,
   },
   'rule.license': {
     type: 'keyword',

x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts

@@ -20,9 +20,9 @@ export const technicalRuleFieldMap = {
     Fields.RULE_CATEGORY,
     Fields.TAGS
   ),
-  [Fields.ALERT_OWNER]: { type: 'keyword' },
+  [Fields.ALERT_OWNER]: { type: 'keyword', required: true },
   [Fields.ALERT_PRODUCER]: { type: 'keyword' },
-  [Fields.SPACE_IDS]: { type: 'keyword', array: true },
+  [Fields.SPACE_IDS]: { type: 'keyword', array: true, required: true },
   [Fields.ALERT_UUID]: { type: 'keyword' },
   [Fields.ALERT_ID]: { type: 'keyword' },
   [Fields.ALERT_START]: { type: 'date' },

x-pack/plugins/rule_registry/server/routes/get_alert_by_id.test.ts

@@ -13,6 +13,7 @@ import {
   CONSUMERS,
   ECS_VERSION,
   RULE_ID,
+  SPACE_IDS,
   TIMESTAMP,
   VERSION,
 } from '@kbn/rule-data-utils';
@@ -33,6 +34,7 @@ const getMockAlert = (): ParsedTechnicalFields => ({
   [ALERT_OWNER]: 'apm',
   [ALERT_STATUS]: 'open',
   [ALERT_RULE_RISK_SCORE]: 20,
+  [SPACE_IDS]: ['fake-space-id'],
   [ALERT_RULE_SEVERITY]: 'warning',
 });
 

x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts

@@ -23,6 +23,9 @@ import {
   ALERT_STATUS,
   EVENT_ACTION,
   EVENT_KIND,
+  RULE_ID,
+  ALERT_OWNER,
+  SPACE_IDS,
 } from '../../common/technical_rule_data_field_names';
 import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock';
 import { createLifecycleExecutor } from './create_lifecycle_executor';
@@ -128,12 +131,16 @@ describe('createLifecycleExecutor', () => {
           {
             fields: {
               [ALERT_ID]: 'TEST_ALERT_0',
+              [ALERT_OWNER]: 'CONSUMER',
+              [RULE_ID]: 'RULE_TYPE_ID',
               labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
             },
           },
           {
             fields: {
               [ALERT_ID]: 'TEST_ALERT_1',
+              [ALERT_OWNER]: 'CONSUMER',
+              [RULE_ID]: 'RULE_TYPE_ID',
               labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
             },
           },
@@ -222,6 +229,9 @@ describe('createLifecycleExecutor', () => {
             fields: {
               '@timestamp': '',
               [ALERT_ID]: 'TEST_ALERT_0',
+              [ALERT_OWNER]: 'CONSUMER',
+              [RULE_ID]: 'RULE_TYPE_ID',
+              [SPACE_IDS]: ['fake-space-id'],
               labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc
             },
           },
@@ -229,6 +239,9 @@ describe('createLifecycleExecutor', () => {
             fields: {
               '@timestamp': '',
               [ALERT_ID]: 'TEST_ALERT_1',
+              [ALERT_OWNER]: 'CONSUMER',
+              [RULE_ID]: 'RULE_TYPE_ID',
+              [SPACE_IDS]: ['fake-space-id'],
               labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
             },
           },

x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts

@@ -30,6 +30,7 @@ import {
   EVENT_ACTION,
   EVENT_KIND,
   ALERT_OWNER,
+  RULE_ID,
   RULE_UUID,
   TIMESTAMP,
   SPACE_IDS,
@@ -154,6 +155,8 @@ export const createLifecycleExecutor = (
       currentAlerts[id] = {
         ...fields,
         [ALERT_ID]: id,
+        [RULE_ID]: rule.ruleTypeId,
+        [ALERT_OWNER]: rule.consumer,
       };
       return alertInstanceFactory(id);
     },
@@ -226,6 +229,8 @@ export const createLifecycleExecutor = (
       alertsDataMap[alertId] = {
         ...fields,
         [ALERT_ID]: alertId,
+        [RULE_ID]: rule.ruleTypeId,
+        [ALERT_OWNER]: rule.consumer,
       };
     });
   }

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts

@@ -6,7 +6,14 @@
  */
 
 import { estypes } from '@elastic/elasticsearch';
-import { EVENT_ACTION, EVENT_KIND, RULE_ID, SPACE_IDS, TIMESTAMP } from '@kbn/rule-data-utils';
+import {
+  ALERT_OWNER,
+  EVENT_ACTION,
+  EVENT_KIND,
+  RULE_ID,
+  SPACE_IDS,
+  TIMESTAMP,
+} from '@kbn/rule-data-utils';
 import { once } from 'lodash/fp';
 import moment from 'moment';
 import { RuleDataClient } from '../../../../../../rule_registry/server';
@@ -221,6 +228,7 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient {
         [getMetricField(metric)]: value,
         [RULE_ID]: ruleId,
         [TIMESTAMP]: new Date().toISOString(),
+        [ALERT_OWNER]: 'siem',
       },
       namespace
     );
@@ -244,6 +252,7 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient {
         [RULE_STATUS_SEVERITY]: statusSeverityDict[newStatus],
         [RULE_STATUS]: newStatus,
         [TIMESTAMP]: new Date().toISOString(),
+        [ALERT_OWNER]: 'siem',
       },
       namespace
     );

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts

@@ -191,8 +191,10 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({
         );
 
         const wrapHits = wrapHitsFactory({
-          ruleSO,
+          logger,
           mergeStrategy,
+          ruleSO,
+          spaceId,
         });
 
         for (const tuple of tuples) {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts

@@ -0,0 +1,419 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ALERT_OWNER,
+  ALERT_RULE_NAMESPACE,
+  ALERT_STATUS,
+  ALERT_WORKFLOW_STATUS,
+  SPACE_IDS,
+} from '@kbn/rule-data-utils';
+
+import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results';
+import { flattenWithPrefix } from './flatten_with_prefix';
+import {
+  buildAlert,
+  buildParent,
+  buildAncestors,
+  additionalAlertFields,
+  removeClashes,
+} from './build_alert';
+import { Ancestor, SignalSourceHit } from '../../../signals/types';
+import {
+  getRulesSchemaMock,
+  ANCHOR_DATE,
+} from '../../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
+import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock';
+import {
+  ALERT_ANCESTORS,
+  ALERT_ORIGINAL_EVENT,
+  ALERT_ORIGINAL_TIME,
+} from '../../field_maps/field_names';
+import { SERVER_APP_ID } from '../../../../../../common/constants';
+
+type SignalDoc = SignalSourceHit & {
+  _source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
+};
+
+const SPACE_ID = 'space';
+
+describe('buildAlert', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('it builds an alert as expected without original_event if event does not exist', () => {
+    const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    delete doc._source.event;
+    const rule = getRulesSchemaMock();
+    const alert = {
+      ...buildAlert([doc], rule, SPACE_ID),
+      ...additionalAlertFields(doc),
+    };
+    const timestamp = alert['@timestamp'];
+    const expected = {
+      '@timestamp': timestamp,
+      [SPACE_IDS]: [SPACE_ID],
+      [ALERT_OWNER]: SERVER_APP_ID,
+      [ALERT_ANCESTORS]: [
+        {
+          id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z',
+      [ALERT_STATUS]: 'open',
+      [ALERT_WORKFLOW_STATUS]: 'open',
+      ...flattenWithPrefix(ALERT_RULE_NAMESPACE, {
+        author: [],
+        id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+        created_at: new Date(ANCHOR_DATE).toISOString(),
+        updated_at: new Date(ANCHOR_DATE).toISOString(),
+        created_by: 'elastic',
+        description: 'some description',
+        enabled: true,
+        false_positives: ['false positive 1', 'false positive 2'],
+        from: 'now-6m',
+        immutable: false,
+        name: 'Query with a rule id',
+        query: 'user.name: root or user.name: admin',
+        references: ['test 1', 'test 2'],
+        severity: 'high',
+        severity_mapping: [],
+        updated_by: 'elastic_kibana',
+        tags: ['some fake tag 1', 'some fake tag 2'],
+        to: 'now',
+        type: 'query',
+        threat: [],
+        version: 1,
+        status: 'succeeded',
+        status_date: '2020-02-22T16:47:50.047Z',
+        last_success_at: '2020-02-22T16:47:50.047Z',
+        last_success_message: 'succeeded',
+        output_index: '.siem-signals-default',
+        max_signals: 100,
+        risk_score: 55,
+        risk_score_mapping: [],
+        language: 'kuery',
+        rule_id: 'query-rule-id',
+        interval: '5m',
+        exceptions_list: getListArrayMock(),
+      }),
+      'kibana.alert.depth': 1,
+    };
+    expect(alert).toEqual(expected);
+  });
+
+  test('it builds an alert as expected with original_event if is present', () => {
+    const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    doc._source.event = {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    };
+    const rule = getRulesSchemaMock();
+    const alert = {
+      ...buildAlert([doc], rule, SPACE_ID),
+      ...additionalAlertFields(doc),
+    };
+    const timestamp = alert['@timestamp'];
+    const expected = {
+      '@timestamp': timestamp,
+      [SPACE_IDS]: [SPACE_ID],
+      [ALERT_OWNER]: SERVER_APP_ID,
+      [ALERT_ANCESTORS]: [
+        {
+          id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z',
+      [ALERT_ORIGINAL_EVENT]: {
+        action: 'socket_opened',
+        dataset: 'socket',
+        kind: 'event',
+        module: 'system',
+      },
+      [ALERT_STATUS]: 'open',
+      [ALERT_WORKFLOW_STATUS]: 'open',
+      ...flattenWithPrefix(ALERT_RULE_NAMESPACE, {
+        author: [],
+        id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+        created_at: new Date(ANCHOR_DATE).toISOString(),
+        updated_at: new Date(ANCHOR_DATE).toISOString(),
+        created_by: 'elastic',
+        description: 'some description',
+        enabled: true,
+        false_positives: ['false positive 1', 'false positive 2'],
+        from: 'now-6m',
+        immutable: false,
+        name: 'Query with a rule id',
+        query: 'user.name: root or user.name: admin',
+        references: ['test 1', 'test 2'],
+        severity: 'high',
+        severity_mapping: [],
+        updated_by: 'elastic_kibana',
+        tags: ['some fake tag 1', 'some fake tag 2'],
+        to: 'now',
+        type: 'query',
+        threat: [],
+        version: 1,
+        status: 'succeeded',
+        status_date: '2020-02-22T16:47:50.047Z',
+        last_success_at: '2020-02-22T16:47:50.047Z',
+        last_success_message: 'succeeded',
+        output_index: '.siem-signals-default',
+        max_signals: 100,
+        risk_score: 55,
+        risk_score_mapping: [],
+        language: 'kuery',
+        rule_id: 'query-rule-id',
+        interval: '5m',
+        exceptions_list: getListArrayMock(),
+      }),
+      'kibana.alert.depth': 1,
+    };
+    expect(alert).toEqual(expected);
+  });
+
+  test('it builds an ancestor correctly if the parent does not exist', () => {
+    const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    doc._source.event = {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    };
+    const parent = buildParent(doc);
+    const expected: Ancestor = {
+      id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+      type: 'event',
+      index: 'myFakeSignalIndex',
+      depth: 0,
+    };
+    expect(parent).toEqual(expected);
+  });
+
+  test('it builds an ancestor correctly if the parent does exist', () => {
+    const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    doc._source.event = {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    };
+    doc._source.signal = {
+      parents: [
+        {
+          id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      ancestors: [
+        {
+          id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      depth: 1,
+      rule: {
+        id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
+      },
+    };
+    const parent = buildParent(doc);
+    const expected: Ancestor = {
+      rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
+      id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+      type: 'signal',
+      index: 'myFakeSignalIndex',
+      depth: 1,
+    };
+    expect(parent).toEqual(expected);
+  });
+
+  test('it builds an alert ancestor correctly if the parent does not exist', () => {
+    const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    const doc: SignalDoc = {
+      ...sampleDoc,
+      _source: {
+        ...sampleDoc._source,
+        '@timestamp': new Date().toISOString(),
+      },
+    };
+    doc._source.event = {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    };
+    const ancestor = buildAncestors(doc);
+    const expected: Ancestor[] = [
+      {
+        id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+    ];
+    expect(ancestor).toEqual(expected);
+  });
+
+  test('it builds an alert ancestor correctly if the parent does exist', () => {
+    const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+    const doc: SignalDoc = {
+      ...sampleDoc,
+      _source: {
+        ...sampleDoc._source,
+        '@timestamp': new Date().toISOString(),
+      },
+    };
+    doc._source.event = {
+      action: 'socket_opened',
+      dataset: 'socket',
+      kind: 'event',
+      module: 'system',
+    };
+    doc._source.signal = {
+      parents: [
+        {
+          id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      ancestors: [
+        {
+          id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+      ],
+      rule: {
+        id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
+      },
+      depth: 1,
+    };
+    const ancestors = buildAncestors(doc);
+    const expected: Ancestor[] = [
+      {
+        id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
+        type: 'event',
+        index: 'myFakeSignalIndex',
+        depth: 0,
+      },
+      {
+        rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
+        id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
+        type: 'signal',
+        index: 'myFakeSignalIndex',
+        depth: 1,
+      },
+    ];
+    expect(ancestors).toEqual(expected);
+  });
+
+  describe('removeClashes', () => {
+    test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => {
+      const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+      const doc: SignalDoc = {
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          '@timestamp': new Date().toISOString(),
+        },
+      };
+      const output = removeClashes(doc);
+      expect(output).toBe(doc); // reference check
+    });
+
+    test('it will call renameClashes with a regular doc and not change anything', () => {
+      const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+      const doc: SignalDoc = {
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          '@timestamp': new Date().toISOString(),
+        },
+      };
+      const output = removeClashes(doc);
+      expect(output).toEqual(doc); // deep equal check
+    });
+
+    test('it will remove a "signal" numeric clash', () => {
+      const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+      const doc = ({
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          signal: 127,
+        },
+      } as unknown) as SignalDoc;
+      const output = removeClashes(doc);
+      const timestamp = output._source['@timestamp'];
+      expect(output).toEqual({
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          '@timestamp': timestamp,
+        },
+      });
+    });
+
+    test('it will remove a "signal" object clash', () => {
+      const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+      const doc = ({
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          signal: { child_1: { child_2: 'Test nesting' } },
+        },
+      } as unknown) as SignalDoc;
+      const output = removeClashes(doc);
+      const timestamp = output._source['@timestamp'];
+      expect(output).toEqual({
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          '@timestamp': timestamp,
+        },
+      });
+    });
+
+    test('it will not remove a "signal" if that is signal is one of our signals', () => {
+      const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
+      const doc = ({
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          signal: { rule: { id: '123' } },
+        },
+      } as unknown) as SignalDoc;
+      const output = removeClashes(doc);
+      const timestamp = output._source['@timestamp'];
+      const expected = {
+        ...sampleDoc,
+        _source: {
+          ...sampleDoc._source,
+          signal: { rule: { id: '123' } },
+          '@timestamp': timestamp,
+        },
+      };
+      expect(output).toEqual(expected);
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts

@@ -5,124 +5,113 @@
  * 2.0.
  */
 
-import { ALERT_STATUS, ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils';
-import { SearchTypes } from '../../../../../../common/detection_engine/types';
+import {
+  ALERT_OWNER,
+  ALERT_RULE_NAMESPACE,
+  ALERT_STATUS,
+  ALERT_WORKFLOW_STATUS,
+  SPACE_IDS,
+} from '@kbn/rule-data-utils';
 import { RulesSchema } from '../../../../../../common/detection_engine/schemas/response/rules_schema';
 import { isEventTypeSignal } from '../../../signals/build_event_type_signal';
+import { Ancestor, BaseSignalHit, SimpleHit } from '../../../signals/types';
 import {
-  Ancestor,
-  BaseSignalHit,
-  SignalHit,
-  SignalSourceHit,
-  ThresholdResult,
-} from '../../../signals/types';
-import { getValidDateFromDoc } from '../../../signals/utils';
+  getField,
+  getValidDateFromDoc,
+  isWrappedRACAlert,
+  isWrappedSignalHit,
+} from '../../../signals/utils';
 import { invariant } from '../../../../../../common/utils/invariant';
-import { DEFAULT_MAX_SIGNALS } from '../../../../../../common/constants';
+import { RACAlert } from '../../types';
+import { flattenWithPrefix } from './flatten_with_prefix';
+import {
+  ALERT_ANCESTORS,
+  ALERT_DEPTH,
+  ALERT_ORIGINAL_EVENT,
+  ALERT_ORIGINAL_TIME,
+} from '../../field_maps/field_names';
+import { SERVER_APP_ID } from '../../../../../../common/constants';
 
 /**
- * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child
- * signal's `signal.parents` array.
- * @param doc The parent signal or event
+ * Takes an event document and extracts the information needed for the corresponding entry in the child
+ * alert's ancestors array.
+ * @param doc The parent event
  */
-export const buildParent = (doc: BaseSignalHit): Ancestor => {
-  if (doc._source?.signal != null) {
-    return {
-      rule: doc._source?.signal.rule.id,
-      id: doc._id,
-      type: 'signal',
-      index: doc._index,
-      // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal
-      // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth.
-      depth: doc._source?.signal.depth ?? doc._source?.signal.parent?.depth ?? 1,
-    };
-  } else {
-    return {
-      id: doc._id,
-      type: 'event',
-      index: doc._index,
-      depth: 0,
-    };
+export const buildParent = (doc: SimpleHit): Ancestor => {
+  const isSignal: boolean = isWrappedSignalHit(doc) || isWrappedRACAlert(doc);
+  const parent: Ancestor = {
+    id: doc._id,
+    type: isSignal ? 'signal' : 'event',
+    index: doc._index,
+    depth: isSignal ? getField(doc, 'signal.depth') ?? 1 : 0,
+  };
+  if (isSignal) {
+    parent.rule = getField(doc, 'signal.rule.id');
   }
+  return parent;
 };
 
 /**
- * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array,
+ * Takes a parent event document with N ancestors and adds the parent document to the ancestry array,
  * creating an array of N+1 ancestors.
- * @param doc The parent signal/event for which to extend the ancestry.
+ * @param doc The parent event for which to extend the ancestry.
  */
-export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => {
+export const buildAncestors = (doc: SimpleHit): Ancestor[] => {
   const newAncestor = buildParent(doc);
-  const existingAncestors = doc._source?.signal?.ancestors;
-  if (existingAncestors != null) {
-    return [...existingAncestors, newAncestor];
-  } else {
-    return [newAncestor];
-  }
+  const existingAncestors: Ancestor[] = getField(doc, 'signal.ancestors') ?? [];
+  return [...existingAncestors, newAncestor];
 };
 
 /**
- * This removes any signal name clashes such as if a source index has
+ * This removes any alert name clashes such as if a source index has
  * "signal" but is not a signal object we put onto the object. If this
  * is our "signal object" then we don't want to remove it.
  * @param doc The source index doc to a signal.
  */
-export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => {
-  invariant(doc._source, '_source field not found');
-  const { signal, ...noSignal } = doc._source;
-  if (signal == null || isEventTypeSignal(doc)) {
-    return doc;
-  } else {
-    return {
-      ...doc,
-      _source: { ...noSignal },
-    };
+export const removeClashes = (doc: SimpleHit) => {
+  if (isWrappedSignalHit(doc)) {
+    invariant(doc._source, '_source field not found');
+    const { signal, ...noSignal } = doc._source;
+    if (signal == null || isEventTypeSignal(doc)) {
+      return doc;
+    } else {
+      return {
+        ...doc,
+        _source: { ...noSignal },
+      };
+    }
   }
+  return doc;
 };
 
 /**
- * Builds the `signal.*` fields that are common across all signals.
- * @param docs The parent signals/events of the new signal to be built.
- * @param rule The rule that is generating the new signal.
+ * Builds the `kibana.alert.*` fields that are common across all alerts.
+ * @param docs The parent alerts/events of the new alert to be built.
+ * @param rule The rule that is generating the new alert.
  */
-export const buildAlert = (doc: SignalSourceHit, rule: RulesSchema) => {
-  const removedClashes = removeClashes(doc);
-  const parent = buildParent(removedClashes);
-  const ancestors = buildAncestors(removedClashes);
-  const immutable = doc._source?.signal?.rule.immutable ? 'true' : 'false';
+export const buildAlert = (
+  docs: SimpleHit[],
+  rule: RulesSchema,
+  spaceId: string | null | undefined
+): RACAlert => {
+  const removedClashes = docs.map(removeClashes);
+  const parents = removedClashes.map(buildParent);
+  const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1;
+  const ancestors = removedClashes.reduce(
+    (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)),
+    []
+  );
 
-  const source = doc._source as SignalHit;
-  const signal = source?.signal;
-  const signalRule = signal?.rule;
-
-  return {
-    'kibana.alert.ancestors': ancestors as object[],
+  return ({
+    '@timestamp': new Date().toISOString(),
+    [ALERT_OWNER]: SERVER_APP_ID,
+    [SPACE_IDS]: spaceId != null ? [spaceId] : [],
+    [ALERT_ANCESTORS]: ancestors,
     [ALERT_STATUS]: 'open',
     [ALERT_WORKFLOW_STATUS]: 'open',
-    'kibana.alert.depth': parent.depth,
-    'kibana.alert.rule.false_positives': signalRule?.false_positives ?? [],
-    'kibana.alert.rule.id': rule.id,
-    'kibana.alert.rule.immutable': immutable,
-    'kibana.alert.rule.index': signalRule?.index ?? [],
-    'kibana.alert.rule.language': signalRule?.language ?? 'kuery',
-    'kibana.alert.rule.max_signals': signalRule?.max_signals ?? DEFAULT_MAX_SIGNALS,
-    'kibana.alert.rule.query': signalRule?.query ?? '*:*',
-    'kibana.alert.rule.saved_id': signalRule?.saved_id ?? '',
-    'kibana.alert.rule.threat_index': signalRule?.threat_index,
-    'kibana.alert.rule.threat_indicator_path': signalRule?.threat_indicator_path,
-    'kibana.alert.rule.threat_language': signalRule?.threat_language,
-    'kibana.alert.rule.threat_mapping.field': '', // TODO
-    'kibana.alert.rule.threat_mapping.value': '', // TODO
-    'kibana.alert.rule.threat_mapping.type': '', // TODO
-    'kibana.alert.rule.threshold.field': signalRule?.threshold?.field,
-    'kibana.alert.rule.threshold.value': signalRule?.threshold?.value,
-    'kibana.alert.rule.threshold.cardinality.field': '', // TODO
-    'kibana.alert.rule.threshold.cardinality.value': 0, // TODO
-  };
-};
-
-const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => {
-  return typeof thresholdResult === 'object';
+    [ALERT_DEPTH]: depth,
+    ...flattenWithPrefix(ALERT_RULE_NAMESPACE, rule),
+  } as unknown) as RACAlert;
 };
 
 /**
@@ -131,17 +120,16 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr
  * @param doc The parent signal/event of the new signal to be built.
  */
 export const additionalAlertFields = (doc: BaseSignalHit) => {
-  const thresholdResult = doc._source?.threshold_result;
-  if (thresholdResult != null && !isThresholdResult(thresholdResult)) {
-    throw new Error(`threshold_result failed to validate: ${thresholdResult}`);
-  }
   const originalTime = getValidDateFromDoc({
     doc,
     timestampOverride: undefined,
   });
-  return {
-    'kibana.alert.original_time': originalTime != null ? originalTime.toISOString() : undefined,
-    'kibana.alert.original_event': doc._source?.event ?? undefined,
-    'kibana.alert.threshold_result': thresholdResult,
+  const additionalFields: Record<string, unknown> = {
+    [ALERT_ORIGINAL_TIME]: originalTime != null ? originalTime.toISOString() : undefined,
   };
+  const event = doc._source?.event;
+  if (event != null) {
+    additionalFields[ALERT_ORIGINAL_EVENT] = event;
+  }
+  return additionalFields;
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts

@@ -6,14 +6,21 @@
  */
 
 import { SavedObject } from 'src/core/types';
+import { BaseHit } from '../../../../../../common/detection_engine/types';
 import type { ConfigType } from '../../../../../config';
-import { buildRuleWithOverrides } from '../../../signals/build_rule';
+import { buildRuleWithOverrides, buildRuleWithoutOverrides } from '../../../signals/build_rule';
 import { getMergeStrategy } from '../../../signals/source_fields_merging/strategies';
-import { AlertAttributes, SignalSourceHit } from '../../../signals/types';
+import { AlertAttributes, SignalSource, SignalSourceHit } from '../../../signals/types';
 import { RACAlert } from '../../types';
 import { additionalAlertFields, buildAlert } from './build_alert';
 import { filterSource } from './filter_source';
 
+const isSourceDoc = (
+  hit: SignalSourceHit
+): hit is BaseHit<{ '@timestamp': string; _source: SignalSource }> => {
+  return hit._source != null;
+};
+
 /**
  * Formats the search_after result for insertion into the signals index. We first create a
  * "best effort" merged "fields" with the "_source" object, then build the signal object,
@@ -24,17 +31,25 @@ import { filterSource } from './filter_source';
  * @returns The body that can be added to a bulk call for inserting the signal.
  */
 export const buildBulkBody = (
+  spaceId: string | null | undefined,
   ruleSO: SavedObject<AlertAttributes>,
   doc: SignalSourceHit,
-  mergeStrategy: ConfigType['alertMergeStrategy']
+  mergeStrategy: ConfigType['alertMergeStrategy'],
+  applyOverrides: boolean
 ): RACAlert => {
   const mergedDoc = getMergeStrategy(mergeStrategy)({ doc });
-  const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {});
+  const rule = applyOverrides
+    ? buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {})
+    : buildRuleWithoutOverrides(ruleSO);
   const filteredSource = filterSource(mergedDoc);
-  return {
-    ...filteredSource,
-    ...buildAlert(mergedDoc, rule),
-    ...additionalAlertFields(mergedDoc),
-    '@timestamp': new Date().toISOString(),
-  };
+  if (isSourceDoc(mergedDoc)) {
+    return {
+      ...filteredSource,
+      ...buildAlert([mergedDoc], rule, spaceId),
+      ...additionalAlertFields(mergedDoc),
+      '@timestamp': new Date().toISOString(),
+    };
+  }
+
+  throw Error('Error building alert from source document.');
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SearchTypes } from '../../../../../../common/detection_engine/types';
+
+export const flattenWithPrefix = (
+  prefix: string,
+  obj: Record<string, SearchTypes>
+): Record<string, SearchTypes> => {
+  return Object.keys(obj).reduce((acc: Record<string, SearchTypes>, key) => {
+    return {
+      ...acc,
+      [`${prefix}.${key}`]: obj[key],
+    };
+  }, {});
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { Logger } from 'kibana/server';
+
 import { SearchAfterAndBulkCreateParams, SignalSourceHit, WrapHits } from '../../signals/types';
 import { buildBulkBody } from './utils/build_bulk_body';
 import { generateId } from '../../signals/utils';
@@ -13,24 +15,33 @@ import type { ConfigType } from '../../../../config';
 import { WrappedRACAlert } from '../types';
 
 export const wrapHitsFactory = ({
-  ruleSO,
+  logger,
   mergeStrategy,
+  ruleSO,
+  spaceId,
 }: {
+  logger: Logger;
   ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  spaceId: string | null | undefined;
 }): WrapHits => (events) => {
-  const wrappedDocs: WrappedRACAlert[] = events.flatMap((doc) => [
-    {
-      _index: '',
-      _id: generateId(
-        doc._index,
-        doc._id,
-        String(doc._version),
-        ruleSO.attributes.params.ruleId ?? ''
-      ),
-      _source: buildBulkBody(ruleSO, doc as SignalSourceHit, mergeStrategy),
-    },
-  ]);
+  try {
+    const wrappedDocs: WrappedRACAlert[] = events.flatMap((doc) => [
+      {
+        _index: '',
+        _id: generateId(
+          doc._index,
+          doc._id,
+          String(doc._version),
+          ruleSO.attributes.params.ruleId ?? ''
+        ),
+        _source: buildBulkBody(spaceId, ruleSO, doc as SignalSourceHit, mergeStrategy, true),
+      },
+    ]);
 
-  return filterDuplicateSignals(ruleSO.id, wrappedDocs, true);
+    return filterDuplicateSignals(ruleSO.id, wrappedDocs, true);
+  } catch (error) {
+    logger.error(error);
+    return [];
+  }
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts

@@ -43,6 +43,21 @@ export const alertsFieldMap: FieldMap = {
     array: false,
     required: true,
   },
+  'kibana.alert.group': {
+    type: 'object',
+    array: false,
+    required: false,
+  },
+  'kibana.alert.group.id': {
+    type: 'keyword',
+    array: false,
+    required: false,
+  },
+  'kibana.alert.group.index': {
+    type: 'keyword',
+    array: false,
+    required: false,
+  },
   'kibana.alert.original_event': {
     type: 'object',
     array: false,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
+
+export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors`;
+export const ALERT_DEPTH = `${ALERT_NAMESPACE}.depth`;
+export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event`;
+export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time`;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts

@@ -168,6 +168,22 @@ export const sampleDocNoSortId = (
   sort: [],
 });
 
+export const sampleDocNoSortIdWithTimestamp = (
+  someUuid: string = sampleIdGuid,
+  ip?: string
+): SignalSourceHit & {
+  _source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
+} => {
+  const doc = sampleDocNoSortId(someUuid, ip);
+  return {
+    ...doc,
+    _source: {
+      ...doc._source,
+      '@timestamp': new Date().toISOString(),
+    },
+  };
+};
+
 export const sampleDocSeverity = (severity?: unknown, fieldName?: string): SignalSourceHit => {
   const doc = {
     _index: 'myFakeSignalIndex',

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts

@@ -8,37 +8,21 @@
 import { WrappedRACAlert } from '../rule_types/types';
 import { Ancestor, SimpleHit, WrappedSignalHit } from './types';
 
-const isWrappedSignalHit = (
-  signals: SimpleHit[],
-  isRuleRegistryEnabled: boolean
-): signals is WrappedSignalHit[] => {
-  return !isRuleRegistryEnabled;
-};
-
-const isWrappedRACAlert = (
-  signals: SimpleHit[],
-  isRuleRegistryEnabled: boolean
-): signals is WrappedRACAlert[] => {
-  return isRuleRegistryEnabled;
-};
-
 export const filterDuplicateSignals = (
   ruleId: string,
   signals: SimpleHit[],
   isRuleRegistryEnabled: boolean
 ) => {
-  if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) {
-    return signals.filter(
+  if (!isRuleRegistryEnabled) {
+    return (signals as WrappedSignalHit[]).filter(
       (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
     );
-  } else if (isWrappedRACAlert(signals, isRuleRegistryEnabled)) {
-    return signals.filter(
+  } else {
+    return (signals as WrappedRACAlert[]).filter(
       (doc) =>
         !(doc._source['kibana.alert.ancestors'] as Ancestor[]).some(
           (ancestor) => ancestor.rule === ruleId
         )
     );
-  } else {
-    return signals;
   }
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -33,6 +33,8 @@ import { BuildRuleMessage } from './rule_messages';
 import { TelemetryEventsSender } from '../../telemetry/sender';
 import { RuleParams } from '../schemas/rule_schemas';
 import { GenericBulkCreateResponse } from './bulk_create_factory';
+import { EcsFieldMap } from '../../../../../rule_registry/common/assets/field_maps/ecs_field_map';
+import { TypeOfFieldMap } from '../../../../../rule_registry/common/field_map';
 
 // used for gap detection code
 // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -166,6 +168,12 @@ export interface GetResponse {
   _source: SearchTypes;
 }
 
+export type EventHit = Exclude<TypeOfFieldMap<EcsFieldMap>, '@timestamp'> & {
+  '@timestamp': string;
+  [key: string]: SearchTypes;
+};
+export type WrappedEventHit = BaseHit<EventHit>;
+
 export type SignalSearchResponse = estypes.SearchResponse<SignalSource>;
 export type SignalSourceHit = estypes.SearchHit<SignalSource>;
 export type WrappedSignalHit = BaseHit<SignalHit>;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts

@@ -5,18 +5,20 @@
  * 2.0.
  */
 import { createHash } from 'crypto';
+import { chunk, get, isEmpty, partition } from 'lodash';
 import moment from 'moment';
 import uuidv5 from 'uuid/v5';
+
 import dateMath from '@elastic/datemath';
 import type { estypes } from '@elastic/elasticsearch';
-import { chunk, isEmpty, partition } from 'lodash';
 import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport';
-
+import { ALERT_ID } from '@kbn/rule-data-utils';
 import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants';
 import { hasLargeValueList } from '@kbn/securitysolution-list-utils';
 import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils';
 import { ElasticsearchClient } from '@kbn/securitysolution-es-utils';
+
 import {
   TimestampOverrideOrUndefined,
   Privilege,
@@ -39,6 +41,8 @@ import {
   RuleRangeTuple,
   BaseSignalHit,
   SignalSourceHit,
+  SimpleHit,
+  WrappedEventHit,
 } from './types';
 import { BuildRuleMessage } from './rule_messages';
 import { ShardError } from '../../types';
@@ -52,6 +56,8 @@ import {
   ThreatRuleParams,
   ThresholdRuleParams,
 } from '../schemas/rule_schemas';
+import { WrappedRACAlert } from '../rule_types/types';
+import { SearchTypes } from '../../../../common/detection_engine/types';
 
 interface SortExceptionsReturn {
   exceptionsWithValueLists: ExceptionListItemSchema[];
@@ -928,3 +934,25 @@ export const buildChunkedOrFilter = (field: string, values: string[], chunkSize:
     })
     .join(' OR ');
 };
+
+export const isWrappedEventHit = (event: SimpleHit): event is WrappedEventHit => {
+  return !isWrappedSignalHit(event) && !isWrappedRACAlert(event);
+};
+
+export const isWrappedSignalHit = (event: SimpleHit): event is WrappedSignalHit => {
+  return (event as WrappedSignalHit)?._source?.signal != null;
+};
+
+export const isWrappedRACAlert = (event: SimpleHit): event is WrappedRACAlert => {
+  return (event as WrappedRACAlert)?._source?.[ALERT_ID] != null;
+};
+
+export const getField = <T extends SearchTypes>(event: SimpleHit, field: string): T | undefined => {
+  if (isWrappedRACAlert(event)) {
+    return event._source, field.replace('signal', 'kibana.alert') as T; // TODO: handle special cases
+  } else if (isWrappedSignalHit(event)) {
+    return get(event._source, field) as T;
+  } else if (isWrappedEventHit(event)) {
+    return get(event._source, field) as T;
+  }
+};