Add intended timestamp (#191717)

## Add new field to alert


Add optional `kibana.alert.intended_timestamp`. For scheduled rules it
has the same values as ALERT_RULE_EXECUTION_TIMESTAMP
(`kibana.alert.rule.execution.timestamp`)

for manual rule runs (backfill) it - will get the startedAtOverridden 

For example if i have event at 14:30

And if we run manual rule run from 14:00-15:00, then alert will have
`kibana.alert.intended_timestamp` at 15:00

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts

@@ -47,6 +47,7 @@ import {
   EVENT_KIND,
   EVENT_ORIGINAL,
   TAGS,
+  ALERT_INTENDED_TIMESTAMP,
 } from '@kbn/rule-data-utils';
 import { MultiField } from './types';
 
@@ -133,6 +134,11 @@ export const alertFieldMap = {
     array: false,
     required: false,
   },
+  [ALERT_INTENDED_TIMESTAMP]: {
+    type: 'date',
+    array: false,
+    required: false,
+  },
   [ALERT_RULE_EXECUTION_UUID]: {
     type: 'keyword',
     array: false,

packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts

@@ -93,6 +93,7 @@ const AlertOptional = rt.partial({
   'kibana.alert.end': schemaDate,
   'kibana.alert.flapping': schemaBoolean,
   'kibana.alert.flapping_history': schemaBooleanArray,
+  'kibana.alert.intended_timestamp': schemaDate,
   'kibana.alert.last_detected': schemaDate,
   'kibana.alert.maintenance_window_ids': schemaStringArray,
   'kibana.alert.previous_action_group': schemaString,

packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts

@@ -137,6 +137,7 @@ const SecurityAlertOptional = rt.partial({
   'kibana.alert.group.id': schemaString,
   'kibana.alert.group.index': schemaNumber,
   'kibana.alert.host.criticality_level': schemaString,
+  'kibana.alert.intended_timestamp': schemaDate,
   'kibana.alert.last_detected': schemaDate,
   'kibana.alert.maintenance_window_ids': schemaStringArray,
   'kibana.alert.new_terms': schemaStringArray,

packages/kbn-rule-data-utils/src/default_alerts_as_data.ts

@@ -59,6 +59,9 @@ const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const;
 // kibana.alert.last_detected - timestamp when the alert was last seen
 const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected` as const;
 
+// kiana.alert.intended_timestamp - timestamp when the alert was intended to be detected, useful for backfilling
+const ALERT_INTENDED_TIMESTAMP = `${ALERT_NAMESPACE}.intended_timestamp` as const;
+
 // kibana.alert.reason - human readable reason that this alert is active
 const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const;
 
@@ -141,6 +144,7 @@ const fields = {
   ALERT_RULE_CATEGORY,
   ALERT_RULE_CONSUMER,
   ALERT_RULE_EXECUTION_TIMESTAMP,
+  ALERT_INTENDED_TIMESTAMP,
   ALERT_RULE_EXECUTION_UUID,
   ALERT_RULE_NAME,
   ALERT_RULE_PARAMETERS,
@@ -185,6 +189,7 @@ export {
   ALERT_RULE_CATEGORY,
   ALERT_RULE_CONSUMER,
   ALERT_RULE_EXECUTION_TIMESTAMP,
+  ALERT_INTENDED_TIMESTAMP,
   ALERT_RULE_EXECUTION_UUID,
   ALERT_RULE_NAME,
   ALERT_RULE_PARAMETERS,

x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts

@@ -260,6 +260,9 @@ describe('mappingFromFieldMap', () => {
                     },
                   },
                 },
+                intended_timestamp: {
+                  type: 'date',
+                },
                 rule: {
                   properties: {
                     category: {

x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap

@@ -837,6 +837,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -1930,6 +1935,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -3023,6 +3033,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -4116,6 +4131,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -5209,6 +5229,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -6308,6 +6333,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -7401,6 +7431,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,
@@ -8494,6 +8529,11 @@ Object {
       "required": true,
       "type": "keyword",
     },
+    "kibana.alert.intended_timestamp": Object {
+      "array": false,
+      "required": false,
+      "type": "date",
+    },
     "kibana.alert.last_detected": Object {
       "array": false,
       "required": false,

x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts

@@ -80,6 +80,11 @@ it('matches snapshot', () => {
         "required": true,
         "type": "keyword",
       },
+      "kibana.alert.intended_timestamp": Object {
+        "array": false,
+        "required": false,
+        "type": "date",
+      },
       "kibana.alert.last_detected": Object {
         "array": false,
         "required": false,

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts

@@ -25,6 +25,7 @@ import {
   TIMESTAMP,
   VERSION,
   ALERT_RULE_EXECUTION_TIMESTAMP,
+  ALERT_INTENDED_TIMESTAMP,
 } from '@kbn/rule-data-utils';
 import { mapKeys, snakeCase } from 'lodash/fp';
 import type { IRuleDataClient } from '..';
@@ -55,11 +56,13 @@ const augmentAlerts = <T>({
   options,
   kibanaVersion,
   currentTimeOverride,
+  intendedTimestamp,
 }: {
   alerts: Array<{ _id: string; _source: T }>;
   options: RuleExecutorOptions<any, any, any, any, any>;
   kibanaVersion: string;
   currentTimeOverride: Date | undefined;
+  intendedTimestamp: Date | undefined;
 }) => {
   const commonRuleFields = getCommonAlertFields(options);
   return alerts.map((alert) => {
@@ -69,6 +72,9 @@ const augmentAlerts = <T>({
         [ALERT_RULE_EXECUTION_TIMESTAMP]: new Date(),
         [ALERT_START]: currentTimeOverride ?? new Date(),
         [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
+        [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp
+          ? intendedTimestamp
+          : currentTimeOverride ?? new Date(),
         [VERSION]: kibanaVersion,
         ...(options?.maintenanceWindowIds?.length
           ? { [ALERT_MAINTENANCE_WINDOW_IDS]: options.maintenanceWindowIds }
@@ -251,6 +257,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
             ...options.services,
             alertWithPersistence: async (alerts, refresh, maxAlerts = undefined, enrichAlerts) => {
               const numAlerts = alerts.length;
+
               logger.debug(`Found ${numAlerts} alerts.`);
 
               const ruleDataClientWriter = await ruleDataClient.getWriter({
@@ -297,11 +304,17 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   alertsWereTruncated = true;
                 }
 
+                let intendedTimestamp;
+                if (options.startedAtOverridden) {
+                  intendedTimestamp = options.startedAt;
+                }
+
                 const augmentedAlerts = augmentAlerts({
                   alerts: enrichedAlerts,
                   options,
                   kibanaVersion: ruleDataClient.kibanaVersion,
                   currentTimeOverride: undefined,
+                  intendedTimestamp,
                 });
 
                 const response = await ruleDataClientWriter.bulk({
@@ -381,6 +394,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
 
               let alertsWereTruncated = false;
 
+              let intendedTimestamp;
+              if (options.startedAtOverridden) {
+                intendedTimestamp = options.startedAt;
+              }
+
               if (writeAlerts && alerts.length > 0) {
                 const suppressionWindowStart = dateMath.parse(suppressionWindow, {
                   forceNow: currentTimeOverride,
@@ -560,6 +578,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   options,
                   kibanaVersion: ruleDataClient.kibanaVersion,
                   currentTimeOverride,
+                  intendedTimestamp,
                 });
 
                 const bulkResponse = await ruleDataClientWriter.bulk({

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts

@@ -18,6 +18,8 @@ import {
   ALERT_SUPPRESSION_TERMS,
   TIMESTAMP,
   ALERT_LAST_DETECTED,
+  ALERT_INTENDED_TIMESTAMP,
+  ALERT_RULE_EXECUTION_TIMESTAMP,
 } from '@kbn/rule-data-utils';
 import { flattenWithPrefix } from '@kbn/securitysolution-rules';
 import { Rule } from '@kbn/alerting-plugin/common';
@@ -538,6 +540,19 @@ export default ({ getService }: FtrProviderContext) => {
       expect(previewAlerts.length).toEqual(1);
     });
 
+    it('should generate alerts with the correct intended timestamp fields', async () => {
+      const rule: QueryRuleCreateProps = {
+        ...getRuleForAlertTesting(['auditbeat-*']),
+        query: `_id:${ID}`,
+      };
+
+      const { previewId } = await previewRule({ supertest, rule });
+      const previewAlerts = await getPreviewAlerts({ es, previewId });
+      const alert = previewAlerts[0]._source;
+
+      expect(alert?.[ALERT_INTENDED_TIMESTAMP]).toEqual(alert?.[TIMESTAMP]);
+    });
+
     describe('with suppression enabled', () => {
       before(async () => {
         await esArchiver.load('x-pack/test/functional/es_archives/security_solution/suppression');
@@ -2447,6 +2462,56 @@ export default ({ getService }: FtrProviderContext) => {
         );
       });
 
+      it('alerts has intended_timestamp set to the time of the manual run', async () => {
+        const id = uuidv4();
+        const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString();
+        const secondTimestamp = new Date().toISOString();
+        const firstDocument = {
+          id,
+          '@timestamp': firstTimestamp,
+          agent: {
+            name: 'agent-1',
+          },
+        };
+        const secondDocument = {
+          id,
+          '@timestamp': secondTimestamp,
+          agent: {
+            name: 'agent-2',
+          },
+        };
+        await indexListOfDocuments([firstDocument, secondDocument]);
+
+        const rule: QueryRuleCreateProps = {
+          ...getRuleForAlertTesting(['ecs_compliant']),
+          rule_id: 'rule-1',
+          query: `id:${id}`,
+          from: 'now-1h',
+          interval: '1h',
+        };
+        const createdRule = await createRule(supertest, log, rule);
+        const alerts = await getAlerts(supertest, log, es, createdRule);
+
+        expect(alerts.hits.hits).toHaveLength(1);
+
+        expect(alerts.hits.hits[0]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual(
+          alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TIMESTAMP]
+        );
+
+        const backfillStartDate = moment(firstTimestamp).startOf('hour');
+        const backfillEndDate = moment(backfillStartDate).add(1, 'h');
+        const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
+          startDate: backfillStartDate,
+          endDate: backfillEndDate,
+        });
+
+        await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
+        const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
+        expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual(
+          backfillEndDate.toISOString()
+        );
+      });
+
       it('alerts when run on a time range that the rule has not previously seen, and deduplicates if run there more than once', async () => {
         const id = uuidv4();
         const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString();

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts

@@ -153,6 +153,7 @@ function alertsAreTheSame(alertsA: any[], alertsB: any[]): void {
       'kibana.alert.rule.uuid',
       'kibana.alert.rule.execution.uuid',
       'kibana.alert.rule.execution.timestamp',
+      'kibana.alert.intended_timestamp',
       'kibana.alert.start',
       'kibana.alert.reason',
       'kibana.alert.uuid',

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/remove_random_valued_properties_from_alert.ts

@@ -6,7 +6,7 @@
  */
 
 import { DetectionAlert } from '@kbn/security-solution-plugin/common/api/detection_engine';
-import { ALERT_LAST_DETECTED, ALERT_START } from '@kbn/rule-data-utils';
+import { ALERT_LAST_DETECTED, ALERT_START, ALERT_INTENDED_TIMESTAMP } from '@kbn/rule-data-utils';
 
 export const removeRandomValuedPropertiesFromAlert = (alert: DetectionAlert | undefined) => {
   if (!alert) {
@@ -24,6 +24,7 @@ export const removeRandomValuedPropertiesFromAlert = (alert: DetectionAlert | un
     'kibana.alert.url': alertURL,
     [ALERT_START]: alertStart,
     [ALERT_LAST_DETECTED]: lastDetected,
+    [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp,
     ...restOfAlert
   } = alert;
   return restOfAlert;