[Detection Engine] Make 7.x signals/alerts compatible with 8.18 alerts UI (#209936)

### Summary

Addresses https://github.com/elastic/kibana/issues/208204. 

The general idea here is to add aliases to the legacy indices in order
to "backfill" some newer fields that are now required in various alerts
UIs. We already have the mechanism to apply mappings changes for just
this scenario, so this was just a matter of adding the new mappings and
bumping versions appropriately.

While the consumer field is static ('siem'), we're opting to use a
`keyword` runtime field instead of a `constant_keyword` in order to
allow the possibility of other values for this field in the future.

While we do not expect the legacy indices to be written to at all in
8.x+, new legacy indices are still created every 30d due to the rollover
policy, and so neglecting to add the mappings to those new indices could
lead to mapping conflicts. So: while the mappings on the index template
are not expected to be used, we still update them for
consistency/future-proofing.

### Screenshots
<details>
<summary>
</summary>
<kbd>
Alerts Table before fix:
<img width="1727" alt="Screenshot 2025-02-06 at 5 15 59 PM"
src="https://github.com/user-attachments/assets/1f83d6f3-a1b0-4025-b1e3-78e34340ef91"
/>
</kbd>


<kbd>
Alerts Table after fix:
<img width="1727" alt="Screenshot 2025-02-06 at 5 40 02 PM"
src="https://github.com/user-attachments/assets/c68ad198-226c-413e-a4ac-7f590e6d500b"
/>
</kbd>
</details>

### Steps to review
1. Create a 7.17 cloud GCP instance (on us-west-2 so that you can
upgrade to 8.18)
2. Create several rules of different types (since part of this bug
involves mapping existing rule types to the new `rule_type_id` field)
3. Generate alerts with those rules (I prefer to use auditbeat to
quickly ingest some ECS data)
4. (optional) Take a snapshot of your instance (so as to skip steps 2-3
in subsequent tests)
5. Upgrade the instance to 8.18
6. Observe the absence of any 7.17 alerts in the 8.18 Alerts table
(note: if you have no 8.18 alerts, the table will be empty)
7. Connect your local kibana (running this branch) to your cloud
instance:

    ```yaml
    elasticsearch:
username: USERNAME_HERE # Note: user will need index management
privileges
      password: PASSWORD_HERE
      hosts: https://ES_ENDPOINT_HERE.es.us-west2.gcp.elastic-cloud.com
      ssl:
        verificationMode: none
      ignoreVersionMismatch: true
    
    node.roles: ['ui'] # Only run UI (not task manager) locally
    ```
8. Navigate to a security page in your browser. This will trigger the
call to update 7.x signals
9. Observe that the 7.x alerts are now visible on the alerts table, both
locally and using the cloud instance's kibana



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/8_18_alerts_compatibility_mappings.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ALERT_RULE_CONSUMER, ALERT_RULE_TYPE, ALERT_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { ruleTypeMappings } from '@kbn/securitysolution-rules';
+
+export const createMappingsFor818Compatibility = () => ({
+  runtime: {
+    [ALERT_RULE_CONSUMER]: {
+      type: 'keyword',
+      script: { source: "emit('siem')" },
+    },
+    [ALERT_RULE_TYPE_ID]: {
+      type: 'keyword',
+      script: { source: mapRuleTypeToRuleTypeIdScript(ruleTypeMappings) },
+    },
+  },
+});
+
+const mapRuleTypeToRuleTypeIdScript = (ruleTypeToRuleTypeIdMap: Record<string, string>): string => `
+  String rule_type = doc['${ALERT_RULE_TYPE}'].value;
+  ${Object.entries(ruleTypeToRuleTypeIdMap)
+    .map(
+      ([ruleType, ruleTypeId]) => `if (rule_type == '${ruleType}') return emit('${ruleTypeId}');`
+    )
+    .join('')}`;

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap

@@ -3,7 +3,7 @@
 exports[`get_signals_template backwards compatibility mappings for version 45 should match snapshot 1`] = `
 Object {
   "_meta": Object {
-    "aliases_version": 4,
+    "aliases_version": 5,
     "version": 45,
   },
   "dynamic": false,
@@ -531,6 +531,20 @@ Object {
       },
       "type": "keyword",
     },
+    "kibana.alert.rule.consumer": Object {
+      "script": Object {
+        "source": "emit('siem')",
+      },
+      "type": "keyword",
+    },
+    "kibana.alert.rule.rule_type_id": Object {
+      "script": Object {
+        "source": "
+  String rule_type = doc['kibana.alert.rule.type'].value;
+  if (rule_type == 'eql') return emit('siem.eqlRule');if (rule_type == 'esql') return emit('siem.esqlRule');if (rule_type == 'machine_learning') return emit('siem.mlRule');if (rule_type == 'query') return emit('siem.queryRule');if (rule_type == 'saved_query') return emit('siem.savedQueryRule');if (rule_type == 'threat_match') return emit('siem.indicatorRule');if (rule_type == 'threshold') return emit('siem.thresholdRule');if (rule_type == 'new_terms') return emit('siem.newTermsRule');",
+      },
+      "type": "keyword",
+    },
   },
 }
 `;
@@ -538,7 +552,7 @@ Object {
 exports[`get_signals_template backwards compatibility mappings for version 57 should match snapshot 1`] = `
 Object {
   "_meta": Object {
-    "aliases_version": 4,
+    "aliases_version": 5,
     "version": 57,
   },
   "dynamic": false,
@@ -1059,6 +1073,22 @@ Object {
       "type": "object",
     },
   },
+  "runtime": Object {
+    "kibana.alert.rule.consumer": Object {
+      "script": Object {
+        "source": "emit('siem')",
+      },
+      "type": "keyword",
+    },
+    "kibana.alert.rule.rule_type_id": Object {
+      "script": Object {
+        "source": "
+  String rule_type = doc['kibana.alert.rule.type'].value;
+  if (rule_type == 'eql') return emit('siem.eqlRule');if (rule_type == 'esql') return emit('siem.esqlRule');if (rule_type == 'machine_learning') return emit('siem.mlRule');if (rule_type == 'query') return emit('siem.queryRule');if (rule_type == 'saved_query') return emit('siem.savedQueryRule');if (rule_type == 'threat_match') return emit('siem.indicatorRule');if (rule_type == 'threshold') return emit('siem.thresholdRule');if (rule_type == 'new_terms') return emit('siem.newTermsRule');",
+      },
+      "type": "keyword",
+    },
+  },
 }
 `;
 
@@ -1075,8 +1105,8 @@ Object {
     },
     "mappings": Object {
       "_meta": Object {
-        "aliases_version": 4,
-        "version": 77,
+        "aliases_version": 5,
+        "version": 87,
       },
       "dynamic": false,
       "properties": Object {
@@ -6739,6 +6769,22 @@ Object {
           },
         },
       },
+      "runtime": Object {
+        "kibana.alert.rule.consumer": Object {
+          "script": Object {
+            "source": "emit('siem')",
+          },
+          "type": "keyword",
+        },
+        "kibana.alert.rule.rule_type_id": Object {
+          "script": Object {
+            "source": "
+  String rule_type = doc['kibana.alert.rule.type'].value;
+  if (rule_type == 'eql') return emit('siem.eqlRule');if (rule_type == 'esql') return emit('siem.esqlRule');if (rule_type == 'machine_learning') return emit('siem.mlRule');if (rule_type == 'query') return emit('siem.queryRule');if (rule_type == 'saved_query') return emit('siem.savedQueryRule');if (rule_type == 'threat_match') return emit('siem.indicatorRule');if (rule_type == 'threshold') return emit('siem.thresholdRule');if (rule_type == 'new_terms') return emit('siem.newTermsRule');",
+          },
+          "type": "keyword",
+        },
+      },
     },
     "settings": Object {
       "index": Object {
@@ -6754,6 +6800,6 @@ Object {
       },
     },
   },
-  "version": 77,
+  "version": 87,
 }
 `;

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts

@@ -12,6 +12,7 @@ import ecsMapping from './ecs_mapping.json';
 import otherMapping from './other_mappings.json';
 import aadFieldConversion from './signal_aad_mapping.json';
 import signalExtraFields from './signal_extra_fields.json';
+import { createMappingsFor818Compatibility } from './8_18_alerts_compatibility_mappings';
 
 /**
   @constant
@@ -28,7 +29,7 @@ import signalExtraFields from './signal_extra_fields.json';
   incremented by 10 in order to add "room" for the aforementioned patch
   release
 */
-export const SIGNALS_TEMPLATE_VERSION = 77;
+export const SIGNALS_TEMPLATE_VERSION = 87;
 /**
   @constant
   @type {number}
@@ -42,7 +43,7 @@ export const SIGNALS_TEMPLATE_VERSION = 77;
   UI will call create_index_route and and go through the index update process. Increment this number if
   making changes to the field aliases we use to make signals forwards-compatible.
 */
-export const SIGNALS_FIELD_ALIASES_VERSION = 4;
+export const SIGNALS_FIELD_ALIASES_VERSION = 5;
 
 /**
   @constant
@@ -77,19 +78,25 @@ export const getSignalsTemplate = (index: string, aadIndexAliasName: string, spa
         },
       },
       mappings: {
-        dynamic: false,
-        properties: merge(
-          ecsMapping.mappings.properties,
-          otherMapping.mappings.properties,
-          fieldAliases,
-          signalsMapping.mappings.properties,
+        ...merge(
           {
-            [SPACE_IDS]: {
-              type: 'constant_keyword',
-              value: spaceId,
-            },
-          }
+            properties: merge(
+              ecsMapping.mappings.properties,
+              otherMapping.mappings.properties,
+              fieldAliases,
+              signalsMapping.mappings.properties,
+              {
+                [SPACE_IDS]: {
+                  type: 'constant_keyword',
+                  value: spaceId,
+                },
+              }
+            ),
+          },
+          createMappingsFor818Compatibility(),
+          { dynamic: false }
         ),
+
         _meta: {
           version: SIGNALS_TEMPLATE_VERSION,
           [ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
@@ -169,6 +176,11 @@ export const backwardsCompatibilityMappings = (spaceId: string) => [
       },
     },
   },
+  {
+    minVersion: 0,
+    maxVersion: 77,
+    mapping: createMappingsFor818Compatibility(),
+  },
 ];
 
 export const createBackwardsCompatibilityMapping = (version: number, spaceId: string) => {