[Security Solution] Support rule type changes in the rule upgrade workflow (#161247)

**Fixes: https://github.com/elastic/kibana/issues/161094**

## Summary

- Adds support for rule type changes in the
`/internal/detection_engine/prebuilt_rules/upgrade/_review` endpoint.
- Previously, if any rule had a different `type` in its
`current_version` compared to its `target_version` the request would
fail with `500`.
- This PR:
    - updates this behaviour to accept rule type changes
- creates a new `calculateAllFieldsDiff` method that is responsible for
calculating diffs among all fields of all rule types. Used exclusively
when there has been a rule type change between the current version and
the target version (which can normally happen through upgrades of the
`security_detection_engine` package) OR when the base version has a
different type as the current version (which should not happen under
normal conditions and user behaviour).
- updates the diffable fields types for each specifc rule type (e.g.:
`DiffableCustomQueryFields`,`DiffableEqlFields`,`DiffableThreatMatchFields`,
etc) , replacing the `data_query` field name for either `eql_query` (for
EQL type rules) or `kql_query` (for all others).


## How to test
1. With a clean Kibana state, use the
`xpack.securitySolution.prebuiltRulesPackageVersion` config to force
Kibana to install a package that contains the rules with their original
type:
```
xpack.securitySolution.prebuiltRulesPackageVersion: '8.3.1'
```
2. Install the four "offending" rules, [listed
below.](https://github.com/elastic/kibana/pull/161247#issuecomment-1622132120)
3. Remove the config, restart Kibana and navigate to the Rules Page so
that the latest package is installed.
4. Navigate to the Rule Updates table. The four installed rules should
have updates available. Update them.
5. All the listed rule types should be updated, as well as their
corresponding fields.


### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: jpdjere <jpdjeredjian@gmail.com>

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts

@@ -114,7 +114,7 @@ export type DiffableCustomQueryFields = t.TypeOf<typeof DiffableCustomQueryField
 export const DiffableCustomQueryFields = buildSchema({
   required: {
     type: t.literal('query'),
-    data_query: RuleKqlQuery, // NOTE: new field
+    kql_query: RuleKqlQuery, // NOTE: new field
   },
   optional: {
     data_source: RuleDataSource, // NOTE: new field
@@ -126,7 +126,7 @@ export type DiffableSavedQueryFields = t.TypeOf<typeof DiffableSavedQueryFields>
 export const DiffableSavedQueryFields = buildSchema({
   required: {
     type: t.literal('saved_query'),
-    data_query: RuleKqlQuery, // NOTE: new field
+    kql_query: RuleKqlQuery, // NOTE: new field
   },
   optional: {
     data_source: RuleDataSource, // NOTE: new field
@@ -138,7 +138,7 @@ export type DiffableEqlFields = t.TypeOf<typeof DiffableEqlFields>;
 export const DiffableEqlFields = buildSchema({
   required: {
     type: t.literal('eql'),
-    data_query: RuleEqlQuery, // NOTE: new field
+    eql_query: RuleEqlQuery, // NOTE: new field
   },
   optional: {
     data_source: RuleDataSource, // NOTE: new field
@@ -152,7 +152,7 @@ export type DiffableThreatMatchFields = t.TypeOf<typeof DiffableThreatMatchField
 export const DiffableThreatMatchFields = buildSchema({
   required: {
     type: t.literal('threat_match'),
-    data_query: RuleKqlQuery, // NOTE: new field
+    kql_query: RuleKqlQuery, // NOTE: new field
     threat_query: InlineKqlQuery, // NOTE: new field
     threat_index,
     threat_mapping,
@@ -169,7 +169,7 @@ export type DiffableThresholdFields = t.TypeOf<typeof DiffableThresholdFields>;
 export const DiffableThresholdFields = buildSchema({
   required: {
     type: t.literal('threshold'),
-    data_query: RuleKqlQuery, // NOTE: new field
+    kql_query: RuleKqlQuery, // NOTE: new field
     threshold: Threshold,
   },
   optional: {
@@ -191,7 +191,7 @@ export type DiffableNewTermsFields = t.TypeOf<typeof DiffableNewTermsFields>;
 export const DiffableNewTermsFields = buildSchema({
   required: {
     type: t.literal('new_terms'),
-    data_query: InlineKqlQuery, // NOTE: new field
+    kql_query: InlineKqlQuery, // NOTE: new field
     new_terms_fields: NewTermsFields,
     history_window_start: HistoryWindowStart,
   },
@@ -239,3 +239,21 @@ export const DiffableRule = t.intersection([
     DiffableNewTermsFields,
   ]),
 ]);
+
+/**
+ * This is a merge of all fields from all rule types into a single TS type.
+ * This is NOT a union discriminated by rule type, as DiffableRule is.
+ */
+export type DiffableAllFields = DiffableCommonFields &
+  Omit<DiffableCustomQueryFields, 'type'> &
+  Omit<DiffableSavedQueryFields, 'type'> &
+  Omit<DiffableEqlFields, 'type'> &
+  Omit<DiffableThreatMatchFields, 'type'> &
+  Omit<DiffableThresholdFields, 'type'> &
+  Omit<DiffableMachineLearningFields, 'type'> &
+  Omit<DiffableNewTermsFields, 'type'> &
+  DiffableRuleTypeField;
+
+interface DiffableRuleTypeField {
+  type: DiffableRule['type'];
+}

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff.ts

@@ -6,6 +6,7 @@
  */
 
 import type {
+  DiffableAllFields,
   DiffableCommonFields,
   DiffableCustomQueryFields,
   DiffableEqlFields,
@@ -18,6 +19,7 @@ import type {
 
 import type { FieldsDiff } from './fields_diff';
 
+export type AllFieldsDiff = FieldsDiff<DiffableAllFields>;
 export type CommonFieldsDiff = FieldsDiff<DiffableCommonFields>;
 export type CustomQueryFieldsDiff = FieldsDiff<DiffableCustomQueryFields>;
 export type SavedQueryFieldsDiff = FieldsDiff<DiffableSavedQueryFields>;

x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts

@@ -9,6 +9,7 @@ import { assertUnreachable } from '../../../../../../../common/utility_types';
 import { invariant } from '../../../../../../../common/utils/invariant';
 
 import type {
+  DiffableAllFields,
   DiffableCommonFields,
   DiffableCustomQueryFields,
   DiffableEqlFields,
@@ -20,6 +21,7 @@ import type {
   DiffableThresholdFields,
 } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule';
 import type {
+  AllFieldsDiff,
   CommonFieldsDiff,
   CustomQueryFieldsDiff,
   EqlFieldsDiff,
@@ -53,6 +55,24 @@ export const calculateRuleFieldsDiff = (
   const { base_version, current_version, target_version } = ruleVersions;
   const hasBaseVersion = base_version !== MissingVersion;
 
+  const isRuleTypeDifferentInTargetVersion = current_version.type !== target_version.type;
+  const isRuleTypeDifferentInBaseVersion = hasBaseVersion
+    ? current_version.type !== base_version.type
+    : false;
+
+  if (isRuleTypeDifferentInTargetVersion || isRuleTypeDifferentInBaseVersion) {
+    // If rule type has been changed by Elastic in the target version (can happen)
+    // or by user in the current version (should never happen), we can't calculate the diff
+    // only for fields of a single rule type, and need to calculate it for all fields
+    // of all the rule types we have.
+    // TODO: Try to get rid of "as" casting
+    return calculateAllFieldsDiff({
+      base_version: base_version as DiffableAllFields | MissingVersion,
+      current_version: current_version as DiffableAllFields,
+      target_version: target_version as DiffableAllFields,
+    }) as RuleFieldsDiff;
+  }
+
   switch (current_version.type) {
     case 'query': {
       if (hasBaseVersion) {
@@ -175,7 +195,7 @@ const calculateCustomQueryFieldsDiff = (
 
 const customQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableCustomQueryFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  kql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   alert_suppression: simpleDiffAlgorithm,
 };
@@ -188,7 +208,7 @@ const calculateSavedQueryFieldsDiff = (
 
 const savedQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableSavedQueryFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  kql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   alert_suppression: simpleDiffAlgorithm,
 };
@@ -201,7 +221,7 @@ const calculateEqlFieldsDiff = (
 
 const eqlFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableEqlFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  eql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   event_category_override: simpleDiffAlgorithm,
   timestamp_field: simpleDiffAlgorithm,
@@ -216,7 +236,7 @@ const calculateThreatMatchFieldsDiff = (
 
 const threatMatchFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableThreatMatchFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  kql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   threat_query: simpleDiffAlgorithm,
   threat_index: simpleDiffAlgorithm,
@@ -234,7 +254,7 @@ const calculateThresholdFieldsDiff = (
 
 const thresholdFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableThresholdFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  kql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   threshold: simpleDiffAlgorithm,
 };
@@ -260,8 +280,26 @@ const calculateNewTermsFieldsDiff = (
 
 const newTermsFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableNewTermsFields> = {
   type: simpleDiffAlgorithm,
-  data_query: simpleDiffAlgorithm,
+  kql_query: simpleDiffAlgorithm,
   data_source: simpleDiffAlgorithm,
   new_terms_fields: simpleDiffAlgorithm,
   history_window_start: simpleDiffAlgorithm,
 };
+
+const calculateAllFieldsDiff = (
+  ruleVersions: ThreeVersionsOf<DiffableAllFields>
+): AllFieldsDiff => {
+  return calculateFieldsDiffFor(ruleVersions, allFieldsDiffAlgorithms);
+};
+
+const allFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor<DiffableAllFields> = {
+  ...commonFieldsDiffAlgorithms,
+  ...customQueryFieldsDiffAlgorithms,
+  ...savedQueryFieldsDiffAlgorithms,
+  ...eqlFieldsDiffAlgorithms,
+  ...threatMatchFieldsDiffAlgorithms,
+  ...thresholdFieldsDiffAlgorithms,
+  ...machineLearningFieldsDiffAlgorithms,
+  ...newTermsFieldsDiffAlgorithms,
+  type: simpleDiffAlgorithm,
+};

x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts

@@ -146,7 +146,7 @@ const extractDiffableCustomQueryFields = (
 ): DiffableCustomQueryFields => {
   return {
     type: rule.type,
-    data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
+    kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     alert_suppression: rule.alert_suppression,
   };
@@ -157,7 +157,7 @@ const extractDiffableSavedQueryFieldsFromRuleObject = (
 ): DiffableSavedQueryFields => {
   return {
     type: rule.type,
-    data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
+    kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     alert_suppression: rule.alert_suppression,
   };
@@ -168,7 +168,7 @@ const extractDiffableEqlFieldsFromRuleObject = (
 ): DiffableEqlFields => {
   return {
     type: rule.type,
-    data_query: extractRuleEqlQuery(rule.query, rule.language, rule.filters),
+    eql_query: extractRuleEqlQuery(rule.query, rule.language, rule.filters),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     event_category_override: rule.event_category_override,
     timestamp_field: rule.timestamp_field,
@@ -181,7 +181,7 @@ const extractDiffableThreatMatchFieldsFromRuleObject = (
 ): DiffableThreatMatchFields => {
   return {
     type: rule.type,
-    data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
+    kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     threat_query: extractInlineKqlQuery(
       rule.threat_query,
@@ -201,7 +201,7 @@ const extractDiffableThresholdFieldsFromRuleObject = (
 ): DiffableThresholdFields => {
   return {
     type: rule.type,
-    data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
+    kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     threshold: rule.threshold,
   };
@@ -222,7 +222,7 @@ const extractDiffableNewTermsFieldsFromRuleObject = (
 ): DiffableNewTermsFields => {
   return {
     type: rule.type,
-    data_query: extractInlineKqlQuery(rule.query, rule.language, rule.filters),
+    kql_query: extractInlineKqlQuery(rule.query, rule.language, rule.filters),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
     new_terms_fields: rule.new_terms_fields,
     history_window_start: rule.history_window_start,