[8.x] [Security Solution] Suppress prebuilt rule SO duplicates in review install endpoint (#218123) (#218248)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Security Solution] Suppress prebuilt rule SO duplicates in review
install endpoint
(#218123)](https://github.com/elastic/kibana/pull/218123)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Maxim
Palenov","email":"maxim.palenov@elastic.co"},"sourceCommit":{"committedDate":"2025-04-15T11:45:08Z","message":"[Security
Solution] Suppress prebuilt rule SO duplicates in review install
endpoint (#218123)\n\n## Summary\n\nThis PR makes sure a buggy
`security_detection_engine` package doesn't affect a preview
installation endpoint. Older security detection rules package versions
contain saved object rule duplicates affecting the endpoint.\n\nHaving
`security_detection_engine` v`8.17.1` package installed
`/internal/detection_engine/prebuilt_rules/status` and
`/internal/detection_engine/prebuilt_rules/installation/_review`
endpoints return a different number of rules available to install.\n\n##
Details\n\nOlder `security_detection_engine` package versions contain
rule saved objects duplicates representing the latest version. For
example, `8.17.1` version has a rule `Microsoft 365 User Restricted from
Sending Email` with `rule_id` = `0136b315-b566-482f-866c-1d8e2477ba16`
and the latest version `206`. Since a package may contain multiple
historical rule versions it sticks to the following format
`<rule_id>_<version>` where `<rule_id>` is the unique rule's UUID and
`<version>` it's version. Some older package versions omit `<version>`
for the latest rule version. `Microsoft 365 User Restricted from Sending
Email` rule mentioned above has two equal assets corresponding to the
latest version with the only difference in the saved object id
`0136b315-b566-482f-866c-1d8e2477ba16` and
`0136b315-b566-482f-866c-1d8e2477ba16_206`.\n\nPrebuilt rules preview
endpoint was designed to handle `<rule_id>_<version>` format only.
Consequently, it improperly handles older prebuilt rules package
version.\n\nThis bug manifested in
https://github.com/elastic/kibana/pull/217544 where
`security_detection_engine` version has been bumped to `8.18.1`. It
resulted in a failed integration test. Further investigation has shown
that the test installs an older package version `8.17.1` to assert
prebuilt rules upgrade workflow works correctly.\n\nThe fix is
implemented in `PrebuiltRuleAssetsClient.fetchAssetsByVersion()` by
using `Map` to deduplicate prebuilt rule
assets.","sha":"87f8274f4160f4d94f25d19f7d71ec4c35f4431e","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:skip","impact:high","Team:Detections
and Resp","Team: SecuritySolution","Team:Detection Rule
Management","Feature:Prebuilt Detection
Rules","backport:version","v9.1.0","v8.19.0","v8.18.1","v9.0.1"],"title":"[Security
Solution] Suppress prebuilt rule SO duplicates in review install
endpoint","number":218123,"url":"https://github.com/elastic/kibana/pull/218123","mergeCommit":{"message":"[Security
Solution] Suppress prebuilt rule SO duplicates in review install
endpoint (#218123)\n\n## Summary\n\nThis PR makes sure a buggy
`security_detection_engine` package doesn't affect a preview
installation endpoint. Older security detection rules package versions
contain saved object rule duplicates affecting the endpoint.\n\nHaving
`security_detection_engine` v`8.17.1` package installed
`/internal/detection_engine/prebuilt_rules/status` and
`/internal/detection_engine/prebuilt_rules/installation/_review`
endpoints return a different number of rules available to install.\n\n##
Details\n\nOlder `security_detection_engine` package versions contain
rule saved objects duplicates representing the latest version. For
example, `8.17.1` version has a rule `Microsoft 365 User Restricted from
Sending Email` with `rule_id` = `0136b315-b566-482f-866c-1d8e2477ba16`
and the latest version `206`. Since a package may contain multiple
historical rule versions it sticks to the following format
`<rule_id>_<version>` where `<rule_id>` is the unique rule's UUID and
`<version>` it's version. Some older package versions omit `<version>`
for the latest rule version. `Microsoft 365 User Restricted from Sending
Email` rule mentioned above has two equal assets corresponding to the
latest version with the only difference in the saved object id
`0136b315-b566-482f-866c-1d8e2477ba16` and
`0136b315-b566-482f-866c-1d8e2477ba16_206`.\n\nPrebuilt rules preview
endpoint was designed to handle `<rule_id>_<version>` format only.
Consequently, it improperly handles older prebuilt rules package
version.\n\nThis bug manifested in
https://github.com/elastic/kibana/pull/217544 where
`security_detection_engine` version has been bumped to `8.18.1`. It
resulted in a failed integration test. Further investigation has shown
that the test installs an older package version `8.17.1` to assert
prebuilt rules upgrade workflow works correctly.\n\nThe fix is
implemented in `PrebuiltRuleAssetsClient.fetchAssetsByVersion()` by
using `Map` to deduplicate prebuilt rule
assets.","sha":"87f8274f4160f4d94f25d19f7d71ec4c35f4431e"}},"sourceBranch":"main","suggestedTargetBranches":["8.x","8.18","9.0"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/218123","number":218123,"mergeCommit":{"message":"[Security
Solution] Suppress prebuilt rule SO duplicates in review install
endpoint (#218123)\n\n## Summary\n\nThis PR makes sure a buggy
`security_detection_engine` package doesn't affect a preview
installation endpoint. Older security detection rules package versions
contain saved object rule duplicates affecting the endpoint.\n\nHaving
`security_detection_engine` v`8.17.1` package installed
`/internal/detection_engine/prebuilt_rules/status` and
`/internal/detection_engine/prebuilt_rules/installation/_review`
endpoints return a different number of rules available to install.\n\n##
Details\n\nOlder `security_detection_engine` package versions contain
rule saved objects duplicates representing the latest version. For
example, `8.17.1` version has a rule `Microsoft 365 User Restricted from
Sending Email` with `rule_id` = `0136b315-b566-482f-866c-1d8e2477ba16`
and the latest version `206`. Since a package may contain multiple
historical rule versions it sticks to the following format
`<rule_id>_<version>` where `<rule_id>` is the unique rule's UUID and
`<version>` it's version. Some older package versions omit `<version>`
for the latest rule version. `Microsoft 365 User Restricted from Sending
Email` rule mentioned above has two equal assets corresponding to the
latest version with the only difference in the saved object id
`0136b315-b566-482f-866c-1d8e2477ba16` and
`0136b315-b566-482f-866c-1d8e2477ba16_206`.\n\nPrebuilt rules preview
endpoint was designed to handle `<rule_id>_<version>` format only.
Consequently, it improperly handles older prebuilt rules package
version.\n\nThis bug manifested in
https://github.com/elastic/kibana/pull/217544 where
`security_detection_engine` version has been bumped to `8.18.1`. It
resulted in a failed integration test. Further investigation has shown
that the test installs an older package version `8.17.1` to assert
prebuilt rules upgrade workflow works correctly.\n\nThe fix is
implemented in `PrebuiltRuleAssetsClient.fetchAssetsByVersion()` by
using `Map` to deduplicate prebuilt rule
assets.","sha":"87f8274f4160f4d94f25d19f7d71ec4c35f4431e"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"9.0","label":"v9.0.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Maxim Palenov <maxim.palenov@elastic.co>

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { uniqBy } from 'lodash';
 import type {
   AggregationsMultiBucketAggregateBase,
   AggregationsTopHitsAggregate,
@@ -143,6 +144,9 @@ export const createPrebuiltRuleAssetsClient = (
           .map((v) => `(${attr}.rule_id: ${v.rule_id} AND ${attr}.version: ${v.version})`)
           .join(' OR ');
 
+        // Usage of savedObjectsClient.bulkGet() is ~25% more performant and
+        // simplifies deduplication but too many tests get broken.
+        // See https://github.com/elastic/kibana/issues/218198
         const findResult = await savedObjectsClient.find<PrebuiltRuleAsset>({
           type: PREBUILT_RULE_ASSETS_SO_TYPE,
           filter,
@@ -150,7 +154,11 @@ export const createPrebuiltRuleAssetsClient = (
         });
 
         const ruleAssets = findResult.saved_objects.map((so) => so.attributes);
-        return validatePrebuiltRuleAssets(ruleAssets);
+        // Rule assets may have duplicates we have to get rid of.
+        // In particular prebuilt rule assets package v8.17.1 has duplicates.
+        const uniqueRuleAssets = uniqBy(ruleAssets, 'rule_id');
+
+        return validatePrebuiltRuleAssets(uniqueRuleAssets);
       });
     },
   };