[Security] Adds pre-packaged rule updates through the "Prebuilt Security Detection Rules" Fleet integration (#96698)

* Make the prepackaged rules functions async
* Fix type for getPrepackagedRules mock
* Install updates from saved objects & FS
* Mock getLatestPrepackagedRules instead of getPrepackagedRules
* Cleanup ruleAssetSavedObjectsClientFactory.all
* Fix comment for "most recent version"
* Switch to ruleMap.get() for less typescript errors
* Remove unneeded constants
* Fix SO.attributes sig and use custom validation

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts

@@ -25,7 +25,7 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo
 
 jest.mock('../../rules/get_prepackaged_rules', () => {
   return {
-    getPrepackagedRules: (): AddPrepackagedRulesSchemaDecoded[] => {
+    getLatestPrepackagedRules: async (): Promise<AddPrepackagedRulesSchemaDecoded[]> => {
       return [
         {
           author: ['Elastic'],

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts

@@ -25,12 +25,13 @@ import { SetupPlugins } from '../../../../plugin';
 import { buildFrameworkRequest } from '../../../timeline/utils/common';
 
 import { getIndexExists } from '../../index/get_index_exists';
-import { getPrepackagedRules } from '../../rules/get_prepackaged_rules';
+import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules';
 import { installPrepackagedRules } from '../../rules/install_prepacked_rules';
 import { updatePrepackagedRules } from '../../rules/update_prepacked_rules';
 import { getRulesToInstall } from '../../rules/get_rules_to_install';
 import { getRulesToUpdate } from '../../rules/get_rules_to_update';
 import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules';
+import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client';
 
 import { transformError, buildSiemResponse } from '../utils';
 import { AlertsClient } from '../../../../../../alerting/server';
@@ -110,7 +111,7 @@ export const createPrepackagedRules = async (
   const savedObjectsClient = context.core.savedObjects.client;
   const exceptionsListClient =
     context.lists != null ? context.lists.getExceptionListClient() : exceptionsClient;
-
+  const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient);
   if (!siemClient || !alertsClient) {
     throw new PrepackagedRulesError('', 404);
   }
@@ -120,10 +121,10 @@ export const createPrepackagedRules = async (
     await exceptionsListClient.createEndpointList();
   }
 
-  const rulesFromFileSystem = getPrepackagedRules();
+  const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient);
   const prepackagedRules = await getExistingPrepackagedRules({ alertsClient });
-  const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules);
-  const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules);
+  const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules);
+  const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules);
   const signalsIndex = siemClient.getSignalsIndex();
   if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) {
     const signalsIndexExists = await getIndexExists(esClient.asCurrentUser, signalsIndex);

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts

@@ -23,7 +23,7 @@ import {
 
 jest.mock('../../rules/get_prepackaged_rules', () => {
   return {
-    getPrepackagedRules: () => {
+    getLatestPrepackagedRules: async () => {
       return [
         {
           rule_id: 'rule-1',

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts

@@ -13,11 +13,12 @@ import {
 import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../../common/constants';
 import { transformError, buildSiemResponse } from '../utils';
-import { getPrepackagedRules } from '../../rules/get_prepackaged_rules';
 import { getRulesToInstall } from '../../rules/get_rules_to_install';
 import { getRulesToUpdate } from '../../rules/get_rules_to_update';
 import { findRules } from '../../rules/find_rules';
+import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules';
 import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules';
+import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client';
 import { buildFrameworkRequest } from '../../../timeline/utils/common';
 import { ConfigType } from '../../../../config';
 import { SetupPlugins } from '../../../../plugin';
@@ -40,15 +41,17 @@ export const getPrepackagedRulesStatusRoute = (
       },
     },
     async (context, request, response) => {
+      const savedObjectsClient = context.core.savedObjects.client;
       const siemResponse = buildSiemResponse(response);
       const alertsClient = context.alerting?.getAlertsClient();
+      const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient);
 
       if (!alertsClient) {
         return siemResponse.error({ statusCode: 404 });
       }
 
       try {
-        const rulesFromFileSystem = getPrepackagedRules();
+        const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient);
         const customRules = await findRules({
           alertsClient,
           perPage: 1,
@@ -61,8 +64,8 @@ export const getPrepackagedRulesStatusRoute = (
         const frameworkRequest = await buildFrameworkRequest(context, security, request);
         const prepackagedRules = await getExistingPrepackagedRules({ alertsClient });
 
-        const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules);
-        const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules);
+        const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules);
+        const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules);
         const prepackagedTimelineStatus = await checkTimelinesStatus(frameworkRequest);
         const [validatedprepackagedTimelineStatus] = validate(
           prepackagedTimelineStatus,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts

@@ -41,8 +41,10 @@ describe('get_existing_prepackaged_rules', () => {
   });
 
   test('should throw an exception with a message having rule_id and name in it', () => {
-    // @ts-expect-error intentionally invalid argument
-    expect(() => getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }])).toThrow(
+    expect(() =>
+      // @ts-expect-error intentionally invalid argument
+      getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }])
+    ).toThrow(
       'name: "rule name", rule_id: "id-123" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n  "name": "rule name",\n  "rule_id": "id-123"\n}'
     );
   });

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts

@@ -19,6 +19,9 @@ import { BadRequestError } from '../errors/bad_request_error';
 
 // TODO: convert rules files to TS and add explicit type definitions
 import { rawRules } from './prepackaged_rules';
+import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client';
+import { IRuleAssetSOAttributes } from './types';
+import { SavedObjectAttributes } from '../../../../../../../src/core/types';
 
 /**
  * Validate the rules from the file system and throw any errors indicating to the developer
@@ -52,7 +55,70 @@ export const validateAllPrepackagedRules = (
   });
 };
 
+/**
+ * Validate the rules from Saved Objects created by Fleet.
+ */
+export const validateAllRuleSavedObjects = (
+  rules: Array<IRuleAssetSOAttributes & SavedObjectAttributes>
+): AddPrepackagedRulesSchemaDecoded[] => {
+  return rules.map((rule) => {
+    const decoded = addPrepackagedRulesSchema.decode(rule);
+    const checked = exactCheck(rule, decoded);
+
+    const onLeft = (errors: t.Errors): AddPrepackagedRulesSchemaDecoded => {
+      const ruleName = rule.name ? rule.name : '(rule name unknown)';
+      const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)';
+      throw new BadRequestError(
+        `name: "${ruleName}", rule_id: "${ruleId}" within the security-rule saved object ` +
+          `is not a valid detection engine rule. Expect the system ` +
+          `to not work with pre-packaged rules until this rule is fixed ` +
+          `or the file is removed. Error is: ${formatErrors(
+            errors
+          ).join()}, Full rule contents are:\n${JSON.stringify(rule, null, 2)}`
+      );
+    };
+
+    const onRight = (schema: AddPrepackagedRulesSchema): AddPrepackagedRulesSchemaDecoded => {
+      return schema as AddPrepackagedRulesSchemaDecoded;
+    };
+    return pipe(checked, fold(onLeft, onRight));
+  });
+};
+
+/**
+ * Retrieve and validate rules that were installed from Fleet as saved objects.
+ */
+export const getFleetInstalledRules = async (
+  client: RuleAssetSavedObjectsClient
+): Promise<AddPrepackagedRulesSchemaDecoded[]> => {
+  const fleetResponse = await client.all();
+  const fleetRules = fleetResponse.map((so) => so.attributes);
+  return validateAllRuleSavedObjects(fleetRules);
+};
+
 export const getPrepackagedRules = (
   // @ts-expect-error mock data is too loosely typed
   rules: AddPrepackagedRulesSchema[] = rawRules
-): AddPrepackagedRulesSchemaDecoded[] => validateAllPrepackagedRules(rules);
+): AddPrepackagedRulesSchemaDecoded[] => {
+  return validateAllPrepackagedRules(rules);
+};
+
+export const getLatestPrepackagedRules = async (
+  client: RuleAssetSavedObjectsClient
+): Promise<AddPrepackagedRulesSchemaDecoded[]> => {
+  // build a map of the most recent version of each rule
+  const prepackaged = getPrepackagedRules();
+  const ruleMap = new Map(prepackaged.map((r) => [r.rule_id, r]));
+
+  // check the rules installed via fleet and create/update if the version is newer
+  const fleetRules = await getFleetInstalledRules(client);
+  const fleetUpdates = fleetRules.filter((r) => {
+    const rule = ruleMap.get(r.rule_id);
+    return rule == null || rule.version < r.version;
+  });
+
+  // add the new or updated rules to the map
+  fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r));
+
+  return Array.from(ruleMap.values());
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  SavedObjectsClientContract,
+  SavedObjectsFindOptions,
+  SavedObjectsFindResponse,
+} from '../../../../../../../src/core/server';
+import { ruleAssetSavedObjectType } from '../rules/saved_object_mappings';
+import { IRuleAssetSavedObject } from '../rules/types';
+
+const DEFAULT_PAGE_SIZE = 100;
+
+export interface RuleAssetSavedObjectsClient {
+  find: (
+    options?: Omit<SavedObjectsFindOptions, 'type'>
+  ) => Promise<SavedObjectsFindResponse<IRuleAssetSavedObject>>;
+  all: () => Promise<IRuleAssetSavedObject[]>;
+}
+
+export const ruleAssetSavedObjectsClientFactory = (
+  savedObjectsClient: SavedObjectsClientContract
+): RuleAssetSavedObjectsClient => {
+  return {
+    find: (options) =>
+      savedObjectsClient.find<IRuleAssetSavedObject>({
+        ...options,
+        type: ruleAssetSavedObjectType,
+      }),
+    all: async () => {
+      const finder = savedObjectsClient.createPointInTimeFinder({
+        perPage: DEFAULT_PAGE_SIZE,
+        type: ruleAssetSavedObjectType,
+      });
+      const responses: IRuleAssetSavedObject[] = [];
+      for await (const response of finder.find()) {
+        responses.push(...response.saved_objects.map((so) => so as IRuleAssetSavedObject));
+      }
+      await finder.close();
+      return responses;
+    },
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts

@@ -164,6 +164,19 @@ export interface IRuleStatusFindType {
   saved_objects: IRuleStatusSavedObject[];
 }
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export interface IRuleAssetSOAttributes extends Record<string, any> {
+  rule_id: string | null | undefined;
+  version: string | null | undefined;
+  name: string | null | undefined;
+}
+
+export interface IRuleAssetSavedObject {
+  type: string;
+  id: string;
+  attributes: IRuleAssetSOAttributes & SavedObjectAttributes;
+}
+
 export interface HapiReadableStream extends Readable {
   hapi: {
     filename: string;