[8.18] [Entity Analytics] Consider Closed alerts for Risk Scoring (#193667) (#210941)

# Backport This will backport the following commits from `main` to `8.18`: - [[Entity Analytics] Consider Closed alerts for Risk Scoring (#193667)](https://github.com/elastic/kibana/pull/193667)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Abhishek Bhatia <117628830+abhishekbhatia1710@users.noreply.github.com>
2025-04-23 09:19:04 -04:00 · 2025-02-12 23:50:13 -06:00 · 2025-02-12 23:50:13 -06:00 · 930465848e
commit 930465848e
parent 6a55095236
11 changed files with 150 additions and 7 deletions
--- a/packages/kbn-check-mappings-update-cli/current_fields.json
+++ b/packages/kbn-check-mappings-update-cli/current_fields.json
@ -887,6 +887,7 @@
    "alertSampleSizePerShard",
    "dataViewId",
    "enabled",
+    "excludeAlertStatuses",
    "filter",
    "identifierType",
    "interval",
--- a/packages/kbn-check-mappings-update-cli/current_mappings.json
+++ b/packages/kbn-check-mappings-update-cli/current_mappings.json
@ -2945,6 +2945,9 @@
      "enabled": {
        "type": "boolean"
      },
+      "excludeAlertStatuses": {
+        "type": "keyword"
+      },
      "filter": {
        "dynamic": false,
        "properties": {}
--- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
+++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@ -148,7 +148,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
        "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352",
        "product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c",
        "query": "501bece68f26fe561286a488eabb1a8ab12f1137",
-        "risk-engine-configuration": "aea0c371a462e6d07c3ceb3aff11891b47feb09d",
+        "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb",
        "rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2",
        "sample-data-telemetry": "37441b12f5b0159c2d6d5138a494c9f440e950b5",
        "search": "0aa6eefb37edd3145be340a8b67779c2ca578b22",
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts
@ -55,6 +55,12 @@ export const RiskScoresPreviewRequest = z.object({
   */
  range: DateRange.optional(),
  weights: RiskScoreWeights.optional(),
+  /**
+   * A list of alert statuses to exclude from the risk score calculation. If unspecified, all alert statuses are included.
+   */
+  excludeAlertStatuses: z
+    .array(z.enum(['open', 'closed', 'in-progress', 'acknowledged']))
+    .optional(),
 });

 export type RiskScoresPreviewResponse = z.infer<typeof RiskScoresPreviewResponse>;
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml
@ -58,6 +58,17 @@ components:
          description: Defines the time period over which scores will be evaluated. If unspecified, a range of `[now, now-30d]` will be used.
        weights:
          $ref: '../common/common.schema.yaml#/components/schemas/RiskScoreWeights'
+        excludeAlertStatuses:
+          description: A list of alert statuses to exclude from the risk score calculation. If unspecified, all alert statuses are included.
+          type: array
+          items:
+            type: string
+            enum:
+              - open
+              - closed
+              - in-progress
+              - acknowledged
+          

    RiskScoresPreviewResponse:
      type: object
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/saved_object/risk_engine_configuration_type.ts
@ -45,6 +45,9 @@ export const riskEngineConfigurationTypeMappings: SavedObjectsType['mappings'] =
        },
      },
    },
+    excludeAlertStatuses: {
+      type: 'keyword',
+    },
  },
 };

@ -59,6 +62,28 @@ const version1: SavedObjectsModelVersion = {
  ],
 };

+const version2: SavedObjectsModelVersion = {
+  changes: [
+    {
+      type: 'mappings_addition',
+      addedMappings: {
+        excludeAlertStatuses: { type: 'keyword' },
+      },
+    },
+    {
+      type: 'data_backfill',
+      backfillFn: (document) => {
+        return {
+          attributes: {
+            ...document.attributes,
+            excludeAlertStatuses: document.attributes.excludeAlertStatuses || ['closed'],
+          },
+        };
+      },
+    },
+  ],
+};
+
 export const riskEngineConfigurationType: SavedObjectsType = {
  name: riskEngineConfigurationTypeName,
  indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
@ -67,5 +92,6 @@ export const riskEngineConfigurationType: SavedObjectsType = {
  mappings: riskEngineConfigurationTypeMappings,
  modelVersions: {
    1: version1,
+    2: version2,
  },
 };
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts
@ -14,6 +14,8 @@ import { calculateRiskScoresMock } from './calculate_risk_scores.mock';
 import { mockGlobalState } from '../../../../public/common/mock';
 import { EntityType } from '../../../../common/search_strategy';

+import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
+
 describe('calculateRiskScores()', () => {
  let params: Parameters<typeof calculateRiskScores>[0];
  let esClient: ElasticsearchClient;
@ -149,6 +151,41 @@ describe('calculateRiskScores()', () => {
        );
      });
    });
+
+    describe('excludeAlertStatuses', () => {
+      it('should not add the filter when excludeAlertStatuses is empty', async () => {
+        params = { ...params, excludeAlertStatuses: [] };
+        await calculateRiskScores(params);
+        expect(
+          (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter
+        ).toEqual(
+          expect.not.arrayContaining([
+            {
+              bool: {
+                must_not: { terms: { [ALERT_WORKFLOW_STATUS]: params.excludeAlertStatuses } },
+              },
+            },
+          ])
+        );
+      });
+
+      it('should add the filter when excludeAlertStatuses is not empty', async () => {
+        esClient.search as jest.Mock;
+        params = { ...params, excludeAlertStatuses: ['closed'] };
+        await calculateRiskScores(params);
+        expect(
+          (esClient.search as jest.Mock).mock.calls[0][0].query.function_score.query.bool.filter
+        ).toEqual(
+          expect.arrayContaining([
+            {
+              bool: {
+                must_not: { terms: { [ALERT_WORKFLOW_STATUS]: params.excludeAlertStatuses } },
+              },
+            },
+          ])
+        );
+      });
+    });
  });

  describe('outputs', () => {
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts
@ -98,7 +98,7 @@ const formatForResponse = ({
  };
 };

-const filterFromRange = (range: CalculateScoresParams['range']): QueryDslQueryContainer => ({
+export const filterFromRange = (range: CalculateScoresParams['range']): QueryDslQueryContainer => ({
  range: { '@timestamp': { lt: range.end, gte: range.start } },
 });

@ -221,6 +221,7 @@ export const calculateRiskScores = async ({
  weights,
  alertSampleSizePerShard = 10_000,
  experimentalFeatures,
+  excludeAlertStatuses = [],
 }: {
  assetCriticalityService: AssetCriticalityService;
  esClient: ElasticsearchClient;
@ -230,11 +231,12 @@ export const calculateRiskScores = async ({
  withSecuritySpan('calculateRiskScores', async () => {
    const now = new Date().toISOString();
    const scriptedMetricPainless = await getPainlessScripts();
-    const filter = [
-      filterFromRange(range),
-      { bool: { must_not: { term: { [ALERT_WORKFLOW_STATUS]: 'closed' } } } },
-      { exists: { field: ALERT_RISK_SCORE } },
-    ];
+    const filter = [filterFromRange(range), { exists: { field: ALERT_RISK_SCORE } }];
+    if (excludeAlertStatuses.length > 0) {
+      filter.push({
+        bool: { must_not: { terms: { [ALERT_WORKFLOW_STATUS]: excludeAlertStatuses } } },
+      });
+    }
    if (!isEmpty(userFilter)) {
      filter.push(userFilter as QueryDslQueryContainer);
    }
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/preview.ts
@ -66,6 +66,7 @@ export const riskScorePreviewRoute = (
          filter,
          range: userRange,
          weights,
+          excludeAlertStatuses,
        } = request.body;

        const entityAnalyticsConfig = await riskScoreService.getConfigurationWithDefaults(
@ -96,6 +97,7 @@ export const riskScorePreviewRoute = (
            runtimeMappings,
            weights,
            alertSampleSizePerShard,
+            excludeAlertStatuses,
          });

          securityContext.getAuditLogger()?.log({
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts
@ -94,6 +94,7 @@ export interface CalculateScoresParams {
  runtimeMappings: MappingRuntimeFields;
  weights?: RiskScoreWeights;
  alertSampleSizePerShard?: number;
+  excludeAlertStatuses?: string[];
 }

 export interface CalculateAndPersistScoresParams {
--- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_preview.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_preview.ts
@ -257,6 +257,60 @@ export default ({ getService }: FtrProviderContext): void => {
            },
          ]);
        });
+
+        it('calculates risk from 5 alerts, all in closed state, all for the same host', async () => {
+          const documentId = uuidv4();
+          const doc = buildDocument(
+            { host: { name: 'host-1' }, kibana: { alert: { workflow_status: 'closed' } } },
+            documentId
+          );
+          await indexListOfDocuments(Array(10).fill(doc));
+
+          const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, {
+            alerts: 5,
+          });
+
+          expect(sanitizeScores(body.scores.host!)).to.eql([
+            {
+              calculated_level: 'Unknown',
+              calculated_score: 41.90206636025764,
+              calculated_score_norm: 16.163426307767953,
+              category_1_count: 10,
+              category_1_score: 16.163426307767953,
+              id_field: 'host.name',
+              id_value: 'host-1',
+            },
+          ]);
+        });
+        it('calculates risk from 10 alerts, some in closed state, some in open state, all for the same host', async () => {
+          const documentId = uuidv4();
+          const docStatusClosed = buildDocument(
+            { host: { name: 'host-1' }, kibana: { alert: { workflow_status: 'closed' } } },
+            documentId
+          );
+          const docStatusOpen = buildDocument(
+            { host: { name: 'host-1' }, kibana: { alert: { workflow_status: 'open' } } },
+            documentId
+          );
+          await indexListOfDocuments(Array(5).fill(docStatusClosed));
+          await indexListOfDocuments(Array(5).fill(docStatusOpen));
+
+          const body = await getRiskScoreAfterRuleCreationAndExecution(documentId, {
+            alerts: 10,
+          });
+
+          expect(sanitizeScores(body.scores.host!)).to.eql([
+            {
+              calculated_level: 'Unknown',
+              calculated_score: 41.90206636025764,
+              calculated_score_norm: 16.163426307767953,
+              category_1_count: 10,
+              category_1_score: 16.163426307767953,
+              id_field: 'host.name',
+              id_value: 'host-1',
+            },
+          ]);
+        });
      });

      context('with a rule generating alerts with risk_score of 100', () => {