Added more request validation to entity store enablement (#212657)

# Purpose

This change introduces new validations that ensure no loss of data is
possible if a user accidentally sets the Security Entity Store enrich
policy execution interval to a value that “doesn’t play nicely” with the
lookback period value.

The specific logic (greater than or equal to half the value) was chosen
to not only ensure no loss of data, but also provide extra resiliency in
case of a failed enrich policy execution.

(Note that this is not considered a breaking change, as the parameters
are not yet available on any version of Elastic, including Serverless.)

# How to test

1. Load appropriate entity log data to your Kibana instance (for
example, using the
[security-documents-generator](https://github.com/elastic/security-documents-generator))
2. Navigate to the Developer console
3. Attempt to enable the Entity Store via the /enable or /init routes
(examples below), and pass in values that are expected to error. For
example, “lookbackPeriod”: “24h” and “enrichPolicyExecutionInterval”:
“24h” should fail, because of the validation logic
4. Expect results similar to those shown below, specifically a 400
error, or else a success message

<img width="1902" alt="Screenshot 2025-02-27 at 12 57 45 AM"
src="https://github.com/user-attachments/assets/a7f4b0fb-9899-4e00-a0ae-d172245bd506"
/>
<img width="1909" alt="Screenshot 2025-02-27 at 12 58 06 AM"
src="https://github.com/user-attachments/assets/372acde2-9d7b-4c75-8596-af8374088f79"
/>

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

oas_docs/output/kibana.serverless.yaml

@@ -11196,6 +11196,8 @@ paths:
                   succeeded:
                     type: boolean
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize the Entity Store
       tags:
         - Security Entity Analytics API
@@ -11333,6 +11335,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize an Entity Engine
       tags:
         - Security Entity Analytics API
@@ -56805,7 +56809,7 @@ components:
         - dsl
         - response
     Security_Entity_Analytics_API_Interval:
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
       example: 1h
       pattern: ^[1-9]\d*[smh]$
       type: string

oas_docs/output/kibana.yaml

@@ -13213,6 +13213,8 @@ paths:
                   succeeded:
                     type: boolean
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize the Entity Store
       tags:
         - Security Entity Analytics API
@@ -13346,6 +13348,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize an Entity Engine
       tags:
         - Security Entity Analytics API
@@ -64387,7 +64391,7 @@ components:
         - dsl
         - response
     Security_Entity_Analytics_API_Interval:
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
       example: 1h
       pattern: ^[1-9]\d*[smh]$
       type: string

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -129,7 +129,7 @@ export const InspectQuery = z.object({
 });
 
 /**
- * Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+ * Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
  */
 export type Interval = z.infer<typeof Interval>;
 export const Interval = z.string().regex(/^[1-9]\d*[smh]$/);

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -194,6 +194,6 @@ components:
         - response
     Interval:
       type: string
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
       pattern: '^[1-9]\d*[smh]$' # any number except zero followed by one of the suffixes 's', 'm', 'h'
       example: '1h'

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml

@@ -74,3 +74,5 @@ paths:
                     type: array
                     items:
                       $ref: './common.schema.yaml#/components/schemas/EngineDescriptor'
+        '400':
+          description: Invalid request

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml

@@ -62,8 +62,6 @@ paths:
                 docsPerSecond:
                   type: integer
                   description: The number of documents per second to process.
-
-   
       responses:
         '200':
           description: Successful response
@@ -71,4 +69,5 @@ paths:
             application/json:
               schema:
                 $ref: '../common.schema.yaml#/components/schemas/EngineDescriptor'
-                  
+        '400':
+          description: Invalid request

x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -374,6 +374,8 @@ paths:
                   succeeded:
                     type: boolean
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize the Entity Store
       tags:
         - Security Entity Analytics API
@@ -509,6 +511,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/EngineDescriptor'
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize an Entity Engine
       tags:
         - Security Entity Analytics API
@@ -1321,7 +1325,8 @@ components:
     Interval:
       description: >-
         Interval in which enrich policy runs. For example, `"1h"` means the rule
-        runs every hour.
+        runs every hour. Must be less than or equal to half the duration of the
+        lookback period,
       example: 1h
       pattern: ^[1-9]\d*[smh]$
       type: string

x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -374,6 +374,8 @@ paths:
                   succeeded:
                     type: boolean
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize the Entity Store
       tags:
         - Security Entity Analytics API
@@ -509,6 +511,8 @@ paths:
               schema:
                 $ref: '#/components/schemas/EngineDescriptor'
           description: Successful response
+        '400':
+          description: Invalid request
       summary: Initialize an Entity Engine
       tags:
         - Security Entity Analytics API
@@ -1321,7 +1325,8 @@ components:
     Interval:
       description: >-
         Interval in which enrich policy runs. For example, `"1h"` means the rule
-        runs every hour.
+        runs every hour. Must be less than or equal to half the duration of the
+        lookback period,
       example: 1h
       pattern: ^[1-9]\d*[smh]$
       type: string

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/enablement.ts

@@ -8,13 +8,13 @@
 import type { IKibanaResponse, Logger } from '@kbn/core/server';
 import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
 import { transformError } from '@kbn/securitysolution-es-utils';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 
+import { buildInitRequestBodyValidation } from './validation';
 import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
-import { InitEntityStoreRequestBody } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
 import { checkAndInitAssetCriticalityResources } from '../../asset_criticality/check_and_init_asset_criticality_resources';
+import { InitEntityStoreRequestBody } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 
 export const enableEntityStoreRoute = (
   router: EntityAnalyticsRoutesDeps['router'],
@@ -36,7 +36,7 @@ export const enableEntityStoreRoute = (
         version: API_VERSIONS.public.v1,
         validate: {
           request: {
-            body: buildRouteValidationWithZod(InitEntityStoreRequestBody),
+            body: buildInitRequestBodyValidation(InitEntityStoreRequestBody),
           },
         },
       },

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/init.ts

@@ -19,6 +19,7 @@ import {
 import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
 import { checkAndInitAssetCriticalityResources } from '../../asset_criticality/check_and_init_asset_criticality_resources';
+import { buildInitRequestBodyValidation } from './validation';
 
 export const initEntityEngineRoute = (
   router: EntityAnalyticsRoutesDeps['router'],
@@ -41,7 +42,7 @@ export const initEntityEngineRoute = (
         validate: {
           request: {
             params: buildRouteValidationWithZod(InitEntityEngineRequestParams),
-            body: buildRouteValidationWithZod(InitEntityEngineRequestBody),
+            body: buildInitRequestBodyValidation(InitEntityEngineRequestBody),
           },
         },
       },

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.test.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { validateInitializationRequestBody } from './validation';
+import type { InitEntityEngineRequestBody } from '../../../../../common/api/entity_analytics';
+import { BadRequestError } from '@kbn/securitysolution-es-utils';
+
+describe('entity store initialization request validation', () => {
+  const defaultRequestBody: InitEntityEngineRequestBody = {
+    fieldHistoryLength: 10,
+    timestampField: '@timestamp',
+    lookbackPeriod: '24h',
+    timeout: '180s',
+    frequency: '1m',
+    delay: '1m',
+    enrichPolicyExecutionInterval: '1h',
+  };
+  it('should allow the default values (24 hour lookback period, 1 hour enrich policy interval)', () => {
+    expect(validateInitializationRequestBody(defaultRequestBody)).toBeUndefined();
+  });
+  it('should allow the enrich policy interval to be exactly half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '12h',
+      })
+    ).toBeUndefined();
+  });
+  it('should allow the enrich policy interval to be barely less than half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '11h',
+      })
+    ).toBeUndefined();
+  });
+  it('should not allow the lookback period and enrich policy interval to be the same', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '1h',
+        enrichPolicyExecutionInterval: '1h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+  it('should not allow the enrich policy interval to be greater than the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '1h',
+        enrichPolicyExecutionInterval: '2h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+  it('should not allow the enrich policy interval to be more than half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '13h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { parseDuration } from '@kbn/alerting-plugin/common';
+import { BadRequestError } from '@kbn/securitysolution-es-utils';
+import type { RouteValidationFunction, RouteValidationResultFactory } from '@kbn/core-http-server';
+import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
+import type { TypeOf, ZodType } from '@kbn/zod';
+import type { InitEntityEngineRequestBody } from '../../../../../common/api/entity_analytics';
+
+export const buildInitRequestBodyValidation =
+  <ZodSchema extends ZodType, Type = TypeOf<ZodSchema>>(
+    schema: ZodSchema
+  ): RouteValidationFunction<Type> =>
+  (inputValue: unknown, validationResultFactory: RouteValidationResultFactory) => {
+    const zodValidationResult = buildRouteValidationWithZod(schema)(
+      inputValue,
+      validationResultFactory
+    );
+    if (zodValidationResult.error) return zodValidationResult;
+    const additionalValidationResult = validateInitializationRequestBody(zodValidationResult.value);
+    if (additionalValidationResult)
+      return validationResultFactory.badRequest(additionalValidationResult);
+    return zodValidationResult;
+  };
+
+/**
+ * Validations performed:
+ * - Ensures that the enrich policy execution interval is less than or equal to half the duration of the lookback period,
+ * as the execution policy must run successfully at least once within the lookback period in order to ensure no loss of
+ * data
+ */
+export const validateInitializationRequestBody = (requestBody: InitEntityEngineRequestBody) => {
+  const { lookbackPeriod, enrichPolicyExecutionInterval } = requestBody;
+  if (!lookbackPeriod || !enrichPolicyExecutionInterval) return;
+  const lookbackPeriodMillis = parseDuration(lookbackPeriod);
+  const enrichPolicyExecutionIntervalMillis = parseDuration(enrichPolicyExecutionInterval);
+  if (enrichPolicyExecutionIntervalMillis > lookbackPeriodMillis / 2) {
+    return new BadRequestError(
+      'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+    );
+  }
+};