Added more request validation to entity store enablement (#212657)

# Purpose This change introduces new validations that ensure no loss of data is possible if a user accidentally sets the Security Entity Store enrich policy execution interval to a value that “doesn’t play nicely” with the lookback period value. The specific logic (greater than or equal to half the value) was chosen to not only ensure no loss of data, but also provide extra resiliency in case of a failed enrich policy execution. (Note that this is not considered a breaking change, as the parameters are not yet available on any version of Elastic, including Serverless.) # How to test 1. Load appropriate entity log data to your Kibana instance (for example, using the [security-documents-generator](https://github.com/elastic/security-documents-generator)) 2. Navigate to the Developer console 3. Attempt to enable the Entity Store via the /enable or /init routes (examples below), and pass in values that are expected to error. For example, “lookbackPeriod”: “24h” and “enrichPolicyExecutionInterval”: “24h” should fail, because of the validation logic 4. Expect results similar to those shown below, specifically a 400 error, or else a success message <img width="1902" alt="Screenshot 2025-02-27 at 12 57 45 AM" src="https://github.com/user-attachments/assets/a7f4b0fb-9899-4e00-a0ae-d172245bd506" /> <img width="1909" alt="Screenshot 2025-02-27 at 12 58 06 AM" src="https://github.com/user-attachments/assets/372acde2-9d7b-4c75-8596-af8374088f79" /> --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2025-06-28 11:05:39 -04:00 · 2025-03-19 18:31:31 -05:00 · 2025-03-19 18:31:31 -05:00 · 64743b3a82
commit 64743b3a82
parent 81f69713f3
12 changed files with 183 additions and 34 deletions
--- a/oas_docs/output/kibana.serverless.yaml
+++ b/oas_docs/output/kibana.serverless.yaml
@ -11196,6 +11196,8 @@ paths:
                  succeeded:
                    type: boolean
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize the Entity Store
      tags:
        - Security Entity Analytics API
@ -11333,6 +11335,8 @@ paths:
              schema:
                $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize an Entity Engine
      tags:
        - Security Entity Analytics API
@ -56805,7 +56809,7 @@ components:
        - dsl
        - response
    Security_Entity_Analytics_API_Interval:
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
      example: 1h
      pattern: ^[1-9]\d*[smh]$
      type: string
--- a/oas_docs/output/kibana.yaml
+++ b/oas_docs/output/kibana.yaml
@ -13213,6 +13213,8 @@ paths:
                  succeeded:
                    type: boolean
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize the Entity Store
      tags:
        - Security Entity Analytics API
@ -13346,6 +13348,8 @@ paths:
              schema:
                $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize an Entity Engine
      tags:
        - Security Entity Analytics API
@ -64387,7 +64391,7 @@ components:
        - dsl
        - response
    Security_Entity_Analytics_API_Interval:
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
      example: 1h
      pattern: ^[1-9]\d*[smh]$
      type: string
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@ -129,7 +129,7 @@ export const InspectQuery = z.object({
 });

 /**
- * Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+ * Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
 */
 export type Interval = z.infer<typeof Interval>;
 export const Interval = z.string().regex(/^[1-9]\d*[smh]$/);
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@ -194,6 +194,6 @@ components:
        - response
    Interval:
      type: string
-      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour.
+      description: Interval in which enrich policy runs. For example, `"1h"` means the rule runs every hour. Must be less than or equal to half the duration of the lookback period,
      pattern: '^[1-9]\d*[smh]$' # any number except zero followed by one of the suffixes 's', 'm', 'h'
      example: '1h'
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
@ -74,3 +74,5 @@ paths:
                    type: array
                    items:
                      $ref: './common.schema.yaml#/components/schemas/EngineDescriptor'
+        '400':
+          description: Invalid request
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml
@ -62,8 +62,6 @@ paths:
                docsPerSecond:
                  type: integer
                  description: The number of documents per second to process.
-
-   
      responses:
        '200':
          description: Successful response
@ -71,4 +69,5 @@ paths:
            application/json:
              schema:
                $ref: '../common.schema.yaml#/components/schemas/EngineDescriptor'
-                  
+        '400':
+          description: Invalid request
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@ -374,6 +374,8 @@ paths:
                  succeeded:
                    type: boolean
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize the Entity Store
      tags:
        - Security Entity Analytics API
@ -509,6 +511,8 @@ paths:
              schema:
                $ref: '#/components/schemas/EngineDescriptor'
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize an Entity Engine
      tags:
        - Security Entity Analytics API
@ -1321,7 +1325,8 @@ components:
    Interval:
      description: >-
        Interval in which enrich policy runs. For example, `"1h"` means the rule
-        runs every hour.
+        runs every hour. Must be less than or equal to half the duration of the
+        lookback period,
      example: 1h
      pattern: ^[1-9]\d*[smh]$
      type: string
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@ -374,6 +374,8 @@ paths:
                  succeeded:
                    type: boolean
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize the Entity Store
      tags:
        - Security Entity Analytics API
@ -509,6 +511,8 @@ paths:
              schema:
                $ref: '#/components/schemas/EngineDescriptor'
          description: Successful response
+        '400':
+          description: Invalid request
      summary: Initialize an Entity Engine
      tags:
        - Security Entity Analytics API
@ -1321,7 +1325,8 @@ components:
    Interval:
      description: >-
        Interval in which enrich policy runs. For example, `"1h"` means the rule
-        runs every hour.
+        runs every hour. Must be less than or equal to half the duration of the
+        lookback period,
      example: 1h
      pattern: ^[1-9]\d*[smh]$
      type: string
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/enablement.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/enablement.ts
@ -8,13 +8,13 @@
 import type { IKibanaResponse, Logger } from '@kbn/core/server';
 import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
 import { transformError } from '@kbn/securitysolution-es-utils';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';

+import { buildInitRequestBodyValidation } from './validation';
 import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
-import { InitEntityStoreRequestBody } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
 import { checkAndInitAssetCriticalityResources } from '../../asset_criticality/check_and_init_asset_criticality_resources';
+import { InitEntityStoreRequestBody } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';

 export const enableEntityStoreRoute = (
  router: EntityAnalyticsRoutesDeps['router'],
@ -36,7 +36,7 @@ export const enableEntityStoreRoute = (
        version: API_VERSIONS.public.v1,
        validate: {
          request: {
-            body: buildRouteValidationWithZod(InitEntityStoreRequestBody),
+            body: buildInitRequestBodyValidation(InitEntityStoreRequestBody),
          },
        },
      },
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/init.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/init.ts
@ -19,6 +19,7 @@ import {
 import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
 import { checkAndInitAssetCriticalityResources } from '../../asset_criticality/check_and_init_asset_criticality_resources';
+import { buildInitRequestBodyValidation } from './validation';

 export const initEntityEngineRoute = (
  router: EntityAnalyticsRoutesDeps['router'],
@ -41,7 +42,7 @@ export const initEntityEngineRoute = (
        validate: {
          request: {
            params: buildRouteValidationWithZod(InitEntityEngineRequestParams),
-            body: buildRouteValidationWithZod(InitEntityEngineRequestBody),
+            body: buildInitRequestBodyValidation(InitEntityEngineRequestBody),
          },
        },
      },
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.test.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.test.ts
@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { validateInitializationRequestBody } from './validation';
+import type { InitEntityEngineRequestBody } from '../../../../../common/api/entity_analytics';
+import { BadRequestError } from '@kbn/securitysolution-es-utils';
+
+describe('entity store initialization request validation', () => {
+  const defaultRequestBody: InitEntityEngineRequestBody = {
+    fieldHistoryLength: 10,
+    timestampField: '@timestamp',
+    lookbackPeriod: '24h',
+    timeout: '180s',
+    frequency: '1m',
+    delay: '1m',
+    enrichPolicyExecutionInterval: '1h',
+  };
+  it('should allow the default values (24 hour lookback period, 1 hour enrich policy interval)', () => {
+    expect(validateInitializationRequestBody(defaultRequestBody)).toBeUndefined();
+  });
+  it('should allow the enrich policy interval to be exactly half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '12h',
+      })
+    ).toBeUndefined();
+  });
+  it('should allow the enrich policy interval to be barely less than half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '11h',
+      })
+    ).toBeUndefined();
+  });
+  it('should not allow the lookback period and enrich policy interval to be the same', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '1h',
+        enrichPolicyExecutionInterval: '1h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+  it('should not allow the enrich policy interval to be greater than the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '1h',
+        enrichPolicyExecutionInterval: '2h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+  it('should not allow the enrich policy interval to be more than half the lookback period', () => {
+    expect(
+      validateInitializationRequestBody({
+        ...defaultRequestBody,
+        lookbackPeriod: '24h',
+        enrichPolicyExecutionInterval: '13h',
+      })
+    ).toEqual(
+      new BadRequestError(
+        'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+      )
+    );
+  });
+});
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/validation.ts
@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { parseDuration } from '@kbn/alerting-plugin/common';
+import { BadRequestError } from '@kbn/securitysolution-es-utils';
+import type { RouteValidationFunction, RouteValidationResultFactory } from '@kbn/core-http-server';
+import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
+import type { TypeOf, ZodType } from '@kbn/zod';
+import type { InitEntityEngineRequestBody } from '../../../../../common/api/entity_analytics';
+
+export const buildInitRequestBodyValidation =
+  <ZodSchema extends ZodType, Type = TypeOf<ZodSchema>>(
+    schema: ZodSchema
+  ): RouteValidationFunction<Type> =>
+  (inputValue: unknown, validationResultFactory: RouteValidationResultFactory) => {
+    const zodValidationResult = buildRouteValidationWithZod(schema)(
+      inputValue,
+      validationResultFactory
+    );
+    if (zodValidationResult.error) return zodValidationResult;
+    const additionalValidationResult = validateInitializationRequestBody(zodValidationResult.value);
+    if (additionalValidationResult)
+      return validationResultFactory.badRequest(additionalValidationResult);
+    return zodValidationResult;
+  };
+
+/**
+ * Validations performed:
+ * - Ensures that the enrich policy execution interval is less than or equal to half the duration of the lookback period,
+ * as the execution policy must run successfully at least once within the lookback period in order to ensure no loss of
+ * data
+ */
+export const validateInitializationRequestBody = (requestBody: InitEntityEngineRequestBody) => {
+  const { lookbackPeriod, enrichPolicyExecutionInterval } = requestBody;
+  if (!lookbackPeriod || !enrichPolicyExecutionInterval) return;
+  const lookbackPeriodMillis = parseDuration(lookbackPeriod);
+  const enrichPolicyExecutionIntervalMillis = parseDuration(enrichPolicyExecutionInterval);
+  if (enrichPolicyExecutionIntervalMillis > lookbackPeriodMillis / 2) {
+    return new BadRequestError(
+      'The enrich policy execution interval must be less than or equal to half the duration of the lookback period.'
+    );
+  }
+};