Entity Store Config - Lookback period (#206421)

## Summary This PR enables configurability of the lookbackPeriod for Security’s Entity Store through the enable API. ### Testing 1. Open the latest instance of Kibana. 2. Ensure both the Entity Store and Risk Score are enabled. 3. Navigate to Dev Tools. 4. Use the enable endpoint with lookbackPeriod, where the lookbackPeriod must be a number followed by one of the supported units (m, s, or h). ``` POST kbn:api/entity_store/enable { "lookbackPeriod": "72h" // example value } ``` 5. After enabling, use the status endpoint to confirm the configuration: ``` GET kbn:api/entity_store/status {} ``` 6. Post some documents, to check the entities available within the lookbackPeriod - confirm you can only see those within this period, and not outside of this period e.g. "5h" period, post with a timestamp of today (within 5 hours) and from yesterday and view results. ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Today", "entity_name": "LookbackPeriod Test Today", "timestamp": "2025-01-17T14:31:00Z" } ``` ``` POST lookback-period-test/_doc { "entity_id": "LookbackPeriod Test Yesterday", "entity_name": "LookbackPeriod Test Yesterday", "timestamp": "2025-01-16T14:31:00Z" } ``` 7. Can also go to entity store, check one of the transforms and check the JSON gte ranges from here too, they should show the new lookbackPeriod. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-23 09:19:04 -04:00 · 2025-01-24 21:59:44 +00:00 · 2025-01-24 21:59:44 +00:00 · 667040fbec
commit 667040fbec
parent b08eaa7d5a
14 changed files with 127 additions and 18 deletions
--- a/oas_docs/output/kibana.serverless.yaml
+++ b/oas_docs/output/kibana.serverless.yaml
@ -9735,6 +9735,11 @@ paths:
                  type: string
                indexPattern:
                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
        description: Schema for the entity store initialization
        required: true
      responses:
@ -50354,6 +50359,10 @@ components:
          type: string
        indexPattern:
          $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
        status:
          $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
        type:
--- a/oas_docs/output/kibana.yaml
+++ b/oas_docs/output/kibana.yaml
@ -11883,6 +11883,11 @@ paths:
                  type: string
                indexPattern:
                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
        description: Schema for the entity store initialization
        required: true
      responses:
@ -57206,6 +57211,10 @@ components:
          type: string
        indexPattern:
          $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
        status:
          $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
        type:
--- a/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts
+++ b/x-pack/platform/packages/shared/kbn-entities-schema/src/schema/common.ts
@ -44,7 +44,9 @@ export const docCountMetricSchema = z.object({
  filter: filterSchema,
 });

-export const durationSchema = z.string().regex(/^\d+[m|d|s|h]$/);
+export const durationSchema = z.string().regex(/^\d+[m|s|d|h]$/, {
+  message: 'Must be a number followed by one of the units: m, s, d or h',
+});

 export const durationSchemaWithMinimum = (minimumMinutes: number) =>
  durationSchema.refine(
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@ -36,6 +36,11 @@ export const EngineDescriptor = z.object({
  status: EngineStatus,
  filter: z.string().optional(),
  fieldHistoryLength: z.number().int(),
+  lookbackPeriod: z
+    .string()
+    .regex(/[smdh]$/)
+    .optional()
+    .default('24h'),
  error: z.object({}).optional(),
 });

--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@ -32,6 +32,10 @@ components:
          type: string
        fieldHistoryLength:
          type: integer
+        lookbackPeriod: 
+          type: string
+          default: 24h
+          pattern: '[smdh]$'
        error:
          type: object

--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
@ -24,6 +24,14 @@ export const InitEntityStoreRequestBody = z.object({
   * The number of historical values to keep for each field.
   */
  fieldHistoryLength: z.number().int().optional().default(10),
+  /**
+   * The lookback period for the entity store
+   */
+  lookbackPeriod: z
+    .string()
+    .regex(/[smdh]$/)
+    .optional()
+    .default('24h'),
  indexPattern: IndexPattern.optional(),
  filter: z.string().optional(),
  entityTypes: z.array(EntityType).optional(),
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
@ -23,6 +23,11 @@ paths:
                  type: integer
                  description: The number of historical values to keep for each field.
                  default: 10
+                lookbackPeriod: 
+                  type: string
+                  description: The lookback period for the entity store
+                  default: 24h
+                  pattern: '[smdh]$'
                indexPattern:
                  $ref: './common.schema.yaml#/components/schemas/IndexPattern'
                filter:
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@ -321,6 +321,11 @@ paths:
                  type: string
                indexPattern:
                  $ref: '#/components/schemas/IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
        description: Schema for the entity store initialization
        required: true
      responses:
@ -1010,6 +1015,10 @@ components:
          type: string
        indexPattern:
          $ref: '#/components/schemas/IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
        status:
          $ref: '#/components/schemas/EngineStatus'
        type:
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@ -321,6 +321,11 @@ paths:
                  type: string
                indexPattern:
                  $ref: '#/components/schemas/IndexPattern'
+                lookbackPeriod:
+                  default: 24h
+                  description: The lookback period for the entity store
+                  pattern: '[smdh]$'
+                  type: string
        description: Schema for the entity store initialization
        required: true
      responses:
@ -1010,6 +1015,10 @@ components:
          type: string
        indexPattern:
          $ref: '#/components/schemas/IndexPattern'
+        lookbackPeriod:
+          default: 24h
+          pattern: '[smdh]$'
+          type: string
        status:
          $ref: '#/components/schemas/EngineStatus'
        type:
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
@ -130,6 +130,18 @@ interface SearchEntitiesParams {
  sortOrder: SortOrder;
 }

+export const DEFAULT_INIT_ENTITY_STORE: InitEntityStoreRequestBody = {
+  indexPattern: '',
+  lookbackPeriod: '24h',
+  filter: '',
+  fieldHistoryLength: 10,
+  enrichPolicyExecutionInterval: DEFAULT_INTERVAL,
+};
+
+const DEFAULT_ENTITY_ENGINE: InitEntityEngineRequestBody & { lookbackPeriod?: string } = {
+  ...DEFAULT_INIT_ENTITY_STORE,
+};
+
 export class EntityStoreDataClient {
  private engineClient: EngineDescriptorClient;
  private assetCriticalityMigrationClient: AssetCriticalityMigrationClient;
@ -204,19 +216,25 @@ export class EntityStoreDataClient {
  }

  public async enable(
-    {
-      indexPattern = '',
-      filter = '',
-      fieldHistoryLength = 10,
-      entityTypes,
-      enrichPolicyExecutionInterval,
-    }: InitEntityStoreRequestBody,
+    requestBodyOverrides: Partial<InitEntityStoreRequestBody> = {},
    { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
  ): Promise<InitEntityStoreResponse> {
    if (!this.options.taskManager) {
      throw new Error('Task Manager is not available');
    }

+    const {
+      indexPattern,
+      lookbackPeriod,
+      filter,
+      fieldHistoryLength,
+      entityTypes,
+      enrichPolicyExecutionInterval,
+    } = {
+      ...DEFAULT_INIT_ENTITY_STORE,
+      ...requestBodyOverrides,
+    };
+
    // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
    const run = <T>(fn: () => Promise<T>) =>
      new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
@ -233,7 +251,13 @@ export class EntityStoreDataClient {
      run(() =>
        this.init(
          entity,
-          { indexPattern, filter, fieldHistoryLength, enrichPolicyExecutionInterval },
+          {
+            indexPattern,
+            lookbackPeriod,
+            filter,
+            fieldHistoryLength,
+            enrichPolicyExecutionInterval,
+          },
          { pipelineDebugMode }
        )
      )
@ -293,14 +317,21 @@ export class EntityStoreDataClient {

  public async init(
    entityType: EntityType,
-    {
-      indexPattern = '',
-      filter = '',
-      fieldHistoryLength = 10,
-      enrichPolicyExecutionInterval = DEFAULT_INTERVAL,
-    }: InitEntityEngineRequestBody,
+    InitEntityEngineRequestBodyOverrides: Partial<typeof DEFAULT_ENTITY_ENGINE> = {},
    { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
  ): Promise<InitEntityEngineResponse> {
+    const mergedRequest = {
+      ...DEFAULT_ENTITY_ENGINE,
+      ...InitEntityEngineRequestBodyOverrides,
+    } as Required<typeof DEFAULT_ENTITY_ENGINE>;
+
+    const {
+      indexPattern,
+      filter,
+      fieldHistoryLength,
+      lookbackPeriod,
+      enrichPolicyExecutionInterval,
+    } = mergedRequest;
    const { experimentalFeatures } = this.options;

    if (entityType === EntityType.universal && !experimentalFeatures.assetInventoryStoreEnabled) {
@ -348,6 +379,7 @@ export class EntityStoreDataClient {
    const descriptor = await this.engineClient.init(entityType, {
      filter,
      fieldHistoryLength,
+      lookbackPeriod,
      indexPattern,
    });
    this.log('debug', entityType, `Initialized engine saved object`);
@ -355,6 +387,7 @@ export class EntityStoreDataClient {
    this.asyncSetup(
      entityType,
      fieldHistoryLength,
+      lookbackPeriod,
      enrichPolicyExecutionInterval,
      this.options.taskManager,
      indexPattern,
@ -371,6 +404,7 @@ export class EntityStoreDataClient {
  private async asyncSetup(
    entityType: EntityType,
    fieldHistoryLength: number,
+    lookbackPeriod: string,
    enrichPolicyExecutionInterval: string,
    taskManager: TaskManagerStartContract,
    indexPattern: string,
@ -386,7 +420,7 @@ export class EntityStoreDataClient {
      const description = createEngineDescription({
        entityType,
        namespace,
-        requestParams: { indexPattern, fieldHistoryLength },
+        requestParams: { indexPattern, fieldHistoryLength, lookbackPeriod },
        defaultIndexPatterns,
        config,
      });
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
@ -42,6 +42,7 @@ interface EngineDescriptionParams {
  requestParams?: {
    indexPattern?: string;
    fieldHistoryLength?: number;
+    lookbackPeriod?: string;
  };
  defaultIndexPatterns: string[];
 }
@ -59,7 +60,10 @@ export const createEngineDescription = (options: EngineDescriptionParams) => {
  const settings: EntityEngineInstallationDescriptor['settings'] = {
    syncDelay: `${config.syncDelay.asSeconds()}s`,
    frequency: `${config.frequency.asSeconds()}s`,
-    lookbackPeriod: description.settings?.lookbackPeriod || DEFAULT_LOOKBACK_PERIOD,
+    lookbackPeriod:
+      requestParams?.lookbackPeriod ||
+      description.settings?.lookbackPeriod ||
+      DEFAULT_LOOKBACK_PERIOD,
    timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
  };

--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts
@ -37,7 +37,8 @@ export class EngineDescriptorClient {
      filter,
      fieldHistoryLength,
      indexPattern,
-    }: { filter: string; fieldHistoryLength: number; indexPattern: string }
+      lookbackPeriod,
+    }: { filter: string; fieldHistoryLength: number; indexPattern: string; lookbackPeriod: string }
  ) {
    const engineDescriptor = await this.find(entityType);

@ -54,6 +55,7 @@ export class EngineDescriptorClient {
        filter,
        fieldHistoryLength,
        indexPattern,
+        lookbackPeriod,
      };
      await this.deps.soClient.update<EngineDescriptor>(
        entityEngineDescriptorTypeName,
@ -73,6 +75,7 @@ export class EngineDescriptorClient {
        indexPattern,
        filter,
        fieldHistoryLength,
+        lookbackPeriod,
      },
      { id: this.getSavedObjectId(entityType) }
    );
--- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts
@ -90,6 +90,7 @@ export default ({ getService }: FtrProviderContext) => {
            indexPattern: '',
            filter: '',
            fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
          });
        });

@ -106,6 +107,7 @@ export default ({ getService }: FtrProviderContext) => {
            indexPattern: '',
            filter: '',
            fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
          });
        });
      });
@ -124,6 +126,7 @@ export default ({ getService }: FtrProviderContext) => {
              indexPattern: '',
              filter: '',
              fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
            },
            {
              status: 'started',
@ -131,6 +134,7 @@ export default ({ getService }: FtrProviderContext) => {
              indexPattern: '',
              filter: '',
              fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
            },
          ]);
        });
--- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts
@ -77,6 +77,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
            type: 'host',
            filter: '',
            fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
            indexPattern: '',
          });
        });
@ -96,6 +97,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
            type: 'user',
            filter: '',
            fieldHistoryLength: 10,
+            lookbackPeriod: '24h',
            indexPattern: '',
          });
        });
@ -114,6 +116,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
              type: 'host',
              filter: '',
              fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
              indexPattern: '',
            },
            {
@ -121,6 +124,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
              type: 'user',
              filter: '',
              fieldHistoryLength: 10,
+              lookbackPeriod: '24h',
              indexPattern: '',
            },
          ]);