[8.x] [SecuritySolution] Load entity store indices from security solution data view (#195862) (#196209)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[SecuritySolution] Load entity store indices from security solution
data view (#195862)](https://github.com/elastic/kibana/pull/195862)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Pablo
Machado","email":"pablo.nevesmachado@elastic.co"},"sourceCommit":{"committedDate":"2024-10-14T20:56:58Z","message":"[SecuritySolution]
Load entity store indices from security solution data view
(#195862)\n\n## Summary\r\n\r\n* Update the Entity Store to retrieve
indices from the security solution\r\ndata view.\r\n* Create a new API
that updates all installed entity engine
indices\r\n(`api/entity_store/engines/apply_dataview_indices`)\r\n\r\n\r\n###
How to test it?\r\n* Install the entity store\r\n* Check if the
transform index has the security solutions data view\r\nindices\r\n*
Call `apply_dataview_indices` API; it should not return changes\r\n*
Update the security solution data view indices\r\n* Call
`apply_dataview_indices` API and if the API response contains
the\r\nupdated indices\r\n* Check if the transform index also got
updated\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"489c0901ffd335879d9652424ab15ef9f39cc4cb","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","v9.0.0","Team:
SecuritySolution","backport:prev-minor","Theme:
entity_analytics","Feature:Entity Analytics","Team:Entity
Analytics"],"title":"[SecuritySolution] Load entity store indices from
security solution data
view","number":195862,"url":"https://github.com/elastic/kibana/pull/195862","mergeCommit":{"message":"[SecuritySolution]
Load entity store indices from security solution data view
(#195862)\n\n## Summary\r\n\r\n* Update the Entity Store to retrieve
indices from the security solution\r\ndata view.\r\n* Create a new API
that updates all installed entity engine
indices\r\n(`api/entity_store/engines/apply_dataview_indices`)\r\n\r\n\r\n###
How to test it?\r\n* Install the entity store\r\n* Check if the
transform index has the security solutions data view\r\nindices\r\n*
Call `apply_dataview_indices` API; it should not return changes\r\n*
Update the security solution data view indices\r\n* Call
`apply_dataview_indices` API and if the API response contains
the\r\nupdated indices\r\n* Check if the transform index also got
updated\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"489c0901ffd335879d9652424ab15ef9f39cc4cb"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195862","number":195862,"mergeCommit":{"message":"[SecuritySolution]
Load entity store indices from security solution data view
(#195862)\n\n## Summary\r\n\r\n* Update the Entity Store to retrieve
indices from the security solution\r\ndata view.\r\n* Create a new API
that updates all installed entity engine
indices\r\n(`api/entity_store/engines/apply_dataview_indices`)\r\n\r\n\r\n###
How to test it?\r\n* Install the entity store\r\n* Check if the
transform index has the security solutions data view\r\nindices\r\n*
Call `apply_dataview_indices` API; it should not return changes\r\n*
Update the security solution data view indices\r\n* Call
`apply_dataview_indices` API and if the API response contains
the\r\nupdated indices\r\n* Check if the transform index also got
updated\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"489c0901ffd335879d9652424ab15ef9f39cc4cb"}}]}]
BACKPORT-->

Co-authored-by: Pablo Machado <pablo.nevesmachado@elastic.co>

x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export class EntityDefinitionUpdateConflict extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'EntityDefinitionUpdateConflict';
+  }
+}

x-pack/plugins/entity_manager/server/lib/entity_client.ts

@@ -5,17 +5,23 @@
  * 2.0.
  */
 
-import { EntityDefinition } from '@kbn/entities-schema';
+import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
-import { installEntityDefinition } from './entities/install_entity_definition';
+import {
+  installEntityDefinition,
+  installationInProgress,
+  reinstallEntityDefinition,
+} from './entities/install_entity_definition';
 import { startTransforms } from './entities/start_transforms';
-import { findEntityDefinitions } from './entities/find_entity_definition';
+import { findEntityDefinitionById, findEntityDefinitions } from './entities/find_entity_definition';
 import { uninstallEntityDefinition } from './entities/uninstall_entity_definition';
 import { EntityDefinitionNotFound } from './entities/errors/entity_not_found';
 
 import { stopTransforms } from './entities/stop_transforms';
+import { EntityDefinitionWithState } from './entities/types';
+import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
 
 export class EntityClient {
   constructor(
@@ -47,6 +53,50 @@ export class EntityClient {
     return installedDefinition;
   }
 
+  async updateEntityDefinition({
+    id,
+    definitionUpdate,
+  }: {
+    id: string;
+    definitionUpdate: EntityDefinitionUpdate;
+  }) {
+    const definition = await findEntityDefinitionById({
+      id,
+      soClient: this.options.soClient,
+      esClient: this.options.esClient,
+      includeState: true,
+    });
+
+    if (!definition) {
+      const message = `Unable to find entity definition with [${id}]`;
+      this.options.logger.error(message);
+      throw new EntityDefinitionNotFound(message);
+    }
+
+    if (installationInProgress(definition)) {
+      const message = `Entity definition [${definition.id}] has changes in progress`;
+      this.options.logger.error(message);
+      throw new EntityDefinitionUpdateConflict(message);
+    }
+
+    const shouldRestartTransforms = (
+      definition as EntityDefinitionWithState
+    ).state.components.transforms.some((transform) => transform.running);
+
+    const updatedDefinition = await reinstallEntityDefinition({
+      definition,
+      definitionUpdate,
+      soClient: this.options.soClient,
+      esClient: this.options.esClient,
+      logger: this.options.logger,
+    });
+
+    if (shouldRestartTransforms) {
+      await startTransforms(this.options.esClient, updatedDefinition, this.options.logger);
+    }
+    return updatedDefinition;
+  }
+
   async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
     const [definition] = await findEntityDefinitions({
       id,

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -25,7 +25,7 @@ export type IndexPattern = z.infer<typeof IndexPattern>;
 export const IndexPattern = z.string();
 
 export type EngineStatus = z.infer<typeof EngineStatus>;
-export const EngineStatus = z.enum(['installing', 'started', 'stopped']);
+export const EngineStatus = z.enum(['installing', 'started', 'stopped', 'updating']);
 export type EngineStatusEnum = typeof EngineStatus.enum;
 export const EngineStatusEnum = EngineStatus.enum;
 

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -37,6 +37,7 @@ components:
         - installing
         - started
         - stopped
+        - updating
 
     IndexPattern:
       type: string

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Apply DataView indices to all installed engines
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+
+export type EngineDataviewUpdateResult = z.infer<typeof EngineDataviewUpdateResult>;
+export const EngineDataviewUpdateResult = z.object({
+  type: z.string(),
+  changes: z
+    .object({
+      indexPatterns: z.array(z.string()).optional(),
+    })
+    .optional(),
+});
+
+export type ApplyEntityEngineDataviewIndicesResponse = z.infer<
+  typeof ApplyEntityEngineDataviewIndicesResponse
+>;
+export const ApplyEntityEngineDataviewIndicesResponse = z.object({
+  success: z.boolean().optional(),
+  result: z.array(EngineDataviewUpdateResult).optional(),
+});

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml

@@ -0,0 +1,71 @@
+openapi: 3.0.0
+
+info:
+  title: Apply DataView indices to all installed engines
+  version: '2023-10-31'
+paths:
+  /api/entity_store/engines/apply_dataview_indices:
+    post:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: ApplyEntityEngineDataviewIndices
+      summary: Apply DataView indices to all installed engines
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  success:
+                    type: boolean
+                  result:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+
+        '207':
+          description: Partial successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  success:
+                    type: boolean
+                  result:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+                  errors:
+                    type: array
+                    items:
+                      type: string
+        '500':
+          description: Error response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  body:
+                    type: string
+                  statusCode:
+                    type: number
+components:
+  schemas:
+    EngineDataviewUpdateResult:
+      type: object
+      properties:
+        type:
+          type: string
+        changes:
+          type: object
+          properties:
+            indexPatterns:
+              type: array
+              items:
+                type: string
+      required:
+        - type

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts

@@ -12,3 +12,4 @@ export * from './list.gen';
 export * from './start.gen';
 export * from './stats.gen';
 export * from './stop.gen';
+export * from './apply_dataview_indices.gen';

x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts

@@ -243,6 +243,7 @@ import type {
   InternalUploadAssetCriticalityRecordsResponse,
   UploadAssetCriticalityRecordsResponse,
 } from './entity_analytics/asset_criticality/upload_asset_criticality_csv.gen';
+import type { ApplyEntityEngineDataviewIndicesResponse } from './entity_analytics/entity_store/engine/apply_dataview_indices.gen';
 import type {
   DeleteEntityEngineRequestQueryInput,
   DeleteEntityEngineRequestParamsInput,
@@ -397,6 +398,18 @@ after 30 days. It also deletes other artifacts specific to the migration impleme
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  async applyEntityEngineDataviewIndices() {
+    this.log.info(`${new Date().toISOString()} Calling API ApplyEntityEngineDataviewIndices`);
+    return this.kbnClient
+      .request<ApplyEntityEngineDataviewIndicesResponse>({
+        path: '/api/entity_store/engines/apply_dataview_indices',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
+        },
+        method: 'POST',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   async assetCriticalityGetPrivileges() {
     this.log.info(`${new Date().toISOString()} Calling API AssetCriticalityGetPrivileges`);
     return this.kbnClient

x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -441,6 +441,54 @@ paths:
       summary: Stop an Entity Engine
       tags:
         - Security Entity Analytics API
+  /api/entity_store/engines/apply_dataview_indices:
+    post:
+      operationId: ApplyEntityEngineDataviewIndices
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  result:
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+                    type: array
+                  success:
+                    type: boolean
+          description: Successful response
+        '207':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  errors:
+                    items:
+                      type: string
+                    type: array
+                  result:
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+                    type: array
+                  success:
+                    type: boolean
+          description: Partial successful response
+        '500':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  body:
+                    type: string
+                  statusCode:
+                    type: number
+          description: Error response
+      summary: Apply DataView indices to all installed engines
+      tags:
+        - Security Entity Analytics API
   /api/entity_store/entities/list:
     get:
       description: 'List entities records, paging, sorting and filtering as needed.'
@@ -705,6 +753,20 @@ components:
               $ref: '#/components/schemas/AssetCriticalityLevel'
           required:
             - criticality_level
+    EngineDataviewUpdateResult:
+      type: object
+      properties:
+        changes:
+          type: object
+          properties:
+            indexPatterns:
+              items:
+                type: string
+              type: array
+        type:
+          type: string
+      required:
+        - type
     EngineDescriptor:
       type: object
       properties:
@@ -728,6 +790,7 @@ components:
         - installing
         - started
         - stopped
+        - updating
       type: string
     Entity:
       oneOf:

x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -441,6 +441,54 @@ paths:
       summary: Stop an Entity Engine
       tags:
         - Security Entity Analytics API
+  /api/entity_store/engines/apply_dataview_indices:
+    post:
+      operationId: ApplyEntityEngineDataviewIndices
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  result:
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+                    type: array
+                  success:
+                    type: boolean
+          description: Successful response
+        '207':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  errors:
+                    items:
+                      type: string
+                    type: array
+                  result:
+                    items:
+                      $ref: '#/components/schemas/EngineDataviewUpdateResult'
+                    type: array
+                  success:
+                    type: boolean
+          description: Partial successful response
+        '500':
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  body:
+                    type: string
+                  statusCode:
+                    type: number
+          description: Error response
+      summary: Apply DataView indices to all installed engines
+      tags:
+        - Security Entity Analytics API
   /api/entity_store/entities/list:
     get:
       description: 'List entities records, paging, sorting and filtering as needed.'
@@ -705,6 +753,20 @@ components:
               $ref: '#/components/schemas/AssetCriticalityLevel'
           required:
             - criticality_level
+    EngineDataviewUpdateResult:
+      type: object
+      properties:
+        changes:
+          type: object
+          properties:
+            indexPatterns:
+              items:
+                type: string
+              type: array
+        type:
+          type: string
+      required:
+        - type
     EngineDescriptor:
       type: object
       properties:
@@ -728,6 +790,7 @@ components:
         - installing
         - started
         - stopped
+        - updating
       type: string
     Entity:
       oneOf:

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts

@@ -6,13 +6,11 @@
  */
 
 import type { EngineStatus } from '../../../../common/api/entity_analytics/entity_store/common.gen';
-import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants';
 
 /**
  * Default index pattern for entity store
  * This is the same as the default index pattern for the SIEM app but might diverge in the future
  */
-export const ENTITY_STORE_DEFAULT_SOURCE_INDICES = DEFAULT_INDEX_PATTERN;
 
 export const DEFAULT_LOOKBACK_PERIOD = '24h';
 
@@ -22,6 +20,7 @@ export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {
   INSTALLING: 'installing',
   STARTED: 'started',
   STOPPED: 'stopped',
+  UPDATING: 'updating',
 };
 
 export const MAX_SEARCH_RESPONSE_SIZE = 10_000;

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -13,6 +13,8 @@ import {
 import { EntityStoreDataClient } from './entity_store_data_client';
 import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
 import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen';
+import type { DataViewsService } from '@kbn/data-views-plugin/common';
+import type { AppClient } from '../../..';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
@@ -24,6 +26,8 @@ describe('EntityStoreDataClient', () => {
     namespace: 'default',
     soClient: mockSavedObjectClient,
     kibanaVersion: '9.0.0',
+    dataViewsService: {} as DataViewsService,
+    appClient: {} as AppClient,
   });
 
   const defaultSearchParams = {

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -14,6 +14,10 @@ import type {
 import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
 import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
 import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
+import type { DataViewsService } from '@kbn/data-views-plugin/common';
+import { isEqual } from 'lodash/fp';
+import type { EngineDataviewUpdateResult } from '../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen';
+import type { AppClient } from '../../..';
 import type { Entity } from '../../../../common/api/entity_analytics/entity_store/entities/common.gen';
 import type {
   InitEntityEngineRequestBody,
@@ -24,7 +28,6 @@ import type {
   InspectQuery,
 } from '../../../../common/api/entity_analytics/entity_store/common.gen';
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
-import { buildEntityDefinitionId, getEntitiesIndexName } from './utils';
 import { ENGINE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
 import { AssetCriticalityEcsMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
 import { getUnitedEntityDefinition } from './united_entity_definitions';
@@ -44,6 +47,13 @@ import {
   deleteFieldRetentionEnrichPolicy,
 } from './elasticsearch_assets';
 import { RiskScoreDataClient } from '../risk_score/risk_score_data_client';
+import {
+  buildEntityDefinitionId,
+  buildIndexPatterns,
+  getEntitiesIndexName,
+  isPromiseFulfilled,
+  isPromiseRejected,
+} from './utils';
 
 interface EntityStoreClientOpts {
   logger: Logger;
@@ -53,6 +63,8 @@ interface EntityStoreClientOpts {
   taskManager?: TaskManagerStartContract;
   auditLogger?: AuditLogger;
   kibanaVersion: string;
+  dataViewsService: DataViewsService;
+  appClient: AppClient;
 }
 
 interface SearchEntitiesParams {
@@ -108,7 +120,7 @@ export class EntityStoreDataClient {
       throw new Error('Task Manager is not available');
     }
 
-    const { logger, esClient, namespace, taskManager } = this.options;
+    const { logger, esClient, namespace, taskManager, appClient, dataViewsService } = this.options;
 
     await this.riskScoreDataClient.createRiskScoreLatestIndex();
 
@@ -132,9 +144,11 @@ export class EntityStoreDataClient {
       indexPattern,
     });
     logger.debug(`Initialized engine for ${entityType}`);
+    const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
     // first create the entity definition without starting it
     // so that the index template is created which we can add a component template to
     const unitedDefinition = getUnitedEntityDefinition({
+      indexPatterns,
       entityType,
       namespace,
       fieldHistoryLength,
@@ -221,7 +235,6 @@ export class EntityStoreDataClient {
 
   public async start(entityType: EntityType, options?: { force: boolean }) {
     const descriptor = await this.engineClient.get(entityType);
-
     if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) {
       throw new Error(
         `In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}`
@@ -273,9 +286,11 @@ export class EntityStoreDataClient {
     taskManager: TaskManagerStartContract,
     deleteData: boolean
   ) {
-    const { namespace, logger, esClient } = this.options;
+    const { namespace, logger, esClient, appClient, dataViewsService } = this.options;
     const descriptor = await this.engineClient.maybeGet(entityType);
+    const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
     const unitedDefinition = getUnitedEntityDefinition({
+      indexPatterns,
       entityType,
       namespace: this.options.namespace,
       fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
@@ -368,4 +383,83 @@ export class EntityStoreDataClient {
 
     return { records, total, inspect };
   }
+
+  public async applyDataViewIndices(): Promise<{
+    successes: EngineDataviewUpdateResult[];
+    errors: Error[];
+  }> {
+    const { logger } = this.options;
+    logger.info(
+      `In namespace ${this.options.namespace}: Applying data view indices to the entity store`
+    );
+
+    const { engines } = await this.engineClient.list();
+
+    const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = await engines.map(
+      async (engine) => {
+        const originalStatus = engine.status;
+        const id = buildEntityDefinitionId(engine.type, this.options.namespace);
+        const definition = await this.getExistingEntityDefinition(engine.type);
+
+        if (
+          originalStatus === ENGINE_STATUS.INSTALLING ||
+          originalStatus === ENGINE_STATUS.UPDATING
+        ) {
+          throw new Error(
+            `Error updating entity store: There is an changes already in progress for engine ${id}`
+          );
+        }
+
+        const indexPatterns = await buildIndexPatterns(
+          this.options.namespace,
+          this.options.appClient,
+          this.options.dataViewsService
+        );
+
+        // Skip update if index patterns are the same
+        if (isEqual(definition.indexPatterns, indexPatterns)) {
+          return { type: engine.type, changes: {} };
+        }
+
+        // Update savedObject status
+        await this.engineClient.update(engine.type, ENGINE_STATUS.UPDATING);
+
+        try {
+          // Update entity manager definition
+          await this.entityClient.updateEntityDefinition({
+            id,
+            definitionUpdate: {
+              ...definition,
+              indexPatterns,
+            },
+          });
+
+          // Restore the savedObject status and set the new index pattern
+          await this.engineClient.update(engine.type, originalStatus);
+
+          return { type: engine.type, changes: { indexPatterns } };
+        } catch (error) {
+          // Rollback the engine initial status when the update fails
+          await this.engineClient.update(engine.type, originalStatus);
+
+          throw error;
+        }
+      }
+    );
+
+    const updatedDefinitions = await Promise.allSettled(updateDefinitionPromises);
+
+    const updateErrors = updatedDefinitions
+      .filter(isPromiseRejected)
+      .map((result) => result.reason);
+
+    const updateSuccesses = updatedDefinitions
+      .filter(isPromiseFulfilled<EngineDataviewUpdateResult>)
+      .map((result) => result.value);
+
+    return {
+      successes: updateSuccesses,
+      errors: updateErrors,
+    };
+  }
 }

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IKibanaResponse, Logger } from '@kbn/core/server';
+import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
+import { transformError } from '@kbn/securitysolution-es-utils';
+import type { ApplyEntityEngineDataviewIndicesResponse } from '../../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen';
+import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
+import type { EntityAnalyticsRoutesDeps } from '../../types';
+
+export const applyDataViewIndicesEntityEngineRoute = (
+  router: EntityAnalyticsRoutesDeps['router'],
+  logger: Logger
+) => {
+  router.versioned
+    .post({
+      access: 'public',
+      path: '/api/entity_store/engines/apply_dataview_indices',
+      options: {
+        tags: ['access:securitySolution', `access:${APP_ID}-entity-analytics`],
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: {
+          request: {},
+        },
+      },
+
+      async (
+        context,
+        _,
+        response
+      ): Promise<IKibanaResponse<ApplyEntityEngineDataviewIndicesResponse>> => {
+        const siemResponse = buildSiemResponse(response);
+
+        try {
+          const secSol = await context.securitySolution;
+          const { errors, successes } = await secSol
+            .getEntityStoreDataClient()
+            .applyDataViewIndices();
+
+          const errorMessages = errors.map((e) => e.message);
+
+          if (successes.length === 0 && errors.length > 0) {
+            return siemResponse.error({
+              statusCode: 500,
+              body: `Error in ApplyEntityEngineDataViewIndices. Errors: [${errorMessages.join(
+                ', '
+              )}]`,
+            });
+          }
+
+          if (errors.length === 0) {
+            return response.ok({
+              body: {
+                success: true,
+                result: successes,
+              },
+            });
+          } else {
+            return response.multiStatus({
+              body: {
+                success: false,
+                errors: errorMessages,
+                result: successes,
+              },
+            });
+          }
+        } catch (e) {
+          logger.error('Error in ApplyEntityEngineDataViewIndices:', e);
+          const error = transformError(e);
+          return siemResponse.error({
+            statusCode: error.statusCode,
+            body: error.message,
+          });
+        }
+      }
+    );
+};

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts

@@ -6,6 +6,7 @@
  */
 
 import type { EntityAnalyticsRoutesDeps } from '../../types';
+import { applyDataViewIndicesEntityEngineRoute } from './apply_dataview_indices';
 import { deleteEntityEngineRoute } from './delete';
 import { listEntitiesRoute } from './entities/list';
 import { getEntityEngineRoute } from './get';
@@ -27,4 +28,5 @@ export const registerEntityStoreRoutes = ({
   getEntityEngineRoute(router, logger);
   listEntityEnginesRoute(router, logger);
   listEntitiesRoute(router, logger);
+  applyDataViewIndicesEntityEngineRoute(router, logger);
 };

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts

@@ -8,11 +8,13 @@
 import { getUnitedEntityDefinition } from './get_united_definition';
 
 describe('getUnitedEntityDefinition', () => {
+  const indexPatterns = ['test*'];
   describe('host', () => {
     const unitedDefinition = getUnitedEntityDefinition({
       entityType: 'host',
       namespace: 'test',
       fieldHistoryLength: 10,
+      indexPatterns,
     });
 
     it('mapping', () => {
@@ -151,17 +153,7 @@ describe('getUnitedEntityDefinition', () => {
             },
           ],
           "indexPatterns": Array [
-            "apm-*-transaction*",
-            "auditbeat-*",
-            "endgame-*",
-            "filebeat-*",
-            "logs-*",
-            "packetbeat-*",
-            "traces-apm*",
-            "winlogbeat-*",
-            "-*elastic-cloud-logs-*",
-            ".asset-criticality.asset-criticality-test",
-            "risk-score.risk-score-latest-test",
+            "test*",
           ],
           "latest": Object {
             "lookbackPeriod": "24h",
@@ -286,6 +278,7 @@ describe('getUnitedEntityDefinition', () => {
       entityType: 'user',
       namespace: 'test',
       fieldHistoryLength: 10,
+      indexPatterns,
     });
 
     it('mapping', () => {
@@ -416,17 +409,7 @@ describe('getUnitedEntityDefinition', () => {
             },
           ],
           "indexPatterns": Array [
-            "apm-*-transaction*",
-            "auditbeat-*",
-            "endgame-*",
-            "filebeat-*",
-            "logs-*",
-            "packetbeat-*",
-            "traces-apm*",
-            "winlogbeat-*",
-            "-*elastic-cloud-logs-*",
-            ".asset-criticality.asset-criticality-test",
-            "risk-score.risk-score-latest-test",
+            "test*",
           ],
           "latest": Object {
             "lookbackPeriod": "24h",

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts

@@ -24,10 +24,16 @@ interface Options {
   entityType: EntityType;
   namespace: string;
   fieldHistoryLength: number;
+  indexPatterns: string[];
 }
 
 export const getUnitedEntityDefinition = memoize(
-  ({ entityType, namespace, fieldHistoryLength }: Options): UnitedEntityDefinition => {
+  ({
+    entityType,
+    namespace,
+    fieldHistoryLength,
+    indexPatterns,
+  }: Options): UnitedEntityDefinition => {
     const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);
 
     unitedDefinition.fields.push(
@@ -40,6 +46,7 @@ export const getUnitedEntityDefinition = memoize(
     return new UnitedEntityDefinition({
       ...unitedDefinition,
       namespace,
+      indexPatterns,
     });
   },
   ({ entityType, namespace, fieldHistoryLength }: Options) =>

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts

@@ -7,13 +7,7 @@
 import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
 import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
-import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
-import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
-import {
-  DEFAULT_INTERVAL,
-  DEFAULT_LOOKBACK_PERIOD,
-  ENTITY_STORE_DEFAULT_SOURCE_INDICES,
-} from '../constants';
+import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants';
 import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
 import type {
   FieldRetentionDefinition,
@@ -25,6 +19,7 @@ import { BASE_ENTITY_INDEX_MAPPING } from './constants';
 export class UnitedEntityDefinition {
   version: string;
   entityType: EntityType;
+  indexPatterns: string[];
   fields: UnitedDefinitionField[];
   namespace: string;
   entityManagerDefinition: EntityDefinition;
@@ -34,11 +29,13 @@ export class UnitedEntityDefinition {
   constructor(opts: {
     version: string;
     entityType: EntityType;
+    indexPatterns: string[];
     fields: UnitedDefinitionField[];
     namespace: string;
   }) {
     this.version = opts.version;
     this.entityType = opts.entityType;
+    this.indexPatterns = opts.indexPatterns;
     this.fields = opts.fields;
     this.namespace = opts.namespace;
     this.entityManagerDefinition = this.toEntityManagerDefinition();
@@ -47,7 +44,7 @@ export class UnitedEntityDefinition {
   }
 
   private toEntityManagerDefinition(): EntityDefinition {
-    const { entityType, namespace } = this;
+    const { entityType, namespace, indexPatterns } = this;
     const identityField = getIdentityFieldForEntityType(this.entityType);
     const metadata = this.fields
       .filter((field) => field.definition)
@@ -57,11 +54,7 @@ export class UnitedEntityDefinition {
       id: buildEntityDefinitionId(entityType, namespace),
       name: `Security '${entityType}' Entity Store Definition`,
       type: entityType,
-      indexPatterns: [
-        ...ENTITY_STORE_DEFAULT_SOURCE_INDICES,
-        getAssetCriticalityIndex(namespace),
-        getRiskScoreLatestIndex(namespace),
-      ],
+      indexPatterns,
       identityFields: [identityField],
       displayNameTemplate: `{{${identityField}}}`,
       metadata,

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts

@@ -10,6 +10,10 @@ import {
   ENTITY_SCHEMA_VERSION_V1,
   entitiesIndexPattern,
 } from '@kbn/entities-schema';
+import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
+import type { AppClient } from '../../../../types';
+import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
+import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
 import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
 import { entityEngineDescriptorTypeName } from '../saved_object';
 
@@ -19,6 +23,44 @@ export const getIdentityFieldForEntityType = (entityType: EntityType) => {
   return 'user.name';
 };
 
+export const buildIndexPatterns = async (
+  space: string,
+  appClient: AppClient,
+  dataViewsService: DataViewsService
+) => {
+  const { alertsIndex, securitySolutionDataViewIndices } = await getSecuritySolutionIndices(
+    appClient,
+    dataViewsService
+  );
+  return [
+    ...securitySolutionDataViewIndices.filter((item) => item !== alertsIndex),
+    getAssetCriticalityIndex(space),
+    getRiskScoreLatestIndex(space),
+  ];
+};
+
+const getSecuritySolutionIndices = async (
+  appClient: AppClient,
+  dataViewsService: DataViewsService
+) => {
+  const securitySolutionDataViewId = appClient.getSourcererDataViewId();
+  let dataView: DataView;
+  try {
+    dataView = await dataViewsService.get(securitySolutionDataViewId);
+  } catch (e) {
+    if (e.isBoom && e.output.statusCode === 404) {
+      throw new Error(`Data view not found '${securitySolutionDataViewId}'`);
+    }
+    throw e;
+  }
+
+  const dataViewIndexPattern = dataView.getIndexPattern();
+  return {
+    securitySolutionDataViewIndices: dataViewIndexPattern.split(','),
+    alertsIndex: appClient.getAlertsIndex(),
+  };
+};
+
 export const getByEntityTypeQuery = (entityType: EntityType) => {
   return `${entityEngineDescriptorTypeName}.attributes.type: ${entityType}`;
 };
@@ -33,3 +75,11 @@ export const getEntitiesIndexName = (entityType: EntityType, namespace: string)
 export const buildEntityDefinitionId = (entityType: EntityType, space: string) => {
   return `security_${entityType}_${space}`;
 };
+
+export const isPromiseFulfilled = <T>(
+  result: PromiseSettledResult<T>
+): result is PromiseFulfilledResult<T> => result.status === 'fulfilled';
+
+export const isPromiseRejected = <T>(
+  result: PromiseSettledResult<T>
+): result is PromiseRejectedResult => result.status === 'rejected';

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -74,6 +74,12 @@ export class RequestContextFactory implements IRequestContextFactory {
     const licensing = await context.licensing;
     const actionsClient = await startPlugins.actions.getActionsClientWithRequest(request);
 
+    const dataViewsService = await startPlugins.dataViews.dataViewsServiceFactory(
+      coreContext.savedObjects.client,
+      coreContext.elasticsearch.client.asInternalUser,
+      request
+    );
+
     const getSpaceId = (): string =>
       startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID;
 
@@ -84,6 +90,7 @@ export class RequestContextFactory implements IRequestContextFactory {
       kibanaBranch: options.kibanaBranch,
       buildFlavor: options.buildFlavor,
     });
+    const getAppClient = () => appClientFactory.create(request);
 
     const getAuditLogger = () => security?.audit.asScoped(request);
 
@@ -109,7 +116,7 @@ export class RequestContextFactory implements IRequestContextFactory {
 
       getFrameworkRequest: () => frameworkRequest,
 
-      getAppClient: () => appClientFactory.create(request),
+      getAppClient,
 
       getSpaceId,
 
@@ -197,6 +204,8 @@ export class RequestContextFactory implements IRequestContextFactory {
         const soClient = coreContext.savedObjects.client;
         return new EntityStoreDataClient({
           namespace: getSpaceId(),
+          dataViewsService,
+          appClient: getAppClient(),
           esClient,
           logger,
           soClient,