Added Entity Store data view refresh task (#208543)

# Background

This change introduces a new Kibana task within the Security solution.
When the Security solution's entity store feature is enabled, the task
is scheduled in order to continuously reflect changes to the Security
solution's default data view, thereby updating the Transform associated
with the Entity Store when necessary.

# Implementation notes

A key problem when updating/upgrading a transform in the background is
that Elasticsearch requires a user to make the request for these
changes, but no "user" is present in background tasks. The internal
Kibana user does not suffice, because it does not always have access to
the underlying indices.

To accomplish the above, this PR leverages the Entity Manager's ability
to store the API Key of the user who installed the entity store, and
makes any associated changes to the Transform using that user's stored
API key. Said API key is encrypted, and uses a deterministic ID per
installed space in order to support later retrieval.

A single API key is installed per space, meaning multiple entity
"engines" in a space will leverage a single API key for updates.

# Steps to test locally

1. Pull down the code
2. To assist with a quicker feedback loop, manually edit the task's
`interval` constant to a low value, such as `1m`. This value can be
found
[here](x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/constants.ts)
3. Start Kibana
4. Load desired entity test event data. (For example, the internal
[security-documents-generator](https://github.com/elastic/security-documents-generator)
repository's `yarn start entity-store` command can be used)
5. Enable the Security Entity Store by navigating to "Management->Entity
Store"
6. Validate the list of indices in the current transform by navigating
to `/app/management/data/transform` in Kibana, and clicking
"entities-v1-latest-security_host_default->JSON", looking for the
`source.index` field
7. Update the security default data view's index patterns to include a
new pattern. To do so, navigate to `/app/management/kibana/dataViews` in
Kibana, click the data view with the "Security Data View" badge, click
edit, and change the "Index pattern" by adding a comma and a new pattern
to the end, such as `,some-cool-pattern-*`. Save the change.
8. Wait the appropriate amount of time, as defined in step 2.
(Optionally check the Kibana console logs for activity.)
9. Once again, validate the list of indices within the transform as in
step 6, but this time see that the new index pattern is included.

---------

Co-authored-by: Pablo Machado <pablo.nevesmachado@elastic.co>

x-pack/platform/plugins/shared/entity_manager/server/lib/auth/api_key/saved_object.ts

@@ -6,12 +6,17 @@
  */
 
 import { SavedObjectsErrorHelpers, SavedObjectsClientContract } from '@kbn/core/server';
+import { v5 as uuidv5 } from 'uuid';
 import { EntityDiscoveryApiKeyType } from '../../../saved_objects';
 import { EntityManagerServerSetup } from '../../../types';
 import { EntityDiscoveryAPIKey } from './api_key';
 
 const ENTITY_DISCOVERY_API_KEY_SO_ID = '19540C97-E35C-485B-8566-FB86EC8455E4';
 
+export const getSpaceAwareEntityDiscoverySavedObjectId = (space: string) => {
+  return uuidv5(space, ENTITY_DISCOVERY_API_KEY_SO_ID);
+};
+
 const getEncryptedSOClient = (server: EntityManagerServerSetup) => {
   return server.encryptedSavedObjects.getClient({
     includedHiddenTypes: [EntityDiscoveryApiKeyType.name],

x-pack/platform/plugins/shared/entity_manager/server/plugin.ts

@@ -166,7 +166,6 @@ export class EntityManagerServerPlugin
   ): EntityManagerServerPluginStart {
     if (this.server) {
       this.server.core = core;
-      this.server.isServerless = core.elasticsearch.getCapabilities().serverless;
       this.server.security = plugins.security;
       this.server.encryptedSavedObjects = plugins.encryptedSavedObjects;
     }

x-pack/platform/plugins/shared/entity_manager/server/types.ts

@@ -21,7 +21,6 @@ export interface EntityManagerServerSetup {
   logger: Logger;
   security: SecurityPluginStart;
   encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
-  isServerless: boolean;
 }
 
 export interface ElasticsearchAccessorOptions {

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/auth/api_key.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { KibanaRequest } from '@kbn/core-http-server';
+import { generateEntityDiscoveryAPIKey } from '@kbn/entityManager-plugin/server/lib/auth';
+import { EntityDiscoveryApiKeyType } from '@kbn/entityManager-plugin/server/saved_objects';
+import type { CoreStart } from '@kbn/core-lifecycle-server';
+import type { Logger } from '@kbn/logging';
+import type { SecurityPluginStart } from '@kbn/security-plugin-types-server';
+import type { EncryptedSavedObjectsPluginStart } from '@kbn/encrypted-saved-objects-plugin/server';
+import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request';
+import type { EntityDiscoveryAPIKey } from '@kbn/entityManager-plugin/server/lib/auth/api_key/api_key';
+import { getSpaceAwareEntityDiscoverySavedObjectId } from '@kbn/entityManager-plugin/server/lib/auth/api_key/saved_object';
+import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
+
+export interface ApiKeyManager {
+  generate: () => Promise<void>;
+}
+
+export const getApiKeyManager = ({
+  core,
+  logger,
+  security,
+  encryptedSavedObjects,
+  request,
+  namespace,
+}: {
+  core: CoreStart;
+  logger: Logger;
+  security: SecurityPluginStart;
+  encryptedSavedObjects?: EncryptedSavedObjectsPluginStart;
+  request?: KibanaRequest;
+  namespace: string;
+}) => ({
+  generate: async () => {
+    if (!encryptedSavedObjects) {
+      throw new Error(
+        'Unable to create API key. Ensure encrypted Saved Object client is enabled in this environment.'
+      );
+    } else if (!request) {
+      throw new Error('Unable to create API key due to invalid request');
+    } else {
+      const apiKey = await generateEntityDiscoveryAPIKey(
+        {
+          core,
+          config: {},
+          logger,
+          security,
+          encryptedSavedObjects,
+        },
+        request
+      );
+
+      const soClient = core.savedObjects.getScopedClient(request, {
+        includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
+      });
+
+      await soClient.create(EntityDiscoveryApiKeyType.name, apiKey, {
+        id: getSpaceAwareEntityDiscoverySavedObjectId(namespace),
+        overwrite: true,
+        managed: true,
+      });
+    }
+  },
+  getApiKey: async () => {
+    if (!encryptedSavedObjects) {
+      throw Error(
+        'Unable to retrieve API key. Ensure encrypted Saved Object client is enabled in this environment.'
+      );
+    }
+    try {
+      const encryptedSavedObjectsClient = encryptedSavedObjects.getClient({
+        includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
+      });
+      return (
+        await encryptedSavedObjectsClient.getDecryptedAsInternalUser<EntityDiscoveryAPIKey>(
+          EntityDiscoveryApiKeyType.name,
+          getSpaceAwareEntityDiscoverySavedObjectId(namespace)
+        )
+      ).attributes;
+    } catch (err) {
+      if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
+        return undefined;
+      }
+      throw err;
+    }
+  },
+  getRequestFromApiKey: async (apiKey: EntityDiscoveryAPIKey) => {
+    return getFakeKibanaRequest({
+      id: apiKey.id,
+      api_key: apiKey.apiKey,
+    });
+  },
+  getClientFromApiKey: async (apiKey: EntityDiscoveryAPIKey) => {
+    const fakeRequest = getFakeKibanaRequest({
+      id: apiKey.id,
+      api_key: apiKey.apiKey,
+    });
+    const clusterClient = core.elasticsearch.client.asScoped(fakeRequest);
+    const soClient = core.savedObjects.getScopedClient(fakeRequest, {
+      includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
+    });
+    return {
+      clusterClient,
+      soClient,
+    };
+  },
+});

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -53,7 +53,10 @@ import {
   startEntityStoreFieldRetentionEnrichTask,
   removeEntityStoreFieldRetentionEnrichTask,
   getEntityStoreFieldRetentionEnrichTaskState as getEntityStoreFieldRetentionEnrichTaskStatus,
-} from './task';
+  removeEntityStoreDataViewRefreshTask,
+  startEntityStoreDataViewRefreshTask,
+  getEntityStoreDataViewRefreshTaskState,
+} from './tasks';
 import {
   createEntityIndex,
   deleteEntityIndex,
@@ -91,7 +94,8 @@ import {
   createKeywordBuilderPipeline,
   deleteKeywordBuilderPipeline,
 } from '../../asset_inventory/ingest_pipelines';
-import { DEFAULT_INTERVAL } from './task/constants';
+import { DEFAULT_INTERVAL } from './tasks/field_retention_enrichment/constants';
+import type { ApiKeyManager } from './auth/api_key';
 
 // Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
 export interface TransformHealth extends estypes.TransformGetTransformStatsTransformStatsHealth {
@@ -119,6 +123,7 @@ interface EntityStoreClientOpts {
   config: EntityStoreConfig;
   experimentalFeatures: ExperimentalFeatures;
   telemetry?: AnalyticsServiceSetup;
+  apiKeyManager?: ApiKeyManager;
 }
 
 interface SearchEntitiesParams {
@@ -148,10 +153,20 @@ export class EntityStoreDataClient {
   private entityClient: EntityClient;
   private riskScoreDataClient: RiskScoreDataClient;
   private esClient: ElasticsearchClient;
+  private apiKeyGenerator?: ApiKeyManager;
 
   constructor(private readonly options: EntityStoreClientOpts) {
-    const { clusterClient, logger, soClient, auditLogger, kibanaVersion, namespace } = options;
+    const {
+      clusterClient,
+      logger,
+      soClient,
+      auditLogger,
+      kibanaVersion,
+      namespace,
+      apiKeyManager,
+    } = options;
     this.esClient = clusterClient.asCurrentUser;
+    this.apiKeyGenerator = apiKeyManager;
 
     this.entityClient = new EntityClient({
       clusterClient,
@@ -190,6 +205,9 @@ export class EntityStoreDataClient {
           ...(taskManager
             ? [getEntityStoreFieldRetentionEnrichTaskStatus({ namespace, taskManager })]
             : []),
+          ...(taskManager
+            ? [getEntityStoreDataViewRefreshTaskState({ namespace, taskManager })]
+            : []),
           getPlatformPipelineStatus({
             engineId: definition.id,
             esClient: this.esClient,
@@ -428,6 +446,10 @@ export class EntityStoreDataClient {
       // clean up any existing entity store
       await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false });
 
+      if (this.apiKeyGenerator) {
+        await this.apiKeyGenerator.generate();
+      }
+
       // set up the entity manager definition
       const definition = convertToEntityManagerDefinition(description, {
         namespace,
@@ -497,6 +519,13 @@ export class EntityStoreDataClient {
         interval: enrichPolicyExecutionInterval,
       });
 
+      // this task will continuously refresh the Entity Store indices based on the Data View
+      await startEntityStoreDataViewRefreshTask({
+        namespace,
+        logger,
+        taskManager,
+      });
+
       this.log(`debug`, entityType, `Started entity store field retention enrich task`);
       this.log(`info`, entityType, `Entity store initialized`);
 
@@ -755,7 +784,16 @@ export class EntityStoreDataClient {
           logger,
           taskManager,
         });
-        this.log('debug', entityType, `Deleted entity store field retention enrich task`);
+        await removeEntityStoreDataViewRefreshTask({
+          namespace,
+          logger,
+          taskManager,
+        });
+        this.log(
+          'debug',
+          entityType,
+          `Deleted entity store field retention and data view refresh tasks`
+        );
       }
 
       logger.info(`[Entity Store] In namespace ${namespace}: Deleted store for ${entityType}`);
@@ -826,13 +864,13 @@ export class EntityStoreDataClient {
     errors: Error[];
   }> {
     const { logger } = this.options;
-    logger.info(
+    logger.debug(
       `In namespace ${this.options.namespace}: Applying data view indices to the entity store`
     );
 
     const { engines } = await this.engineClient.list();
 
-    const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = await engines.map(
+    const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = engines.map(
       async (engine) => {
         const originalStatus = engine.status;
         const id = buildEntityDefinitionId(engine.type, this.options.namespace);
@@ -855,7 +893,14 @@ export class EntityStoreDataClient {
 
         // Skip update if index patterns are the same
         if (isEqual(definition.indexPatterns, indexPatterns)) {
+          logger.debug(
+            `In namespace ${this.options.namespace}: No data view index changes detected.`
+          );
           return { type: engine.type, changes: {} };
+        } else {
+          logger.info(
+            `In namespace ${this.options.namespace}: Data view index changes detected, applying changes to entity definition.`
+          );
         }
 
         // Update savedObject status

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/constants.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const SCOPE = ['securitySolution'];
+export const TYPE = 'entity_store:data_view:refresh';
+export const VERSION = '1.0.0';
+export const INTERVAL = '30m';
+export const TIMEOUT = '10m';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/data_view_refresh_task.test.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  removeEntityStoreDataViewRefreshTask,
+  runEntityStoreDataViewRefreshTask,
+} from './data_view_refresh_task';
+import { TYPE, VERSION } from './constants';
+import { loggerMock } from '@kbn/logging-mocks';
+import { coreMock } from '@kbn/core/server/mocks';
+import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server';
+import { mockGlobalState } from '../../../../../../public/common/mock';
+import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
+
+const mockLog = jest.fn();
+jest.mock('../utils', () => ({
+  entityStoreTaskLogFactory: () => mockLog,
+  entityStoreTaskDebugLogFactory: () => mockLog,
+}));
+
+jest.mock('../../../../telemetry/event_based/events', () => ({
+  ENTITY_STORE_DATA_VIEW_REFRESH_EXECUTION_EVENT: {
+    eventType: 'entity_store_data_view_refresh_execution',
+  },
+}));
+
+const TASK_ID = `${TYPE}:default:${VERSION}`;
+
+describe('data_view_refresh_task', () => {
+  const logger = loggerMock.create();
+  const telemetry = coreMock.createSetup().analytics;
+
+  const taskManager = taskManagerMock.createStart();
+  const experimentalFeatures = mockGlobalState.app.enableExperimental;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('removeEntityStoreDataViewRefreshTask', () => {
+    it('should remove the task', async () => {
+      await removeEntityStoreDataViewRefreshTask({
+        logger,
+        namespace: 'default',
+        taskManager,
+      });
+
+      expect(taskManager.remove).toHaveBeenCalled();
+    });
+  });
+
+  describe('runEntityStoreDataViewRefreshTask', () => {
+    const taskInstance = {
+      id: TASK_ID,
+      state: {
+        namespace: 'default',
+        runs: 0,
+        lastExecutionTimestamp: '',
+      },
+    } as unknown as ConcreteTaskInstance;
+
+    it('should run the task', async () => {
+      const refreshDataViews = jest.fn();
+      const isCancelled = jest.fn().mockReturnValue(false);
+
+      await runEntityStoreDataViewRefreshTask({
+        refreshDataViews,
+        isCancelled,
+        logger,
+        taskInstance,
+        telemetry,
+        experimentalFeatures,
+      });
+
+      expect(refreshDataViews).toHaveBeenCalledWith('default');
+    });
+
+    it('should log an error if task execution fails', async () => {
+      const refreshDataViews = jest.fn().mockRejectedValue(new Error('Execution failed'));
+      const isCancelled = jest.fn().mockReturnValue(false);
+
+      await runEntityStoreDataViewRefreshTask({
+        refreshDataViews,
+        isCancelled,
+        logger,
+        taskInstance,
+        telemetry,
+        experimentalFeatures,
+      });
+
+      expect(mockLog).toHaveBeenCalledWith('Error executing data view refresh: Execution failed');
+    });
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/data_view_refresh_task.ts

@@ -0,0 +1,307 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import type { AnalyticsServiceSetup, AuditLogger } from '@kbn/core/server';
+import { type Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
+import type {
+  ConcreteTaskInstance,
+  TaskManagerSetupContract,
+  TaskManagerStartContract,
+} from '@kbn/task-manager-plugin/server';
+import type { EntityStoreConfig } from '../../types';
+import { EntityStoreDataClient } from '../../entity_store_data_client';
+import { getApiKeyManager } from '../../auth/api_key';
+import type { ExperimentalFeatures } from '../../../../../../common';
+import { EngineComponentResourceEnum } from '../../../../../../common/api/entity_analytics/entity_store';
+import {
+  defaultState,
+  stateSchemaByVersion,
+  type LatestTaskStateSchema as EntityStoreDataViewRefreshTaskState,
+} from './state';
+import { INTERVAL, SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
+import type { EntityAnalyticsRoutesDeps } from '../../../types';
+
+import { ENTITY_STORE_DATA_VIEW_REFRESH_EXECUTION_EVENT } from '../../../../telemetry/event_based/events';
+import { entityStoreTaskDebugLogFactory, entityStoreTaskLogFactory } from '../utils';
+import type { AppClientFactory } from '../../../../../client';
+
+const getTaskName = (): string => TYPE;
+
+const getTaskId = (namespace: string): string => `${TYPE}:${namespace}:${VERSION}`;
+
+export const registerEntityStoreDataViewRefreshTask = ({
+  getStartServices,
+  logger,
+  telemetry,
+  appClientFactory,
+  taskManager,
+  auditLogger,
+  entityStoreConfig,
+  experimentalFeatures,
+  kibanaVersion,
+}: {
+  getStartServices: EntityAnalyticsRoutesDeps['getStartServices'];
+  logger: Logger;
+  telemetry: AnalyticsServiceSetup;
+  appClientFactory: AppClientFactory;
+  taskManager?: TaskManagerSetupContract;
+  auditLogger?: AuditLogger;
+  entityStoreConfig: EntityStoreConfig;
+  experimentalFeatures: ExperimentalFeatures;
+  kibanaVersion: string;
+}): void => {
+  if (!taskManager) {
+    logger.info(
+      '[Entity Store] Task Manager is unavailable; skipping entity store data view refresh.'
+    );
+    return;
+  }
+
+  const refreshDataViews = async (namespace: string): Promise<void> => {
+    const [core, { dataViews, taskManager: taskManagerStart, security, encryptedSavedObjects }] =
+      await getStartServices();
+
+    const apiKeyManager = getApiKeyManager({
+      core,
+      logger,
+      security,
+      encryptedSavedObjects,
+      namespace,
+    });
+
+    const apiKey = await apiKeyManager.getApiKey();
+
+    if (!apiKey) {
+      logger.info(
+        `[Entity Store] No API key found, skipping data view refresh in ${namespace} namespace`
+      );
+      return;
+    }
+
+    const { clusterClient, soClient } = await apiKeyManager.getClientFromApiKey(apiKey);
+
+    const internalUserClient = core.elasticsearch.client.asInternalUser;
+
+    const dataViewsService = await dataViews.dataViewsServiceFactory(soClient, internalUserClient);
+
+    const appClient = appClientFactory.create(await apiKeyManager.getRequestFromApiKey(apiKey));
+
+    const entityStoreClient: EntityStoreDataClient = new EntityStoreDataClient({
+      namespace,
+      clusterClient,
+      soClient,
+      logger,
+      appClient,
+      taskManager: taskManagerStart,
+      experimentalFeatures,
+      auditLogger,
+      telemetry,
+      kibanaVersion,
+      dataViewsService,
+      config: entityStoreConfig,
+    });
+
+    await entityStoreClient.applyDataViewIndices();
+  };
+
+  taskManager.registerTaskDefinitions({
+    [getTaskName()]: {
+      title: 'Entity Analytics Entity Store - Execute Data View Refresh Task',
+      timeout: TIMEOUT,
+      stateSchemaByVersion,
+      createTaskRunner: createEntityStoreDataViewRefreshTaskRunnerFactory({
+        logger,
+        telemetry,
+        refreshDataViews,
+        experimentalFeatures,
+      }),
+    },
+  });
+};
+
+export const startEntityStoreDataViewRefreshTask = async ({
+  logger,
+  namespace,
+  taskManager,
+}: {
+  logger: Logger;
+  namespace: string;
+  taskManager: TaskManagerStartContract;
+}) => {
+  const taskId = getTaskId(namespace);
+  const log = entityStoreTaskLogFactory(logger, taskId);
+
+  log('attempting to schedule');
+  try {
+    await taskManager.ensureScheduled({
+      id: taskId,
+      taskType: getTaskName(),
+      scope: SCOPE,
+      schedule: {
+        interval: INTERVAL,
+      },
+      state: { ...defaultState, namespace },
+      params: { version: VERSION },
+    });
+  } catch (e) {
+    logger.warn(`[Entity Store]  [task ${taskId}]: error scheduling task, received ${e.message}`);
+    throw e;
+  }
+};
+
+export const removeEntityStoreDataViewRefreshTask = async ({
+  logger,
+  namespace,
+  taskManager,
+}: {
+  logger: Logger;
+  namespace: string;
+  taskManager: TaskManagerStartContract;
+}) => {
+  try {
+    await taskManager.remove(getTaskId(namespace));
+    logger.info(
+      `[Entity Store] Removed entity store data view refresh task for namespace ${namespace}`
+    );
+  } catch (err) {
+    if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
+      logger.error(
+        `[Entity Store] Failed to remove entity store data view refresh task: ${err.message}`
+      );
+      throw err;
+    }
+  }
+};
+
+export const runEntityStoreDataViewRefreshTask = async ({
+  refreshDataViews,
+  isCancelled,
+  logger,
+  taskInstance,
+  telemetry,
+  experimentalFeatures,
+}: {
+  logger: Logger;
+  isCancelled: () => boolean;
+  refreshDataViews: (namespace: string) => Promise<void>;
+  taskInstance: ConcreteTaskInstance;
+  telemetry: AnalyticsServiceSetup;
+  experimentalFeatures: ExperimentalFeatures;
+}): Promise<{
+  state: EntityStoreDataViewRefreshTaskState;
+}> => {
+  const state = taskInstance.state as EntityStoreDataViewRefreshTaskState;
+  const taskId = taskInstance.id;
+  const log = entityStoreTaskLogFactory(logger, taskId);
+  const debugLog = entityStoreTaskDebugLogFactory(logger, taskId);
+  try {
+    const taskStartTime = moment().utc().toISOString();
+    log('running task');
+
+    const updatedState = {
+      lastExecutionTimestamp: taskStartTime,
+      namespace: state.namespace,
+      runs: state.runs + 1,
+    };
+
+    if (taskId !== getTaskId(state.namespace)) {
+      log('outdated task; exiting');
+      return { state: updatedState };
+    }
+
+    const start = Date.now();
+    debugLog(`Executing data view refresh`);
+    try {
+      await refreshDataViews(state.namespace);
+      log(`Executed data view refresh in ${Date.now() - start}ms`);
+    } catch (e) {
+      log(`Error executing data view refresh: ${e.message}`);
+    }
+
+    const taskCompletionTime = moment().utc().toISOString();
+    const taskDurationInSeconds = moment(taskCompletionTime).diff(moment(taskStartTime), 'seconds');
+    log(`Task run completed in ${taskDurationInSeconds} seconds`);
+
+    telemetry.reportEvent(ENTITY_STORE_DATA_VIEW_REFRESH_EXECUTION_EVENT.eventType, {
+      duration: taskDurationInSeconds,
+      interval: INTERVAL,
+    });
+
+    return {
+      state: updatedState,
+    };
+  } catch (e) {
+    logger.error(`[Entity Store] [task ${taskId}]: error running task, received ${e.message}`);
+    throw e;
+  }
+};
+
+const createEntityStoreDataViewRefreshTaskRunnerFactory =
+  ({
+    logger,
+    telemetry,
+    refreshDataViews,
+    experimentalFeatures,
+  }: {
+    logger: Logger;
+    telemetry: AnalyticsServiceSetup;
+    refreshDataViews: (namespace: string) => Promise<void>;
+    experimentalFeatures: ExperimentalFeatures;
+  }) =>
+  ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => {
+    let cancelled = false;
+    const isCancelled = () => cancelled;
+    return {
+      run: async () =>
+        runEntityStoreDataViewRefreshTask({
+          refreshDataViews,
+          isCancelled,
+          logger,
+          taskInstance,
+          telemetry,
+          experimentalFeatures,
+        }),
+      cancel: async () => {
+        cancelled = true;
+      },
+    };
+  };
+
+export const getEntityStoreDataViewRefreshTaskState = async ({
+  namespace,
+  taskManager,
+}: {
+  namespace: string;
+  taskManager: TaskManagerStartContract;
+}) => {
+  const taskId = getTaskId(namespace);
+  try {
+    const taskState = await taskManager.get(taskId);
+
+    return {
+      id: taskState.id,
+      resource: EngineComponentResourceEnum.task,
+      installed: true,
+      enabled: taskState.enabled,
+      status: taskState.status,
+      retryAttempts: taskState.attempts,
+      nextRun: taskState.runAt,
+      lastRun: taskState.state.lastExecutionTimestamp,
+      runs: taskState.state.runs,
+    };
+  } catch (e) {
+    if (SavedObjectsErrorHelpers.isNotFoundError(e)) {
+      return {
+        id: taskId,
+        installed: false,
+        resource: EngineComponentResourceEnum.task,
+      };
+    }
+    throw e;
+  }
+};

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/data_view_refresh/state.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema, type TypeOf } from '@kbn/config-schema';
+
+/**
+ * WARNING: Do not modify the existing versioned schema(s) below, instead define a new version (ex: 2, 3, 4).
+ * This is required to support zero-downtime upgrades and rollbacks. See https://github.com/elastic/kibana/issues/155764.
+ *
+ * As you add a new schema version, don't forget to change latestTaskStateSchema variable to reference the latest schema.
+ * For example, changing stateSchemaByVersion[1].schema to stateSchemaByVersion[2].schema.
+ */
+export const stateSchemaByVersion = {
+  1: {
+    up: (state: Record<string, unknown>) => ({
+      lastExecutionTimestamp: state.lastExecutionTimestamp || undefined,
+      runs: state.runs || 0,
+      namespace: typeof state.namespace === 'string' ? state.namespace : 'default',
+    }),
+    schema: schema.object({
+      lastExecutionTimestamp: schema.maybe(schema.string()),
+      namespace: schema.string(),
+      runs: schema.number(),
+    }),
+  },
+};
+
+const latestTaskStateSchema = stateSchemaByVersion[1].schema;
+export type LatestTaskStateSchema = TypeOf<typeof latestTaskStateSchema>;
+
+export const defaultState: LatestTaskStateSchema = {
+  lastExecutionTimestamp: undefined,
+  namespace: 'default',
+  runs: 0,
+};

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/field_retention_enrichment/field_retention_enrichment_task.ts

@@ -13,38 +13,29 @@ import type {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '@kbn/task-manager-plugin/server';
-import type { ExperimentalFeatures } from '../../../../../common';
-import { getEnabledStoreEntityTypes } from '../../../../../common/entity_analytics/entity_store/utils';
+import type { ExperimentalFeatures } from '../../../../../../common';
+import { getEnabledStoreEntityTypes } from '../../../../../../common/entity_analytics/entity_store/utils';
 import {
   EngineComponentResourceEnum,
   type EntityType,
-} from '../../../../../common/api/entity_analytics/entity_store';
+} from '../../../../../../common/api/entity_analytics/entity_store';
 import {
   defaultState,
   stateSchemaByVersion,
   type LatestTaskStateSchema as EntityStoreFieldRetentionTaskState,
 } from './state';
 import { SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
-import type { EntityAnalyticsRoutesDeps } from '../../types';
+import type { EntityAnalyticsRoutesDeps } from '../../../types';
 
-import { executeFieldRetentionEnrichPolicy } from '../elasticsearch_assets';
+import { executeFieldRetentionEnrichPolicy } from '../../elasticsearch_assets';
 
-import { getEntitiesIndexName } from '../utils';
+import { getEntitiesIndexName } from '../../utils';
 import {
   FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT,
   ENTITY_STORE_USAGE_EVENT,
-} from '../../../telemetry/event_based/events';
-import { VERSIONS_BY_ENTITY_TYPE } from '../entity_definitions/constants';
-
-const logFactory =
-  (logger: Logger, taskId: string) =>
-  (message: string): void =>
-    logger.info(`[Entity Store] [task ${taskId}]: ${message}`);
-
-const debugLogFactory =
-  (logger: Logger, taskId: string) =>
-  (message: string): void =>
-    logger.debug(`[Entity Store] [task ${taskId}]: ${message}`);
+} from '../../../../telemetry/event_based/events';
+import { VERSIONS_BY_ENTITY_TYPE } from '../../entity_definitions/constants';
+import { entityStoreTaskDebugLogFactory, entityStoreTaskLogFactory } from '../utils';
 
 const getTaskName = (): string => TYPE;
 
@@ -105,7 +96,7 @@ export const registerEntityStoreFieldRetentionEnrichTask = ({
       title: 'Entity Analytics Entity Store - Execute Enrich Policy Task',
       timeout: TIMEOUT,
       stateSchemaByVersion,
-      createTaskRunner: createTaskRunnerFactory({
+      createTaskRunner: createEntityStoreFieldRetentionEnrichTaskRunnerFactory({
         logger,
         telemetry,
         getStoreSize,
@@ -128,8 +119,7 @@ export const startEntityStoreFieldRetentionEnrichTask = async ({
   interval: string;
 }) => {
   const taskId = getTaskId(namespace);
-  const log = logFactory(logger, taskId);
-  log('starting task');
+  const log = entityStoreTaskLogFactory(logger, taskId);
 
   log('attempting to schedule');
   try {
@@ -173,7 +163,7 @@ export const removeEntityStoreFieldRetentionEnrichTask = async ({
   }
 };
 
-export const runTask = async ({
+export const runEntityStoreFieldRetentionEnrichTask = async ({
   executeEnrichPolicy,
   getStoreSize,
   isCancelled,
@@ -194,8 +184,8 @@ export const runTask = async ({
 }> => {
   const state = taskInstance.state as EntityStoreFieldRetentionTaskState;
   const taskId = taskInstance.id;
-  const log = logFactory(logger, taskId);
-  const debugLog = debugLogFactory(logger, taskId);
+  const log = entityStoreTaskLogFactory(logger, taskId);
+  const debugLog = entityStoreTaskDebugLogFactory(logger, taskId);
   try {
     const taskStartTime = moment().utc().toISOString();
     log('running task');
@@ -255,7 +245,7 @@ export const runTask = async ({
   }
 };
 
-const createTaskRunnerFactory =
+const createEntityStoreFieldRetentionEnrichTaskRunnerFactory =
   ({
     logger,
     telemetry,
@@ -274,7 +264,7 @@ const createTaskRunnerFactory =
     const isCancelled = () => cancelled;
     return {
       run: async () =>
-        runTask({
+        runEntityStoreFieldRetentionEnrichTask({
           executeEnrichPolicy,
           getStoreSize,
           isCancelled,

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/field_retention_enrichment/state.ts

@@ -16,8 +16,6 @@ import { schema, type TypeOf } from '@kbn/config-schema';
  */
 export const stateSchemaByVersion = {
   1: {
-    // A task that was created < 8.10 will go through this "up" migration
-    // to ensure it matches the v1 schema.
     up: (state: Record<string, unknown>) => ({
       lastExecutionTimestamp: state.lastExecutionTimestamp || undefined,
       runs: state.runs || 0,

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/index.ts

@@ -5,4 +5,5 @@
  * 2.0.
  */
 
-export * from './field_retention_enrichment_task';
+export * from './field_retention_enrichment/field_retention_enrichment_task';
+export * from './data_view_refresh/data_view_refresh_task';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/tasks/utils.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/logging';
+
+export const entityStoreTaskLogFactory =
+  (logger: Logger, taskId: string) =>
+  (message: string): void =>
+    logger.info(`[Entity Store] [task ${taskId}]: ${message}`);
+
+export const entityStoreTaskDebugLogFactory =
+  (logger: Logger, taskId: string) =>
+  (message: string): void =>
+    logger.debug(`[Entity Store] [task ${taskId}]: ${message}`);

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts

@@ -150,6 +150,27 @@ export const FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT: EventTypeOpts<{
   },
 };
 
+export const ENTITY_STORE_DATA_VIEW_REFRESH_EXECUTION_EVENT: EventTypeOpts<{
+  duration: number;
+  interval: string;
+}> = {
+  eventType: 'entity_store_data_view_refresh_execution_event',
+  schema: {
+    duration: {
+      type: 'long',
+      _meta: {
+        description: 'Duration (in seconds) of the entity store data view refresh execution time',
+      },
+    },
+    interval: {
+      type: 'keyword',
+      _meta: {
+        description: 'Configured interval for the entity store data view refresh task',
+      },
+    },
+  },
+};
+
 export const ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT: EventTypeOpts<{
   error: string;
 }> = {
@@ -690,6 +711,7 @@ export const events = [
   ENDPOINT_RESPONSE_ACTION_SENT_ERROR_EVENT,
   ENDPOINT_RESPONSE_ACTION_STATUS_CHANGE_EVENT,
   FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT,
+  ENTITY_STORE_DATA_VIEW_REFRESH_EXECUTION_EVENT,
   ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT,
   ENTITY_ENGINE_INITIALIZATION_EVENT,
   ENTITY_STORE_USAGE_EVENT,

x-pack/solutions/security/plugins/security_solution/server/plugin.ts

@@ -19,6 +19,7 @@ import type { ILicense } from '@kbn/licensing-plugin/server';
 import type { NewPackagePolicy, UpdatePackagePolicy } from '@kbn/fleet-plugin/common';
 import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
 
+import { registerEntityStoreDataViewRefreshTask } from './lib/entity_analytics/entity_store/tasks/data_view_refresh/data_view_refresh_task';
 import { ensureIndicesExistsForPolicies } from './endpoint/migrations/ensure_indices_exists_for_policies';
 import { CompleteExternalResponseActionsTask } from './endpoint/lib/response_actions';
 import { registerAgentRoutes } from './endpoint/routes/agent';
@@ -118,7 +119,7 @@ import {
 
 import { ProductFeaturesService } from './lib/product_features_service/product_features_service';
 import { registerRiskScoringTask } from './lib/entity_analytics/risk_score/tasks/risk_scoring_task';
-import { registerEntityStoreFieldRetentionEnrichTask } from './lib/entity_analytics/entity_store/task';
+import { registerEntityStoreFieldRetentionEnrichTask } from './lib/entity_analytics/entity_store/tasks';
 import { registerProtectionUpdatesNoteRoutes } from './endpoint/routes/protection_updates_note';
 import {
   latestRiskScoreIndexPattern,
@@ -253,6 +254,18 @@ export class Plugin implements ISecuritySolutionPlugin {
         taskManager: plugins.taskManager,
         experimentalFeatures,
       });
+
+      registerEntityStoreDataViewRefreshTask({
+        getStartServices: core.getStartServices,
+        appClientFactory,
+        logger: this.logger,
+        telemetry: core.analytics,
+        taskManager: plugins.taskManager,
+        auditLogger: plugins.security?.audit.withoutRequest,
+        entityStoreConfig: config.entityAnalytics.entityStore,
+        experimentalFeatures,
+        kibanaVersion: pluginContext.env.packageInfo.version,
+      });
     }
 
     const requestContextFactory = new RequestContextFactory({

x-pack/solutions/security/plugins/security_solution/server/plugin_contract.ts

@@ -18,7 +18,10 @@ import type {
   PluginStartContract as ActionsPluginStartContract,
 } from '@kbn/actions-plugin/server';
 import type { CasesServerStart, CasesServerSetup } from '@kbn/cases-plugin/server';
-import type { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server';
+import type {
+  EncryptedSavedObjectsPluginSetup,
+  EncryptedSavedObjectsPluginStart,
+} from '@kbn/encrypted-saved-objects-plugin/server';
 import type { IEventLogClientService, IEventLogService } from '@kbn/event-log-plugin/server';
 import type { FeaturesPluginSetup } from '@kbn/features-plugin/server';
 import type { FleetStartContract as FleetPluginStart } from '@kbn/fleet-plugin/server';
@@ -75,6 +78,7 @@ export interface SecuritySolutionPluginStartDependencies {
   cloud: CloudSetup;
   data: DataPluginStart;
   dataViews: DataViewsPluginStart;
+  encryptedSavedObjects?: EncryptedSavedObjectsPluginStart;
   elasticAssistant: ElasticAssistantPluginStart;
   eventLog: IEventLogClientService;
   fleet?: FleetPluginStart;

x-pack/solutions/security/plugins/security_solution/server/request_context_factory.ts

@@ -10,6 +10,8 @@ import { memoize } from 'lodash';
 import type { Logger, KibanaRequest, RequestHandlerContext } from '@kbn/core/server';
 
 import type { BuildFlavor } from '@kbn/config';
+import { EntityDiscoveryApiKeyType } from '@kbn/entityManager-plugin/server/saved_objects';
+import { getApiKeyManager } from './lib/entity_analytics/entity_store/auth/api_key';
 import { DEFAULT_SPACE_ID } from '../common/constants';
 import { AppClientFactory } from './client';
 import type { ConfigType } from './config';
@@ -78,7 +80,7 @@ export class RequestContextFactory implements IRequestContextFactory {
 
     const { lists, ruleRegistry, security } = plugins;
 
-    const [_, startPlugins] = await core.getStartServices();
+    const [coreStart, startPlugins] = await core.getStartServices();
     const frameworkRequest = await buildFrameworkRequest(context, request);
     const coreContext = await context.core;
     const licensing = await context.licensing;
@@ -233,7 +235,11 @@ export class RequestContextFactory implements IRequestContextFactory {
       getEntityStoreDataClient: memoize(() => {
         const clusterClient = coreContext.elasticsearch.client;
         const logger = options.logger;
-        const soClient = coreContext.savedObjects.client;
+
+        const soClient = coreContext.savedObjects.getClient({
+          includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
+        });
+
         return new EntityStoreDataClient({
           namespace: getSpaceId(),
           clusterClient,
@@ -247,6 +253,14 @@ export class RequestContextFactory implements IRequestContextFactory {
           config: config.entityAnalytics.entityStore,
           experimentalFeatures: config.experimentalFeatures,
           telemetry: core.analytics,
+          apiKeyManager: getApiKeyManager({
+            core: coreStart,
+            logger,
+            security: startPlugins.security,
+            encryptedSavedObjects: startPlugins.encryptedSavedObjects,
+            request,
+            namespace: getSpaceId(),
+          }),
         });
       }),
       getAssetInventoryClient: memoize(() => {

x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts

@@ -142,6 +142,7 @@ export default function ({ getService }: FtrProviderContext) {
         'endpoint:complete-external-response-actions',
         'endpoint:metadata-check-transforms-task',
         'endpoint:user-artifact-packager',
+        'entity_store:data_view:refresh',
         'entity_store:field_retention:enrichment',
         'fleet:bump_agent_policies',
         'fleet:check-deleted-files-task',