[eem] add entity definition state (#191933)

~blocked by https://github.com/elastic/kibana/issues/192004~

This change adds an `includeState: boolean` option to methods querying
entity definitions. When true this adds an `EntityDefinitionState`
object containing all the definition components and their state
(installed or not) and stats. Since this may only be used internally (eg
builtin definition installation process) and for troubleshooting,
`includeState` is false by default

#### Testing
- install a definition
- call `GET
kbn:/internal/entities/definition/<definition-id>?includeState=true`
- check and validate the definition `state` block
- manually remove transform/pipeline/template components
- check and validate the definition `state` block

x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts

@@ -6,8 +6,12 @@
  */
 
 import { z } from '@kbn/zod';
+import { BooleanFromString } from '@kbn/zod-helpers';
 
 export const getEntityDefinitionQuerySchema = z.object({
   page: z.optional(z.coerce.number()),
   perPage: z.optional(z.coerce.number()),
+  includeState: z.optional(BooleanFromString).default(false),
 });
+
+export type GetEntityDefinitionQuerySchema = z.infer<typeof getEntityDefinitionQuerySchema>;

x-pack/plugins/observability_solution/entity_manager/server/lib/client/index.ts

@@ -6,6 +6,7 @@
  */
 
 import type { IScopedClusterClient, SavedObjectsClientContract } from '@kbn/core/server';
+import { EntityDefinition } from '@kbn/entities-schema';
 import { findEntityDefinitions } from '../entities/find_entity_definition';
 import type { EntityDefinitionWithState } from '../entities/types';
 
@@ -16,7 +17,7 @@ export class EntityManagerClient {
   ) {}
 
   findEntityDefinitions({ page, perPage }: { page?: number; perPage?: number } = {}): Promise<
-    EntityDefinitionWithState[]
+    EntityDefinition[] | EntityDefinitionWithState[]
   > {
     return findEntityDefinitions({
       esClient: this.esClient.asCurrentUser,

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts

@@ -5,9 +5,10 @@
  * 2.0.
  */
 
-import { compact } from 'lodash';
+import { compact, forEach, reduce } from 'lodash';
 import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
+import { NodesIngestTotal } from '@elastic/elasticsearch/lib/api/types';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import {
   generateHistoryTransformId,
@@ -19,7 +20,7 @@ import {
   generateLatestIndexTemplateId,
 } from './helpers/generate_component_id';
 import { BUILT_IN_ID_PREFIX } from './built_in';
-import { EntityDefinitionWithState } from './types';
+import { EntityDefinitionState, EntityDefinitionWithState } from './types';
 import { isBackfillEnabled } from './helpers/is_backfill_enabled';
 
 export async function findEntityDefinitions({
@@ -29,6 +30,7 @@ export async function findEntityDefinitions({
   id,
   page = 1,
   perPage = 10,
+  includeState = false,
 }: {
   soClient: SavedObjectsClientContract;
   esClient: ElasticsearchClient;
@@ -36,7 +38,8 @@ export async function findEntityDefinitions({
   id?: string;
   page?: number;
   perPage?: number;
-}): Promise<EntityDefinitionWithState[]> {
+  includeState?: boolean;
+}): Promise<EntityDefinition[] | EntityDefinitionWithState[]> {
   const filter = compact([
     typeof builtIn === 'boolean'
       ? `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${BUILT_IN_ID_PREFIX}*)`
@@ -50,6 +53,10 @@ export async function findEntityDefinitions({
     perPage,
   });
 
+  if (!includeState) {
+    return response.saved_objects.map(({ attributes }) => attributes);
+  }
+
   return Promise.all(
     response.saved_objects.map(async ({ attributes }) => {
       const state = await getEntityDefinitionState(esClient, attributes);
@@ -62,15 +69,18 @@ export async function findEntityDefinitionById({
   id,
   esClient,
   soClient,
+  includeState = false,
 }: {
   id: string;
   esClient: ElasticsearchClient;
   soClient: SavedObjectsClientContract;
+  includeState?: boolean;
 }) {
   const [definition] = await findEntityDefinitions({
     esClient,
     soClient,
     id,
+    includeState,
     perPage: 1,
   });
 
@@ -80,43 +90,126 @@ export async function findEntityDefinitionById({
 async function getEntityDefinitionState(
   esClient: ElasticsearchClient,
   definition: EntityDefinition
-) {
-  const historyIngestPipelineId = generateHistoryIngestPipelineId(definition);
-  const latestIngestPipelineId = generateLatestIngestPipelineId(definition);
+): Promise<EntityDefinitionState> {
+  const [ingestPipelines, transforms, indexTemplates] = await Promise.all([
+    getIngestPipelineState({ definition, esClient }),
+    getTransformState({ definition, esClient }),
+    getIndexTemplatesState({ definition, esClient }),
+  ]);
+
+  const installed =
+    ingestPipelines.every((pipeline) => pipeline.installed) &&
+    transforms.every((transform) => transform.installed) &&
+    indexTemplates.every((template) => template.installed);
+  const running = transforms.every((transform) => transform.running);
+
+  return {
+    installed,
+    running,
+    components: { transforms, ingestPipelines, indexTemplates },
+  };
+}
+
+async function getTransformState({
+  definition,
+  esClient,
+}: {
+  definition: EntityDefinition;
+  esClient: ElasticsearchClient;
+}) {
   const transformIds = [
     generateHistoryTransformId(definition),
     generateLatestTransformId(definition),
     ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []),
   ];
-  const [ingestPipelines, indexTemplatesInstalled, transforms] = await Promise.all([
-    esClient.ingest.getPipeline(
-      {
-        id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
-      },
-      { ignore: [404] }
-    ),
-    esClient.indices.existsIndexTemplate({
-      name: `${
-        (generateLatestIndexTemplateId(definition), generateHistoryIndexTemplateId(definition))
-      }`,
-    }),
-    esClient.transform.getTransformStats({
-      transform_id: transformIds,
+
+  const transformStats = await Promise.all(
+    transformIds.map((id) => esClient.transform.getTransformStats({ transform_id: id }))
+  ).then((results) => results.map(({ transforms }) => transforms).flat());
+
+  return transformIds.map((id) => {
+    const stats = transformStats.find((transform) => transform.id === id);
+    if (!stats) {
+      return { id, installed: false, running: false };
+    }
+
+    return {
+      id,
+      installed: true,
+      running: stats.state === 'started' || stats.state === 'indexing',
+      stats,
+    };
+  });
+}
+
+async function getIngestPipelineState({
+  definition,
+  esClient,
+}: {
+  definition: EntityDefinition;
+  esClient: ElasticsearchClient;
+}) {
+  const ingestPipelineIds = [
+    generateHistoryIngestPipelineId(definition),
+    generateLatestIngestPipelineId(definition),
+  ];
+  const [ingestPipelines, ingestPipelinesStats] = await Promise.all([
+    esClient.ingest.getPipeline({ id: ingestPipelineIds.join(',') }, { ignore: [404] }),
+    esClient.nodes.stats({
+      metric: 'ingest',
+      filter_path: ingestPipelineIds.map((id) => `nodes.*.ingest.pipelines.${id}`),
     }),
   ]);
 
-  const ingestPipelinesInstalled = !!(
-    ingestPipelines[historyIngestPipelineId] && ingestPipelines[latestIngestPipelineId]
+  const ingestStatsByPipeline = reduce(
+    ingestPipelinesStats.nodes,
+    (pipelines, { ingest }) => {
+      forEach(ingest?.pipelines, (value: NodesIngestTotal, key: string) => {
+        if (!pipelines[key]) {
+          pipelines[key] = { count: 0, failed: 0 };
+        }
+        pipelines[key].count += value.count ?? 0;
+        pipelines[key].failed += value.failed ?? 0;
+      });
+      return pipelines;
+    },
+    {} as Record<string, { count: number; failed: number }>
   );
-  const transformsInstalled = transforms.count === transformIds.length;
-  const transformsRunning =
-    transformsInstalled &&
-    transforms.transforms.every(
-      (transform) => transform.state === 'started' || transform.state === 'indexing'
-    );
 
-  return {
-    installed: ingestPipelinesInstalled && transformsInstalled && indexTemplatesInstalled,
-    running: transformsRunning,
-  };
+  return ingestPipelineIds.map((id) => ({
+    id,
+    installed: !!ingestPipelines[id],
+    stats: ingestStatsByPipeline[id],
+  }));
+}
+
+async function getIndexTemplatesState({
+  definition,
+  esClient,
+}: {
+  definition: EntityDefinition;
+  esClient: ElasticsearchClient;
+}) {
+  const indexTemplatesIds = [
+    generateLatestIndexTemplateId(definition),
+    generateHistoryIndexTemplateId(definition),
+  ];
+  const templates = await Promise.all(
+    indexTemplatesIds.map((id) =>
+      esClient.indices
+        .getIndexTemplate({ name: id }, { ignore: [404] })
+        .then(({ index_templates: indexTemplates }) => indexTemplates?.[0])
+    )
+  ).then(compact);
+  return indexTemplatesIds.map((id) => {
+    const template = templates.find(({ name }) => name === id);
+    if (!template) {
+      return { id, installed: false };
+    }
+    return {
+      id,
+      installed: true,
+      stats: template.index_template,
+    };
+  });
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts

@@ -354,6 +354,7 @@ describe('install_entity_definition', () => {
         version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
       };
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
+      esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
       const soClient = savedObjectsClientMock.create();
 
       soClient.find.mockResolvedValueOnce({
@@ -391,6 +392,7 @@ describe('install_entity_definition', () => {
         version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
       };
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
+      esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
       const soClient = savedObjectsClientMock.create();
 
       soClient.find.mockResolvedValueOnce({
@@ -426,6 +428,7 @@ describe('install_entity_definition', () => {
 
     it('should reinstall when failed installation', async () => {
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
+      esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
       const soClient = savedObjectsClientMock.create();
 
       soClient.find.mockResolvedValueOnce({

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts

@@ -39,8 +39,8 @@ import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_
 import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
 import { EntityIdConflict } from './errors/entity_id_conflict_error';
 import { EntityDefinitionNotFound } from './errors/entity_not_found';
-import { EntityDefinitionWithState } from './types';
 import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update';
+import { EntityDefinitionWithState } from './types';
 import { stopTransforms } from './stop_transforms';
 import { deleteTransforms } from './delete_transforms';
 
@@ -136,6 +136,7 @@ export async function installBuiltInEntityDefinitions({
       esClient,
       soClient,
       id: builtInDefinition.id,
+      includeState: true,
     });
 
     if (!installedDefinition) {
@@ -148,7 +149,12 @@ export async function installBuiltInEntityDefinitions({
     }
 
     // verify existing installation
-    if (!shouldReinstallBuiltinDefinition(installedDefinition, builtInDefinition)) {
+    if (
+      !shouldReinstallBuiltinDefinition(
+        installedDefinition as EntityDefinitionWithState,
+        builtInDefinition
+      )
+    ) {
       return installedDefinition;
     }
 

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts

@@ -5,12 +5,43 @@
  * 2.0.
  */
 
+import {
+  IndicesIndexTemplate,
+  TransformGetTransformStatsTransformStats,
+} from '@elastic/elasticsearch/lib/api/types';
 import { EntityDefinition } from '@kbn/entities-schema';
 
+interface TransformState {
+  id: string;
+  installed: boolean;
+  running: boolean;
+  stats?: TransformGetTransformStatsTransformStats;
+}
+
+interface IngestPipelineState {
+  id: string;
+  installed: boolean;
+  stats?: { count: number; failed: number };
+}
+
+interface IndexTemplateState {
+  id: string;
+  installed: boolean;
+  stats?: IndicesIndexTemplate;
+}
+
 // state is the *live* state of the definition. since a definition
 // is composed of several elasticsearch components that can be
 // modified or deleted outside of the entity manager apis, this can
 // be used to verify the actual installation is complete and running
-export type EntityDefinitionWithState = EntityDefinition & {
-  state: { installed: boolean; running: boolean };
-};
+export interface EntityDefinitionState {
+  installed: boolean;
+  running: boolean;
+  components: {
+    transforms: TransformState[];
+    ingestPipelines: IngestPipelineState[];
+    indexTemplates: IndexTemplateState[];
+  };
+}
+
+export type EntityDefinitionWithState = EntityDefinition & { state: EntityDefinitionState };

x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts

@@ -70,12 +70,24 @@ export class EntityClient {
     });
   }
 
-  async getEntityDefinitions({ page = 1, perPage = 10 }: { page?: number; perPage?: number }) {
+  async getEntityDefinitions({
+    id,
+    page = 1,
+    perPage = 10,
+    includeState = false,
+  }: {
+    id?: string;
+    page?: number;
+    perPage?: number;
+    includeState?: boolean;
+  }) {
     const definitions = await findEntityDefinitions({
       esClient: this.options.esClient,
       soClient: this.options.soClient,
       page,
       perPage,
+      id,
+      includeState,
     });
 
     return { definitions };

x-pack/plugins/observability_solution/entity_manager/server/lib/utils.ts

@@ -19,7 +19,7 @@ export const getClientsFromAPIKey = ({
   server: EntityManagerServerSetup;
 }): { esClient: ElasticsearchClient; soClient: SavedObjectsClientContract } => {
   const fakeRequest = getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey });
-  const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asCurrentUser;
+  const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asSecondaryAuthUser;
   const soClient = server.core.savedObjects.getScopedClient(fakeRequest);
   return { esClient, soClient };
 };

x-pack/plugins/observability_solution/entity_manager/server/plugin.ts

@@ -99,7 +99,7 @@ export class EntityManagerServerPlugin
     request: KibanaRequest;
     coreStart: CoreStart;
   }) {
-    const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser;
+    const esClient = coreStart.elasticsearch.client.asScoped(request).asSecondaryAuthUser;
     const soClient = coreStart.savedObjects.getScopedClient(request);
     return new EntityClient({ esClient, soClient, logger: this.logger });
   }

x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts

@@ -17,6 +17,7 @@ import { checkIfEntityDiscoveryAPIKeyIsValid, readEntityDiscoveryAPIKey } from '
 import { builtInDefinitions } from '../../lib/entities/built_in';
 import { findEntityDefinitions } from '../../lib/entities/find_entity_definition';
 import { getClientsFromAPIKey } from '../../lib/utils';
+import { EntityDefinitionWithState } from '../../lib/entities/types';
 import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
 
 /**
@@ -68,9 +69,13 @@ export const checkEntityDiscoveryEnabledRoute = createEntityManagerServerRoute({
             esClient,
             soClient,
             id: builtInDefinition.id,
+            includeState: true,
           });
 
-          return { installedDefinition: definitions[0], builtInDefinition };
+          return {
+            installedDefinition: definitions[0] as EntityDefinitionWithState,
+            builtInDefinition,
+          };
         })
       ).then((results) =>
         results.reduce(

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts

@@ -4,7 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 import { getEntityDefinitionQuerySchema } from '@kbn/entities-schema';
 import { z } from '@kbn/zod';
 import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
@@ -17,6 +16,12 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  *     tags:
  *       - definitions
  *     parameters:
+ *       - in: path
+ *         name: id
+ *         description: The entity definition ID
+ *         schema:
+ *           $ref: '#/components/schemas/deleteEntityDefinitionParamsSchema/properties/id'
+ *         required: false
  *       - in: query
  *         name: page
  *         schema:
@@ -25,6 +30,10 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  *         name: perPage
  *         schema:
  *           $ref: '#/components/schemas/getEntityDefinitionQuerySchema/properties/perPage'
+ *       - in: query
+ *         name: includeState
+ *         schema:
+ *           $ref: '#/components/schemas/getEntityDefinitionQuerySchema/properties/includeState'
  *     responses:
  *       200:
  *         description: OK
@@ -38,27 +47,21 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  *                   items:
  *                     allOf:
  *                       - $ref: '#/components/schemas/entityDefinitionSchema'
- *                       - type: object
- *                         properties:
- *                           state:
- *                            type: object
- *                            properties:
- *                              installed:
- *                                type: boolean
- *                              running:
- *                                type: boolean
  */
 export const getEntityDefinitionRoute = createEntityManagerServerRoute({
-  endpoint: 'GET /internal/entities/definition',
+  endpoint: 'GET /internal/entities/definition/{id?}',
   params: z.object({
     query: getEntityDefinitionQuerySchema,
+    path: z.object({ id: z.optional(z.string()) }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
       const client = await getScopedClient({ request });
       const result = await client.getEntityDefinitions({
-        page: params?.query?.page,
-        perPage: params?.query?.perPage,
+        id: params.path?.id,
+        page: params.query.page,
+        perPage: params.query.perPage,
+        includeState: params.query.includeState,
       });
 
       return response.ok({ body: result });

x-pack/test/api_integration/apis/entity_manager/helpers/request.ts

@@ -16,11 +16,12 @@ export interface Auth {
 
 export const getInstalledDefinitions = async (
   supertest: Agent,
-  params: { auth?: Auth; id?: string } = {}
+  params: { auth?: Auth; id?: string; includeState?: boolean } = {}
 ): Promise<{ definitions: EntityDefinitionWithState[] }> => {
-  const { auth, id } = params;
+  const { auth, id, includeState = true } = params;
   let req = supertest
     .get(`/internal/entities/definition${id ? `/${id}` : ''}`)
+    .query({ includeState })
     .set('kbn-xsrf', 'xxx');
   if (auth) {
     req = req.auth(auth.username, auth.password);