[eem] definition update endpoint (#190648)

Create `PATCH kbn:/internal/entities/definition` allowing update of
stored definition. The endpoint accepts an update object representing a
partial entity definition with a few key properties that cannot be
updated. The update process will stop and delete the existing transforms
and create new ones, there's
https://github.com/elastic/elastic-entity-model/issues/136 logged as a
follow up improvement.

### Testing
- call `PUT kbn:/internal/entities/definition/{id}` with an update
payload (see [update
schema](7a7fbdf1cd/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts (L62)))
- call `GET kbn:/internal/entities/definition` and verify the update in
reflected in the stored definition
- verify the updated properties are reflected in the corresponding
transform/pipeline components

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/kbn-entities-schema/src/rest_spec/create.ts

@@ -6,9 +6,10 @@
  */
 
 import { z } from '@kbn/zod';
+import { BooleanFromString } from '@kbn/zod-helpers';
 
 export const createEntityDefinitionQuerySchema = z.object({
-  installOnly: z.optional(z.coerce.boolean()).default(false),
+  installOnly: z.optional(BooleanFromString).default(false),
 });
 
 export type CreateEntityDefinitionQuery = z.infer<typeof createEntityDefinitionQuerySchema>;

x-pack/packages/kbn-entities-schema/src/rest_spec/delete.ts

@@ -6,13 +6,14 @@
  */
 
 import { z } from '@kbn/zod';
+import { BooleanFromString } from '@kbn/zod-helpers';
 
 export const deleteEntityDefinitionParamsSchema = z.object({
   id: z.string(),
 });
 
 export const deleteEntityDefinitionQuerySchema = z.object({
-  deleteData: z.optional(z.coerce.boolean().default(false)),
+  deleteData: z.optional(BooleanFromString).default(false),
 });
 
 export type DeleteEntityDefinitionQuery = z.infer<typeof deleteEntityDefinitionQuerySchema>;

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -59,4 +59,20 @@ export const entityDefinitionSchema = z.object({
   installStartedAt: z.optional(z.string()),
 });
 
+export const entityDefinitionUpdateSchema = entityDefinitionSchema
+  .omit({
+    id: true,
+    managed: true,
+    installStatus: true,
+    installStartedAt: true,
+  })
+  .partial()
+  .merge(
+    z.object({
+      history: z.optional(entityDefinitionSchema.shape.history.partial()),
+      version: semVerSchema,
+    })
+  );
+
 export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;
+export type EntityDefinitionUpdate = z.infer<typeof entityDefinitionUpdateSchema>;

x-pack/packages/kbn-entities-schema/tsconfig.json

@@ -16,5 +16,6 @@
   ],
   "kbn_references": [
     "@kbn/zod",
+    "@kbn/zod-helpers",
   ]
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts

@@ -42,9 +42,6 @@ export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
         syncDelay: '2m',
       },
     },
-    latest: {
-      lookback: '5m',
-    },
     identityFields: ['service.name', { field: 'service.environment', optional: true }],
     displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}',
     metadata: [

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts

@@ -58,6 +58,25 @@ export async function findEntityDefinitions({
   );
 }
 
+export async function findEntityDefinitionById({
+  id,
+  esClient,
+  soClient,
+}: {
+  id: string;
+  esClient: ElasticsearchClient;
+  soClient: SavedObjectsClientContract;
+}) {
+  const [definition] = await findEntityDefinitions({
+    esClient,
+    soClient,
+    id,
+    perPage: 1,
+  });
+
+  return definition;
+}
+
 async function getEntityDefinitionState(
   esClient: ElasticsearchClient,
   definition: EntityDefinition

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/merge_definition_update.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
+import { mergeWith, omit } from 'lodash';
+
+export function mergeEntityDefinitionUpdate(
+  definition: EntityDefinition,
+  update: EntityDefinitionUpdate
+) {
+  const updatedDefinition = mergeWith(definition, update, (value, other) => {
+    // we don't want to merge arrays (metrics, metadata..) but overwrite them
+    if (Array.isArray(value)) {
+      return other;
+    }
+  });
+
+  return omit(updatedDefinition, ['state']) as EntityDefinition;
+}

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts

@@ -8,7 +8,7 @@
 import semver from 'semver';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { EntityDefinition } from '@kbn/entities-schema';
+import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
 import {
   generateHistoryIndexTemplateId,
@@ -26,7 +26,7 @@ import {
 import { validateDefinitionCanCreateValidTransformIds } from './transform/validate_transform_ids';
 import { deleteEntityDefinition } from './delete_entity_definition';
 import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
-import { findEntityDefinitions } from './find_entity_definition';
+import { findEntityDefinitionById } from './find_entity_definition';
 import {
   entityDefinitionExists,
   saveEntityDefinition,
@@ -44,6 +44,7 @@ import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities
 import { EntityIdConflict } from './errors/entity_id_conflict_error';
 import { EntityDefinitionNotFound } from './errors/entity_not_found';
 import { EntityDefinitionWithState } from './types';
+import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update';
 
 export interface InstallDefinitionParams {
   esClient: ElasticsearchClient;
@@ -140,13 +141,13 @@ export async function installBuiltInEntityDefinitions({
 
   logger.debug(`Starting installation of ${definitions.length} built-in definitions`);
   const installPromises = definitions.map(async (builtInDefinition) => {
-    const installedDefinitions = await findEntityDefinitions({
+    const installedDefinition = await findEntityDefinitionById({
       esClient,
       soClient,
       id: builtInDefinition.id,
     });
 
-    if (installedDefinitions.length === 0) {
+    if (!installedDefinition) {
       return await installEntityDefinition({
         definition: builtInDefinition,
         esClient,
@@ -156,20 +157,19 @@ export async function installBuiltInEntityDefinitions({
     }
 
     // verify existing installation
-    const installedDefinition = installedDefinitions[0];
-    if (!shouldReinstall(installedDefinition, builtInDefinition)) {
+    if (!shouldReinstallBuiltinDefinition(installedDefinition, builtInDefinition)) {
       return installedDefinition;
     }
 
     logger.debug(
       `Detected failed or outdated installation of definition [${installedDefinition.id}] v${installedDefinition.version}, installing v${builtInDefinition.version}`
     );
-    return await reinstall({
+    return await reinstallEntityDefinition({
       soClient,
       esClient,
       logger,
       definition: installedDefinition,
-      latestDefinition: builtInDefinition,
+      definitionUpdate: builtInDefinition,
     });
   });
 
@@ -223,30 +223,36 @@ async function install({
   ]).then(throwIfRejected);
 
   await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed' });
-
   return { ...definition, installStatus: 'installed' };
 }
 
 // stop and delete the current transforms and reinstall all the components
-async function reinstall({
+export async function reinstallEntityDefinition({
   esClient,
   soClient,
   definition,
-  latestDefinition,
+  definitionUpdate,
   logger,
-}: InstallDefinitionParams & { latestDefinition: EntityDefinition }): Promise<EntityDefinition> {
-  logger.debug(
-    `Reinstalling definition ${definition.id} from v${definition.version} to v${latestDefinition.version}`
-  );
-
+}: InstallDefinitionParams & {
+  definitionUpdate: EntityDefinitionUpdate;
+}): Promise<EntityDefinition> {
   try {
-    await updateEntityDefinition(soClient, latestDefinition.id, {
-      ...latestDefinition,
+    const updatedDefinition = mergeEntityDefinitionUpdate(definition, definitionUpdate);
+
+    logger.debug(
+      () =>
+        `Reinstalling definition ${definition.id} from v${definition.version} to v${
+          definitionUpdate.version
+        }\n${JSON.stringify(updatedDefinition, null, 2)}`
+    );
+
+    await updateEntityDefinition(soClient, definition.id, {
+      ...updatedDefinition,
       installStatus: 'upgrading',
       installStartedAt: new Date().toISOString(),
     });
 
-    logger.debug(`Stopping transforms for definition ${definition.id} v${definition.version}`);
+    logger.debug(`Deleting transforms for definition ${definition.id} v${definition.version}`);
     await Promise.all([
       stopAndDeleteHistoryTransform(esClient, definition, logger),
       isBackfillEnabled(definition)
@@ -259,10 +265,10 @@ async function reinstall({
       esClient,
       soClient,
       logger,
-      definition: latestDefinition,
+      definition: updatedDefinition,
     });
   } catch (err) {
-    await updateEntityDefinition(soClient, latestDefinition.id, {
+    await updateEntityDefinition(soClient, definition.id, {
       installStatus: 'failed',
     });
 
@@ -271,19 +277,34 @@ async function reinstall({
 }
 
 const INSTALLATION_TIMEOUT = 5 * 60 * 1000;
-const shouldReinstall = (
-  definition: EntityDefinitionWithState,
-  latestDefinition: EntityDefinition
-) => {
+export const installationInProgress = (definition: EntityDefinition) => {
   const { installStatus, installStartedAt } = definition;
 
-  const isStale =
+  return (
     (installStatus === 'installing' || installStatus === 'upgrading') &&
-    Date.now() - Date.parse(installStartedAt!) >= INSTALLATION_TIMEOUT;
-  const isOutdated =
-    installStatus === 'installed' && semver.neq(definition.version, latestDefinition.version);
-  const isFailed = installStatus === 'failed';
-  const isPartial = installStatus === 'installed' && !definition.state.installed;
-
-  return isStale || isOutdated || isFailed || isPartial;
+    Date.now() - Date.parse(installStartedAt!) < INSTALLATION_TIMEOUT
+  );
+};
+
+const installationTimedOut = (definition: EntityDefinition) => {
+  const { installStatus, installStartedAt } = definition;
+
+  return (
+    (installStatus === 'installing' || installStatus === 'upgrading') &&
+    Date.now() - Date.parse(installStartedAt!) >= INSTALLATION_TIMEOUT
+  );
+};
+
+const shouldReinstallBuiltinDefinition = (
+  installedDefinition: EntityDefinitionWithState,
+  latestDefinition: EntityDefinition
+) => {
+  const { installStatus, version, state } = installedDefinition;
+
+  const timedOut = installationTimedOut(installedDefinition);
+  const outdated = installStatus === 'installed' && semver.neq(version, latestDefinition.version);
+  const failed = installStatus === 'failed';
+  const partial = installStatus === 'installed' && !state.installed;
+
+  return timedOut || outdated || failed || partial;
 };

x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts

@@ -10,7 +10,6 @@ import {
   CreateEntityDefinitionQuery,
   createEntityDefinitionQuerySchema,
 } from '@kbn/entities-schema';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import { SetupRouteOptions } from '../types';
 import {
   canEnableEntityDiscovery,
@@ -71,7 +70,7 @@ export function enableEntityDiscoveryRoute<T extends RequestHandlerContext>({
     {
       path: '/internal/entities/managed/enablement',
       validate: {
-        query: buildRouteValidationWithZod(createEntityDefinitionQuerySchema),
+        query: createEntityDefinitionQuerySchema,
       },
     },
     async (context, req, res) => {

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts

@@ -12,13 +12,13 @@ import {
   createEntityDefinitionQuerySchema,
   CreateEntityDefinitionQuery,
 } from '@kbn/entities-schema';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import { SetupRouteOptions } from '../types';
 import { EntityIdConflict } from '../../lib/entities/errors/entity_id_conflict_error';
 import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception';
 import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error';
 import { startTransform } from '../../lib/entities/start_transform';
 import { installEntityDefinition } from '../../lib/entities/install_entity_definition';
+import { EntityDefinitionIdInvalid } from '../../lib/entities/errors/entity_definition_id_invalid';
 
 /**
  * @openapi
@@ -62,8 +62,8 @@ export function createEntityDefinitionRoute<T extends RequestHandlerContext>({
     {
       path: '/internal/entities/definition',
       validate: {
-        body: buildRouteValidationWithZod(entityDefinitionSchema.strict()),
-        query: buildRouteValidationWithZod(createEntityDefinitionQuerySchema),
+        body: entityDefinitionSchema.strict(),
+        query: createEntityDefinitionQuerySchema,
       },
     },
     async (context, req, res) => {
@@ -71,6 +71,7 @@ export function createEntityDefinitionRoute<T extends RequestHandlerContext>({
       const core = await context.core;
       const soClient = core.savedObjects.client;
       const esClient = core.elasticsearch.client.asCurrentUser;
+
       try {
         const definition = await installEntityDefinition({
           soClient,
@@ -85,12 +86,20 @@ export function createEntityDefinitionRoute<T extends RequestHandlerContext>({
 
         return res.ok({ body: definition });
       } catch (e) {
+        logger.error(e);
+
+        if (e instanceof EntityDefinitionIdInvalid) {
+          return res.badRequest({ body: e });
+        }
+
         if (e instanceof EntityIdConflict) {
           return res.conflict({ body: e });
         }
+
         if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) {
           return res.customError({ body: e, statusCode: 400 });
         }
+
         return res.customError({ body: e, statusCode: 500 });
       }
     }

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts

@@ -6,7 +6,6 @@
  */
 
 import { RequestHandlerContext } from '@kbn/core/server';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import {
   deleteEntityDefinitionParamsSchema,
   deleteEntityDefinitionQuerySchema,
@@ -55,18 +54,18 @@ import { uninstallEntityDefinition } from '../../lib/entities/uninstall_entity_d
 export function deleteEntityDefinitionRoute<T extends RequestHandlerContext>({
   router,
   server,
+  logger,
 }: SetupRouteOptions<T>) {
   router.delete<{ id: string }, { deleteData?: boolean }, unknown>(
     {
       path: '/internal/entities/definition/{id}',
       validate: {
-        params: buildRouteValidationWithZod(deleteEntityDefinitionParamsSchema.strict()),
-        query: buildRouteValidationWithZod(deleteEntityDefinitionQuerySchema.strict()),
+        params: deleteEntityDefinitionParamsSchema.strict(),
+        query: deleteEntityDefinitionQuerySchema.strict(),
       },
     },
     async (context, req, res) => {
       try {
-        const { logger } = server;
         const soClient = (await context.core).savedObjects.client;
         const esClient = (await context.core).elasticsearch.client.asCurrentUser;
 
@@ -81,6 +80,8 @@ export function deleteEntityDefinitionRoute<T extends RequestHandlerContext>({
 
         return res.ok({ body: { acknowledged: true } });
       } catch (e) {
+        logger.error(e);
+
         if (e instanceof EntityDefinitionNotFound) {
           return res.notFound({ body: e });
         }

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
+import { z } from '@kbn/zod';
 import { RequestHandlerContext } from '@kbn/core/server';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import { getEntityDefinitionQuerySchema } from '@kbn/entities-schema';
 import { SetupRouteOptions } from '../types';
 import { findEntityDefinitions } from '../../lib/entities/find_entity_definition';
@@ -52,12 +52,14 @@ import { findEntityDefinitions } from '../../lib/entities/find_entity_definition
  */
 export function getEntityDefinitionRoute<T extends RequestHandlerContext>({
   router,
+  logger,
 }: SetupRouteOptions<T>) {
-  router.get<unknown, { page?: number; perPage?: number }, unknown>(
+  router.get<{ id?: string }, { page?: number; perPage?: number }, unknown>(
     {
-      path: '/internal/entities/definition',
+      path: '/internal/entities/definition/{id?}',
       validate: {
-        query: buildRouteValidationWithZod(getEntityDefinitionQuerySchema.strict()),
+        query: getEntityDefinitionQuerySchema.strict(),
+        params: z.object({ id: z.optional(z.string()) }),
       },
     },
     async (context, req, res) => {
@@ -69,9 +71,11 @@ export function getEntityDefinitionRoute<T extends RequestHandlerContext>({
           soClient,
           page: req.query.page ?? 1,
           perPage: req.query.perPage ?? 10,
+          id: req.params.id,
         });
         return res.ok({ body: { definitions } });
       } catch (e) {
+        logger.error(e);
         return res.customError({ body: e, statusCode: 500 });
       }
     }

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/reset.ts

@@ -6,7 +6,6 @@
  */
 
 import { RequestHandlerContext } from '@kbn/core/server';
-import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
 import { resetEntityDefinitionParamsSchema } from '@kbn/entities-schema';
 import { SetupRouteOptions } from '../types';
 import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception';
@@ -43,7 +42,7 @@ export function resetEntityDefinitionRoute<T extends RequestHandlerContext>({
     {
       path: '/internal/entities/definition/{id}/_reset',
       validate: {
-        params: buildRouteValidationWithZod(resetEntityDefinitionParamsSchema.strict()),
+        params: resetEntityDefinitionParamsSchema.strict(),
       },
     },
     async (context, req, res) => {
@@ -75,6 +74,8 @@ export function resetEntityDefinitionRoute<T extends RequestHandlerContext>({
 
         return res.ok({ body: { acknowledged: true } });
       } catch (e) {
+        logger.error(e);
+
         if (e instanceof EntityDefinitionNotFound) {
           return res.notFound({ body: e });
         }

x-pack/plugins/observability_solution/entity_manager/server/routes/entities/update.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import { RequestHandlerContext } from '@kbn/core/server';
+import {
+  createEntityDefinitionQuerySchema,
+  CreateEntityDefinitionQuery,
+  entityDefinitionUpdateSchema,
+  EntityDefinitionUpdate,
+} from '@kbn/entities-schema';
+import { SetupRouteOptions } from '../types';
+import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception';
+import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error';
+import { startTransform } from '../../lib/entities/start_transform';
+import {
+  installationInProgress,
+  reinstallEntityDefinition,
+} from '../../lib/entities/install_entity_definition';
+import { findEntityDefinitionById } from '../../lib/entities/find_entity_definition';
+
+/**
+ * @openapi
+ * /internal/entities/definition:
+ *   put:
+ *     description: Update an entity definition.
+ *     tags:
+ *       - definitions
+ *     parameters:
+ *       - in: query
+ *         name: installOnly
+ *         description: If true, the definition transforms will not be started
+ *         required: false
+ *         schema:
+ *           type: boolean
+ *           default: false
+ *     requestBody:
+ *       description: The definition properties to update
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/entityDefinitionUpdateSchema'
+ *     responses:
+ *       200:
+ *         description: Success
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/entityDefinitionSchema'
+ *       400:
+ *         description: The entity definition cannot be installed; see the error for more details
+ *       404:
+ *         description: The entity definition does not exist
+ *       403:
+ *         description: User is not allowed to update the entity definition
+ *       409:
+ *         description: The entity definition is being updated by another request
+ */
+export function updateEntityDefinitionRoute<T extends RequestHandlerContext>({
+  router,
+  server,
+}: SetupRouteOptions<T>) {
+  router.patch<{ id: string }, CreateEntityDefinitionQuery, EntityDefinitionUpdate>(
+    {
+      path: '/internal/entities/definition/{id}',
+      validate: {
+        body: entityDefinitionUpdateSchema.strict(),
+        query: createEntityDefinitionQuerySchema,
+        params: z.object({ id: z.string() }),
+      },
+    },
+    async (context, req, res) => {
+      const { logger } = server;
+      const core = await context.core;
+      const soClient = core.savedObjects.client;
+      const esClient = core.elasticsearch.client.asCurrentUser;
+
+      try {
+        const installedDefinition = await findEntityDefinitionById({
+          soClient,
+          esClient,
+          id: req.params.id,
+        });
+
+        if (!installedDefinition) {
+          return res.notFound({
+            body: { message: `Entity definition [${req.params.id}] not found` },
+          });
+        }
+
+        if (installedDefinition.managed) {
+          return res.forbidden({
+            body: { message: `Managed definition cannot be modified` },
+          });
+        }
+
+        if (installationInProgress(installedDefinition)) {
+          return res.conflict({
+            body: { message: `Entity definition [${req.params.id}] has changes in progress` },
+          });
+        }
+
+        const updatedDefinition = await reinstallEntityDefinition({
+          soClient,
+          esClient,
+          logger,
+          definition: installedDefinition,
+          definitionUpdate: req.body,
+        });
+
+        if (!req.query.installOnly) {
+          await startTransform(esClient, updatedDefinition, logger);
+        }
+
+        return res.ok({ body: updatedDefinition });
+      } catch (e) {
+        logger.error(e);
+
+        if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) {
+          return res.customError({ body: e, statusCode: 400 });
+        }
+        return res.customError({ body: e, statusCode: 500 });
+      }
+    }
+  );
+}

x-pack/plugins/observability_solution/entity_manager/server/routes/index.ts

@@ -11,6 +11,7 @@ import { createEntityDefinitionRoute } from './entities/create';
 import { deleteEntityDefinitionRoute } from './entities/delete';
 import { resetEntityDefinitionRoute } from './entities/reset';
 import { getEntityDefinitionRoute } from './entities/get';
+import { updateEntityDefinitionRoute } from './entities/update';
 import { checkEntityDiscoveryEnabledRoute } from './enablement/check';
 import { enableEntityDiscoveryRoute } from './enablement/enable';
 import { disableEntityDiscoveryRoute } from './enablement/disable';
@@ -23,4 +24,5 @@ export function setupRoutes<T extends RequestHandlerContext>(dependencies: Setup
   checkEntityDiscoveryEnabledRoute<T>(dependencies);
   enableEntityDiscoveryRoute<T>(dependencies);
   disableEntityDiscoveryRoute<T>(dependencies);
+  updateEntityDefinitionRoute<T>(dependencies);
 }

x-pack/plugins/observability_solution/entity_manager/tsconfig.json

@@ -24,10 +24,10 @@
     "@kbn/core-saved-objects-api-server-mocks",
     "@kbn/entities-schema",
     "@kbn/es-query",
-    "@kbn/zod-helpers",
     "@kbn/security-plugin",
     "@kbn/encrypted-saved-objects-plugin",
     "@kbn/logging-mocks",
     "@kbn/licensing-plugin",
+    "@kbn/zod",
   ]
 }

x-pack/test/api_integration/apis/entity_manager/builtin_definitions.ts

@@ -73,10 +73,9 @@ export default function ({ getService }: FtrProviderContext) {
           const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
           expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
 
-          const definitionsResponse = await getInstalledDefinitions(
-            supertestWithoutAuth,
-            authorizedUser
-          );
+          const definitionsResponse = await getInstalledDefinitions(supertestWithoutAuth, {
+            auth: authorizedUser,
+          });
           expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
           expect(
             builtInDefinitions.every((builtin) =>
@@ -91,7 +90,7 @@ export default function ({ getService }: FtrProviderContext) {
           );
 
           const disableResponse = await disableEntityDiscovery(authorizedUser, 200, {
-            deleteData: false,
+            deleteData: true,
           });
           expect(disableResponse.success).to.eql(
             true,
@@ -121,7 +120,9 @@ export default function ({ getService }: FtrProviderContext) {
 
           await disableEntityDiscovery(unauthorizedUser, 403);
 
-          const disableResponse = await disableEntityDiscovery(authorizedUser, 200);
+          const disableResponse = await disableEntityDiscovery(authorizedUser, 200, {
+            deleteData: true,
+          });
           expect(disableResponse.success).to.eql(true, "authorized user can't disable EEM");
         });
       });
@@ -165,7 +166,7 @@ export default function ({ getService }: FtrProviderContext) {
           )
         ).to.eql(true, 'all builtin definitions are not installed/running');
 
-        await disableEntityDiscovery(authorizedUser, 200);
+        await disableEntityDiscovery(authorizedUser, 200, { deleteData: true });
       });
     });
 
@@ -186,7 +187,7 @@ export default function ({ getService }: FtrProviderContext) {
         isInstalledAndRunning(mockBuiltInEntityDefinition, definitionsResponse.definitions)
       ).to.ok();
 
-      await disableEntityDiscovery(authorizedUser, 200);
+      await disableEntityDiscovery(authorizedUser, 200, { deleteData: true });
     });
   });
 }

x-pack/test/api_integration/apis/entity_manager/definitions.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import semver from 'semver';
 import expect from '@kbn/expect';
 import { entityLatestSchema } from '@kbn/entities-schema';
 import {
@@ -14,7 +15,12 @@ import {
 import { PartialConfig, cleanup, generate } from '@kbn/data-forge';
 import { generateLatestIndexName } from '@kbn/entityManager-plugin/server/lib/entities/helpers/generate_component_id';
 import { FtrProviderContext } from '../../ftr_provider_context';
-import { installDefinition, uninstallDefinition, getInstalledDefinitions } from './helpers/request';
+import {
+  installDefinition,
+  uninstallDefinition,
+  updateDefinition,
+  getInstalledDefinitions,
+} from './helpers/request';
 import { waitForDocumentInIndex } from '../../../alerting_api_integration/observability/helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
@@ -27,43 +33,98 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Entity definitions', () => {
     describe('definitions installations', () => {
       it('can install multiple definitions', async () => {
-        await installDefinition(supertest, mockDefinition);
-        await installDefinition(supertest, mockBackfillDefinition);
+        await installDefinition(supertest, { definition: mockDefinition });
+        await installDefinition(supertest, { definition: mockBackfillDefinition });
 
         const { definitions } = await getInstalledDefinitions(supertest);
         expect(definitions.length).to.eql(2);
         expect(
-          definitions.find(
+          definitions.some(
             (definition) =>
               definition.id === mockDefinition.id &&
               definition.state.installed === true &&
               definition.state.running === true
           )
-        );
+        ).to.eql(true);
         expect(
-          definitions.find(
+          definitions.some(
             (definition) =>
               definition.id === mockBackfillDefinition.id &&
               definition.state.installed === true &&
               definition.state.running === true
           )
-        );
+        ).to.eql(true);
 
-        await uninstallDefinition(supertest, mockDefinition.id);
-        await uninstallDefinition(supertest, mockBackfillDefinition.id);
+        await Promise.all([
+          uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }),
+          uninstallDefinition(supertest, { id: mockBackfillDefinition.id, deleteData: true }),
+        ]);
       });
 
       it('does not start transforms when specified', async () => {
-        await installDefinition(supertest, mockDefinition, { installOnly: true });
+        await installDefinition(supertest, { definition: mockDefinition, installOnly: true });
 
         const { definitions } = await getInstalledDefinitions(supertest);
         expect(definitions.length).to.eql(1);
         expect(definitions[0].state.installed).to.eql(true);
         expect(definitions[0].state.running).to.eql(false);
 
-        await uninstallDefinition(supertest, mockDefinition.id);
+        await uninstallDefinition(supertest, { id: mockDefinition.id });
       });
     });
+
+    describe('definitions update', () => {
+      it('returns 404 if the definitions does not exist', async () => {
+        await updateDefinition(supertest, {
+          id: 'i-dont-exist',
+          update: { version: '1.0.0' },
+          expectedCode: 404,
+        });
+      });
+
+      it('accepts partial updates', async () => {
+        const incVersion = semver.inc(mockDefinition.version, 'major');
+        await installDefinition(supertest, { definition: mockDefinition, installOnly: true });
+        await updateDefinition(supertest, {
+          id: mockDefinition.id,
+          update: {
+            version: incVersion!,
+            history: {
+              timestampField: '@updatedTimestampField',
+            },
+          },
+        });
+
+        const {
+          definitions: [updatedDefinition],
+        } = await getInstalledDefinitions(supertest);
+        expect(updatedDefinition.version).to.eql(incVersion);
+        expect(updatedDefinition.history.timestampField).to.eql('@updatedTimestampField');
+
+        await uninstallDefinition(supertest, { id: mockDefinition.id });
+      });
+
+      it('rejects updates to managed definitions', async () => {
+        await installDefinition(supertest, {
+          definition: { ...mockDefinition, managed: true },
+          installOnly: true,
+        });
+
+        await updateDefinition(supertest, {
+          id: mockDefinition.id,
+          update: {
+            version: '1.0.0',
+            history: {
+              timestampField: '@updatedTimestampField',
+            },
+          },
+          expectedCode: 403,
+        });
+
+        await uninstallDefinition(supertest, { id: mockDefinition.id });
+      });
+    });
+
     describe('entity data', () => {
       let dataForgeConfig: PartialConfig;
       let dataForgeIndices: string[];
@@ -95,12 +156,12 @@ export default function ({ getService }: FtrProviderContext) {
 
       after(async () => {
         await esDeleteAllIndices(dataForgeIndices);
-        await uninstallDefinition(supertest, mockDefinition.id, true);
+        await uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true });
         await cleanup({ client: esClient, config: dataForgeConfig, logger });
       });
 
       it('should create the proper entities in the latest index', async () => {
-        await installDefinition(supertest, mockDefinition);
+        await installDefinition(supertest, { definition: mockDefinition });
         const sample = await waitForDocumentInIndex({
           esClient,
           indexName: generateLatestIndexName(mockDefinition),
@@ -108,6 +169,7 @@ export default function ({ getService }: FtrProviderContext) {
           retryService,
           logger,
         });
+
         const parsedSample = entityLatestSchema.safeParse(sample.hits.hits[0]._source);
         expect(parsedSample.success).to.be(true);
       });

x-pack/test/api_integration/apis/entity_manager/helpers/request.ts

@@ -6,7 +6,7 @@
  */
 
 import { Agent } from 'supertest';
-import { EntityDefinition } from '@kbn/entities-schema';
+import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
 
 export interface Auth {
@@ -16,9 +16,12 @@ export interface Auth {
 
 export const getInstalledDefinitions = async (
   supertest: Agent,
-  auth?: Auth
+  params: { auth?: Auth; id?: string } = {}
 ): Promise<{ definitions: EntityDefinitionWithState[] }> => {
-  let req = supertest.get('/internal/entities/definition').set('kbn-xsrf', 'xxx');
+  const { auth, id } = params;
+  let req = supertest
+    .get(`/internal/entities/definition${id ? `/${id}` : ''}`)
+    .set('kbn-xsrf', 'xxx');
   if (auth) {
     req = req.auth(auth.username, auth.password);
   }
@@ -28,18 +31,28 @@ export const getInstalledDefinitions = async (
 
 export const installDefinition = async (
   supertest: Agent,
-  definition: EntityDefinition,
-  query: Record<string, any> = {}
+  params: {
+    definition: EntityDefinition;
+    installOnly?: boolean;
+  }
 ) => {
+  const { definition, installOnly = false } = params;
   return supertest
     .post('/internal/entities/definition')
-    .query(query)
+    .query({ installOnly })
     .set('kbn-xsrf', 'xxx')
     .send(definition)
     .expect(200);
 };
 
-export const uninstallDefinition = (supertest: Agent, id: string, deleteData = false) => {
+export const uninstallDefinition = (
+  supertest: Agent,
+  params: {
+    id: string;
+    deleteData?: boolean;
+  }
+) => {
+  const { id, deleteData = false } = params;
   return supertest
     .delete(`/internal/entities/definition/${id}`)
     .query({ deleteData })
@@ -48,6 +61,22 @@ export const uninstallDefinition = (supertest: Agent, id: string, deleteData = f
     .expect(200);
 };
 
+export const updateDefinition = (
+  supertest: Agent,
+  params: {
+    id: string;
+    update: EntityDefinitionUpdate;
+    expectedCode?: number;
+  }
+) => {
+  const { id, update, expectedCode = 200 } = params;
+  return supertest
+    .patch(`/internal/entities/definition/${id}`)
+    .set('kbn-xsrf', 'xxx')
+    .send(update)
+    .expect(expectedCode);
+};
+
 export const upgradeBuiltinDefinitions = async (
   supertest: Agent,
   definitions: EntityDefinition[]