[EEM] Add versioning for entity definitions (#187692)

This PR adds a `version` field to the `EntityDefinition` type, making it required in the API calls. It must be a SemVer string. The version is added to the ingest pipelines and transforms as part of their metadata. The version is included in the output documents alongside the schema version. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-23 17:28:26 -04:00 · 2024-07-09 16:31:35 +02:00 · 2024-07-09 16:31:35 +02:00 · 47178a776f
commit 47178a776f
parent b2e82f631e
19 changed files with 111 additions and 2 deletions
--- a/packages/kbn-check-mappings-update-cli/current_fields.json
+++ b/packages/kbn-check-mappings-update-cli/current_fields.json
@ -301,7 +301,8 @@
    "metrics",
    "name",
    "staticFields",
-    "type"
+    "type",
+    "version"
  ],
  "entity-discovery-api-key": [
    "apiKey"
--- a/packages/kbn-check-mappings-update-cli/current_mappings.json
+++ b/packages/kbn-check-mappings-update-cli/current_mappings.json
@ -1028,6 +1028,9 @@
      },
      "type": {
        "type": "keyword"
+      },
+      "version": {
+        "type": "keyword"
      }
    }
  },
--- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
+++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@ -88,7 +88,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
        "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
        "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
        "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "entity-definition": "33fe0194bd896f0bfe479d55f6de20f8ba1d7713",
+        "entity-definition": "331a2ba0ee9f24936ef049683549c8af7e46f03a",
        "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
        "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
        "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
--- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts
+++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts
@ -96,3 +96,9 @@ export const identityFieldsSchema = z
    optional: z.boolean(),
  })
  .or(z.string().transform((value) => ({ field: value, optional: false })));
+
+const semVerRegex = new RegExp(/^[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,}$/);
+export const semVerSchema = z.string().refine((maybeSemVer) => semVerRegex.test(maybeSemVer), {
+  message:
+    'The string does use the Semantic Versioning (Semver) format of {major}.{minor}.{patch} (e.g., 1.0.0), ensure each part contains only digits.',
+});
--- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
+++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
@ -13,10 +13,12 @@ import {
  filterSchema,
  durationSchema,
  identityFieldsSchema,
+  semVerSchema,
 } from './common';

 export const entityDefinitionSchema = z.object({
  id: z.string().regex(/^[\w-]+$/),
+  version: semVerSchema,
  name: z.string(),
  description: z.optional(z.string()),
  type: z.string(),
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts
@ -10,6 +10,7 @@ import { BUILT_IN_ID_PREFIX } from './constants';

 export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
  id: `${BUILT_IN_ID_PREFIX}services`,
+  version: '0.1.0',
  name: 'Services from logs',
  type: 'service',
  managed: true,
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
@ -28,6 +28,9 @@ export async function createAndInstallHistoryIngestPipeline(
        esClient.ingest.putPipeline({
          id: historyId,
          processors: historyProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
        }),
      { logger }
    );
@ -51,6 +54,9 @@ export async function createAndInstallLatestIngestPipeline(
        esClient.ingest.putPipeline({
          id: latestId,
          processors: latestProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
        }),
      { logger }
    );
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts
@ -8,6 +8,7 @@
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const entityDefinition = entityDefinitionSchema.parse({
  id: 'admin-console-services',
+  version: '999.999.999',
  name: 'Services for Admin Console',
  type: 'service',
  indexPatterns: ['kbn-data-forge-fake_stack.*'],
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/snapshots/generate_history_processors.test.ts.snap
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/snapshots/generate_history_processors.test.ts.snap
@ -20,6 +20,18 @@ Array [
      "value": "admin-console-services",
    },
  },
+  Object {
+    "set": Object {
+      "field": "entity.definitionVersion",
+      "value": "999.999.999",
+    },
+  },
+  Object {
+    "set": Object {
+      "field": "entity.schemaVersion",
+      "value": "v1",
+    },
+  },
  Object {
    "set": Object {
      "field": "entity.displayName",
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/snapshots/generate_latest_processors.test.ts.snap
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/snapshots/generate_latest_processors.test.ts.snap
@ -20,6 +20,18 @@ Array [
      "value": "admin-console-services",
    },
  },
+  Object {
+    "set": Object {
+      "field": "entity.definitionVersion",
+      "value": "999.999.999",
+    },
+  },
+  Object {
+    "set": Object {
+      "field": "entity.schemaVersion",
+      "value": "v1",
+    },
+  },
  Object {
    "script": Object {
      "source": "if (ctx.entity?.metadata?.tags.data != null) {
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
@ -6,6 +6,7 @@
 */

 import { EntityDefinition } from '@kbn/entities-schema';
+import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
 import { generateHistoryIndexName } from '../helpers/generate_component_id';

 function createIdTemplate(definition: EntityDefinition) {
@ -62,6 +63,18 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
        value: definition.id,
      },
    },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: ENTITY_SCHEMA_VERSION_V1,
+      },
+    },
    {
      set: {
        field: 'entity.displayName',
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
@ -6,6 +6,7 @@
 */

 import { EntityDefinition } from '@kbn/entities-schema';
+import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
 import { generateLatestIndexName } from '../helpers/generate_component_id';

 function mapDestinationToPainless(destination: string) {
@ -56,6 +57,18 @@ export function generateLatestProcessors(definition: EntityDefinition) {
        value: definition.id,
      },
    },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: ENTITY_SCHEMA_VERSION_V1,
+      },
+    },
    ...(definition.staticFields != null
      ? Object.keys(definition.staticFields).map((field) => ({
          set: { field, value: definition.staticFields![field] },
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts
@ -38,10 +38,16 @@ const assertHasCreatedDefinition = (
  expect(esClient.ingest.putPipeline).toBeCalledWith({
    id: generateHistoryIngestPipelineId(builtInServicesEntityDefinition),
    processors: expect.anything(),
+    _meta: {
+      definitionVersion: '0.1.0',
+    },
  });
  expect(esClient.ingest.putPipeline).toBeCalledWith({
    id: generateLatestIngestPipelineId(builtInServicesEntityDefinition),
    processors: expect.anything(),
+    _meta: {
+      definitionVersion: '0.1.0',
+    },
  });

  expect(esClient.transform.putTransform).toBeCalledTimes(2);
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/snapshots/generate_history_transform.test.ts.snap
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/snapshots/generate_history_transform.test.ts.snap
@ -2,6 +2,9 @@

 exports[`generateHistoryTransform(definition) should generate a valid latest transform 1`] = `
 Object {
+  "_meta": Object {
+    "definitionVersion": "999.999.999",
+  },
  "defer_validation": true,
  "dest": Object {
    "index": ".entities.v1.history.noop",
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/snapshots/generate_latest_transform.test.ts.snap
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/snapshots/generate_latest_transform.test.ts.snap
@ -2,6 +2,9 @@

 exports[`generateLatestTransform(definition) should generate a valid latest transform 1`] = `
 Object {
+  "_meta": Object {
+    "definitionVersion": "999.999.999",
+  },
  "defer_validation": true,
  "dest": Object {
    "index": ".entities.v1.latest.noop",
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts
@ -34,6 +34,9 @@ export function generateHistoryTransform(

  return {
    transform_id: generateHistoryTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
    defer_validation: true,
    source: {
      index: definition.indexPatterns,
--- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
@ -25,6 +25,9 @@ export function generateLatestTransform(
 ): TransformPutTransformRequest {
  return {
    transform_id: generateLatestTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
    defer_validation: true,
    source: {
      index: `${generateHistoryIndexName(definition)}.*`,
--- a/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts
@ -18,6 +18,7 @@ export const entityDefinition: SavedObjectsType = {
    dynamic: false,
    properties: {
      id: { type: 'keyword' },
+      version: { type: 'keyword' },
      name: { type: 'text' },
      description: { type: 'text' },
      type: { type: 'keyword' },
@ -37,4 +38,16 @@ export const entityDefinition: SavedObjectsType = {
      return `EntityDefinition: [${savedObject.attributes.name}]`;
    },
  },
+  modelVersions: {
+    '1': {
+      changes: [
+        {
+          type: 'mappings_addition',
+          addedMappings: {
+            version: { type: 'keyword' },
+          },
+        },
+      ],
+    },
+  },
 };
--- a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts
+++ b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts
@ -41,6 +41,14 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
              ignore_above: 1024,
              type: 'keyword',
            },
+            definitionVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+            schemaVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
            lastSeenTimestamp: {
              type: 'date',
            },