[EEM] Fallback to source for metadata if no destination defined (#188515)

This PR closes
https://github.com/elastic/elastic-entity-model/issues/116 by ensuring
that `destination` is always set when the schema is parsed along with
ensuring that if for some reason desitnation is not set, we fallback in
the actual metadata code as well. I also added a unit test for each of
the different `metadata` formats:
- String
- Object with only `source`
- Object with `source` and `limit`
- Object with `source`, `limit`, and `destination`

---------

Co-authored-by: Chris Cowan <chris@elastic.co>
Co-authored-by: Chris Cowan <chris@chriscowan.us>
Co-authored-by: Nathan L Smith <nathan.smith@elastic.co>

x-pack/packages/kbn-entities-schema/src/schema/common.ts

@@ -88,6 +88,11 @@ export const metadataSchema = z
     destination: z.optional(z.string()),
     limit: z.optional(z.number().default(1000)),
   })
+  .transform((metadata) => ({
+    ...metadata,
+    destination: metadata.destination ?? metadata.source,
+    limit: metadata.limit ?? 1000,
+  }))
   .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 })));
 
 export const identityFieldsSchema = z

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -6,7 +6,7 @@
  */
 
 import { entityDefinitionSchema } from '@kbn/entities-schema';
-export const entityDefinition = entityDefinitionSchema.parse({
+export const rawEntityDefinition = {
   id: 'admin-console-services',
   version: '999.999.999',
   name: 'Services for Admin Console',
@@ -43,4 +43,5 @@ export const entityDefinition = entityDefinitionSchema.parse({
       ],
     },
   ],
-});
+};
+export const entityDefinition = entityDefinitionSchema.parse(rawEntityDefinition);

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts

@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entityDefinitionSchema } from '@kbn/entities-schema';
+import { rawEntityDefinition } from '../helpers/fixtures/entity_definition';
+import {
+  generateHistoryMetadataAggregations,
+  generateLatestMetadataAggregations,
+} from './generate_metadata_aggregations';
+
+describe('Generate Metadata Aggregations for history and latest', () => {
+  describe('generateHistoryMetadataAggregations()', () => {
+    it('should generate metadata aggregations for string format', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: ['host.name'],
+      });
+      expect(generateHistoryMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          terms: {
+            field: 'host.name',
+            size: 1000,
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with only source', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name' }],
+      });
+      expect(generateHistoryMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          terms: {
+            field: 'host.name',
+            size: 1000,
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with source and limit', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name', limit: 10 }],
+      });
+      expect(generateHistoryMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          terms: {
+            field: 'host.name',
+            size: 10,
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with source, limit, and destination', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }],
+      });
+      expect(generateHistoryMetadataAggregations(definition)).toEqual({
+        'entity.metadata.hostName': {
+          terms: {
+            field: 'host.name',
+            size: 10,
+          },
+        },
+      });
+    });
+  });
+
+  describe('generateLatestMetadataAggregations()', () => {
+    it('should generate metadata aggregations for string format', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: ['host.name'],
+      });
+      expect(generateLatestMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-1m',
+              },
+            },
+          },
+          aggs: {
+            data: {
+              terms: {
+                field: 'host.name',
+                size: 1000,
+              },
+            },
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with only source', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name' }],
+      });
+      expect(generateLatestMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-1m',
+              },
+            },
+          },
+          aggs: {
+            data: {
+              terms: {
+                field: 'host.name',
+                size: 1000,
+              },
+            },
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with source and limit', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name', limit: 10 }],
+      });
+      expect(generateLatestMetadataAggregations(definition)).toEqual({
+        'entity.metadata.host.name': {
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-1m',
+              },
+            },
+          },
+          aggs: {
+            data: {
+              terms: {
+                field: 'host.name',
+                size: 10,
+              },
+            },
+          },
+        },
+      });
+    });
+
+    it('should generate metadata aggregations for object format with source, limit, and destination', () => {
+      const definition = entityDefinitionSchema.parse({
+        ...rawEntityDefinition,
+        metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }],
+      });
+      expect(generateLatestMetadataAggregations(definition)).toEqual({
+        'entity.metadata.hostName': {
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-1m',
+              },
+            },
+          },
+          aggs: {
+            data: {
+              terms: {
+                field: 'hostName',
+                size: 10,
+              },
+            },
+          },
+        },
+      });
+    });
+  });
+});

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts

@@ -34,7 +34,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition)
   return definition.metadata.reduce(
     (aggs, metadata) => ({
       ...aggs,
-      [`entity.metadata.${metadata.destination}`]: {
+      [`entity.metadata.${metadata.destination ?? metadata.source}`]: {
         filter: {
           range: {
             'event.ingested': {