[EEM] Use top_metrics for identity fields lifting (#188315)

This PR converts the identity fields in the latest transform from a
`terms` aggregation to a `top_metric` aggregation in an effort to
simplify the implementation since we have to convert a terms aggregation
from keys to an array of strings. With the `top_metrics` implementation,
we just need to use the `dot_expander` processors along with a `set`
processor to get a single (non-array) field.

---------

Co-authored-by: Chris Cowan <chris@elastic.co>

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap

@@ -73,21 +73,27 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) {
     },
   },
   Object {
-    "script": Object {
+    "dot_expander": Object {
-      "if": "ctx.entity.identity.log?.logger != null && ctx.entity.identity.log.logger.size() != 0",
+      "field": "log.logger",
-      "source": "if (ctx.log == null) {
+      "path": "entity.identity.log.logger.top_metric",
-  ctx.log = new HashMap();
-}
-ctx.log.logger = ctx.entity.identity.log.logger.keySet().toArray()[0];",
     },
   },
   Object {
-    "script": Object {
+    "set": Object {
-      "if": "ctx.entity.identity.event?.category != null && ctx.entity.identity.event.category.size() != 0",
+      "field": "log.logger",
-      "source": "if (ctx.event == null) {
+      "value": "{{entity.identity.log.logger.top_metric.log.logger}}",
-  ctx.event = new HashMap();
+    },
-}
+  },
-ctx.event.category = ctx.entity.identity.event.category.keySet().toArray()[0];",
+  Object {
+    "dot_expander": Object {
+      "field": "event.category",
+      "path": "entity.identity.event.category.top_metric",
+    },
+  },
+  Object {
+    "set": Object {
+      "field": "event.category",
+      "value": "{{entity.identity.event.category.top_metric.event.category}}",
     },
   },
   Object {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts

@@ -38,16 +38,30 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
 }
 
 function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
-  return definition.identityFields.map((identityField) => {
+  return definition.identityFields
-    const optionalFieldPath = identityField.field.replaceAll('.', '?.');
+    .map((identityField) => {
-    const assignValue = `ctx.${identityField.field} = ctx.entity.identity.${identityField.field}.keySet().toArray()[0];`;
+      const setProcessor = {
-    return {
+        set: {
-      script: {
+          field: identityField.field,
-        if: `ctx.entity.identity.${optionalFieldPath} != null && ctx.entity.identity.${identityField.field}.size() != 0`,
+          value: `{{entity.identity.${identityField.field}.top_metric.${identityField.field}}}`,
-        source: cleanScript(`${initializePathScript(identityField.field)}\n${assignValue}`),
         },
       };
-  });
+
+      if (!identityField.field.includes('.')) {
+        return [setProcessor];
+      }
+
+      return [
+        {
+          dot_expander: {
+            field: identityField.field,
+            path: `entity.identity.${identityField.field}.top_metric`,
+          },
+        },
+        setProcessor,
+      ];
+    })
+    .flat();
 }
 
 export function generateLatestProcessors(definition: EntityDefinition) {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap

@@ -48,15 +48,37 @@ Object {
         },
       },
       "entity.identity.event.category": Object {
-        "terms": Object {
+        "aggs": Object {
+          "top_metric": Object {
+            "top_metrics": Object {
+              "metrics": Object {
                 "field": "event.category",
-          "size": 1,
+              },
+              "sort": "_score",
+            },
+          },
+        },
+        "filter": Object {
+          "exists": Object {
+            "field": "event.category",
+          },
         },
       },
       "entity.identity.log.logger": Object {
-        "terms": Object {
+        "aggs": Object {
+          "top_metric": Object {
+            "top_metrics": Object {
+              "metrics": Object {
                 "field": "log.logger",
-          "size": 1,
+              },
+              "sort": "_score",
+            },
+          },
+        },
+        "filter": Object {
+          "exists": Object {
+            "field": "log.logger",
+          },
         },
       },
       "entity.lastSeenTimestamp": Object {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_identity_aggregations.ts

@@ -12,9 +12,20 @@ export function generateIdentityAggregations(definition: EntityDefinition) {
     (aggs, identityField) => ({
       ...aggs,
       [`entity.identity.${identityField.field}`]: {
-        terms: {
+        filter: {
+          exists: {
             field: identityField.field,
-          size: 1,
+          },
+        },
+        aggs: {
+          top_metric: {
+            top_metrics: {
+              metrics: {
+                field: identityField.field,
+              },
+              sort: '_score',
+            },
+          },
         },
       },
     }),