[eem] _search sort_by and display_name (#202361)

- allow optional `entity_source.display_name` setting.
`entity.display_name` will always be set on the entities, falling back
to `entity.id` if provided field is not set
- allow `sort_by` parameter to `_search` API
- removed the `metadata.` prefix in the query aggregation. metadata will
now be set at the root of the document (eg for metadata `host.name`
entity = `{ entity.id: 'foo', host.name: 'bar' }`
- timestamp_field is now optional

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -25,8 +25,9 @@ export interface MetadataRecord {
 
 export interface EntityV2 {
   'entity.id': string;
-  'entity.last_seen_timestamp': string;
   'entity.type': string;
+  'entity.display_name': string;
+  'entity.last_seen_timestamp'?: string;
   [metadata: string]: any;
 }
 

x-pack/plugins/entity_manager/server/lib/entity_client.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { without } from 'lodash';
 import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
@@ -23,10 +24,27 @@ import { stopTransforms } from './entities/stop_transforms';
 import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
-import { EntitySource, getEntityInstancesQuery } from './queries';
+import { EntitySource, SortBy, getEntityInstancesQuery } from './queries';
 import { mergeEntitiesList, runESQLQuery } from './queries/utils';
 import { UnknownEntityType } from './entities/errors/unknown_entity_type';
 
+interface SearchCommon {
+  start: string;
+  end: string;
+  sort?: SortBy;
+  metadataFields?: string[];
+  filters?: string[];
+  limit?: number;
+}
+
+export type SearchByType = SearchCommon & {
+  type: string;
+};
+
+export type SearchBySources = SearchCommon & {
+  sources: EntitySource[];
+};
+
 export class EntityClient {
   constructor(
     private options: {
@@ -191,17 +209,11 @@ export class EntityClient {
     type,
     start,
     end,
+    sort,
     metadataFields = [],
     filters = [],
     limit = 10,
-  }: {
-    type: string;
-    start: string;
-    end: string;
-    metadataFields?: string[];
-    filters?: string[];
-    limit?: number;
-  }) {
+  }: SearchByType) {
     const sources = await this.getEntitySources({ type });
     if (sources.length === 0) {
       throw new UnknownEntityType(`No sources found for entity type [${type}]`);
@@ -213,6 +225,7 @@ export class EntityClient {
       end,
       metadataFields,
       filters,
+      sort,
       limit,
     });
   }
@@ -221,21 +234,22 @@ export class EntityClient {
     sources,
     start,
     end,
+    sort,
     metadataFields = [],
     filters = [],
     limit = 10,
-  }: {
-    sources: EntitySource[];
-    start: string;
-    end: string;
-    metadataFields?: string[];
-    filters?: string[];
-    limit?: number;
-  }) {
+  }: SearchBySources) {
     const entities = await Promise.all(
       sources.map(async (source) => {
-        const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
+        const mandatoryFields = [
+          ...source.identity_fields,
+          ...(source.timestamp_field ? [source.timestamp_field] : []),
+          ...(source.display_name ? [source.display_name] : []),
+        ];
         const metaFields = [...metadataFields, ...source.metadata_fields];
+
+        // operations on an unmapped field result in a failing query so we verify
+        // field capabilities beforehand
         const { fields } = await this.options.esClient.fieldCaps({
           index: source.index_patterns,
           fields: [...mandatoryFields, ...metaFields],
@@ -244,15 +258,25 @@ export class EntityClient {
         const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
         if (!sourceHasMandatoryFields) {
           // we can't build entities without id fields so we ignore the source.
-          // filters should likely behave similarly.
+          // TODO filters should likely behave similarly. we should also throw
+          const missingFields = mandatoryFields.filter((field) => !fields[field]);
           this.options.logger.info(
-            `Ignoring source for type [${source.type}] with index_patterns [${source.index_patterns}] because some mandatory fields [${mandatoryFields}] are not mapped`
+            `Ignoring source for type [${source.type}] with index_patterns [${
+              source.index_patterns
+            }] because some mandatory fields [${missingFields.join(', ')}] are not mapped`
           );
           return [];
         }
 
         // but metadata field not being available is fine
         const availableMetadataFields = metaFields.filter((field) => fields[field]);
+        if (availableMetadataFields.length < metaFields.length) {
+          this.options.logger.info(
+            `Ignoring unmapped fields [${without(metaFields, ...availableMetadataFields).join(
+              ', '
+            )}]`
+          );
+        }
 
         const query = getEntityInstancesQuery({
           source: {
@@ -262,6 +286,7 @@ export class EntityClient {
           },
           start,
           end,
+          sort,
           limit,
         });
         this.options.logger.debug(`Entity query: ${query}`);
@@ -271,14 +296,10 @@ export class EntityClient {
           esClient: this.options.esClient,
         });
 
-        return rawEntities.map((entity) => {
-          entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
-          entity['entity.type'] = source.type;
-          return entity;
-        });
+        return rawEntities;
       })
     ).then((results) => results.flat());
 
-    return mergeEntitiesList(entities).slice(0, limit);
+    return mergeEntitiesList(sources, entities).slice(0, limit);
   }
 }

x-pack/plugins/entity_manager/server/lib/queries/index.test.ts

@@ -18,19 +18,21 @@ describe('getEntityInstancesQuery', () => {
           metadata_fields: ['host.name'],
           filters: [],
           timestamp_field: 'custom_timestamp_field',
+          display_name: 'service.id',
         },
         limit: 5,
         start: '2024-11-20T19:00:00.000Z',
         end: '2024-11-20T20:00:00.000Z',
+        sort: { field: 'entity.id', direction: 'DESC' },
       });
 
       expect(query).toEqual(
-        'FROM logs-*,metrics-*|' +
-          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z"|' +
-          'WHERE custom_timestamp_field <= "2024-11-20T20:00:00.000Z"|' +
-          'WHERE service.name IS NOT NULL|' +
-          'STATS entity.last_seen_timestamp=MAX(custom_timestamp_field),metadata.host.name=VALUES(host.name) BY service.name|' +
-          'SORT entity.last_seen_timestamp DESC|' +
+        'FROM logs-*, metrics-* | ' +
+          'WHERE service.name IS NOT NULL | ' +
+          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z" AND custom_timestamp_field <= "2024-11-20T20:00:00.000Z" | ' +
+          'STATS host.name = VALUES(host.name), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id) BY service.name | ' +
+          'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' +
+          'SORT entity.id DESC | ' +
           'LIMIT 5'
       );
     });

x-pack/plugins/entity_manager/server/lib/queries/index.ts

@@ -9,29 +9,35 @@ import { z } from '@kbn/zod';
 
 export const entitySourceSchema = z.object({
   type: z.string(),
-  timestamp_field: z.optional(z.string()).default('@timestamp'),
+  timestamp_field: z.optional(z.string()),
   index_patterns: z.array(z.string()),
   identity_fields: z.array(z.string()),
   metadata_fields: z.array(z.string()),
   filters: z.array(z.string()),
+  display_name: z.optional(z.string()),
 });
 
+export interface SortBy {
+  field: string;
+  direction: 'ASC' | 'DESC';
+}
+
 export type EntitySource = z.infer<typeof entitySourceSchema>;
 
 const sourceCommand = ({ source }: { source: EntitySource }) => {
-  let query = `FROM ${source.index_patterns}`;
+  let query = `FROM ${source.index_patterns.join(', ')}`;
 
   const esMetadataFields = source.metadata_fields.filter((field) =>
     ['_index', '_id'].includes(field)
   );
   if (esMetadataFields.length) {
-    query += ` METADATA ${esMetadataFields.join(',')}`;
+    query += ` METADATA ${esMetadataFields.join(', ')}`;
   }
 
   return query;
 };
 
-const filterCommands = ({
+const whereCommand = ({
   source,
   start,
   end,
@@ -40,32 +46,65 @@ const filterCommands = ({
   start: string;
   end: string;
 }) => {
-  const commands = [
-    `WHERE ${source.timestamp_field} >= "${start}"`,
-    `WHERE ${source.timestamp_field} <= "${end}"`,
+  const filters = [
+    source.identity_fields.map((field) => `${field} IS NOT NULL`).join(' AND '),
+    ...source.filters,
   ];
 
-  source.identity_fields.forEach((field) => {
-    commands.push(`WHERE ${field} IS NOT NULL`);
-  });
+  if (source.timestamp_field) {
+    filters.push(
+      `${source.timestamp_field} >= "${start}" AND ${source.timestamp_field} <= "${end}"`
+    );
+  }
 
-  source.filters.forEach((filter) => {
-    commands.push(`WHERE ${filter}`);
-  });
-
-  return commands;
+  return filters.map((filter) => `WHERE ${filter}`).join(' | ');
 };
 
 const statsCommand = ({ source }: { source: EntitySource }) => {
-  const aggs = [
-    // default 'last_seen' attribute
-    `entity.last_seen_timestamp=MAX(${source.timestamp_field})`,
-    ...source.metadata_fields
-      .filter((field) => !source.identity_fields.some((idField) => idField === field))
-      .map((field) => `metadata.${field}=VALUES(${field})`),
-  ];
+  const aggs = source.metadata_fields
+    .filter((field) => !source.identity_fields.some((idField) => idField === field))
+    .map((field) => `${field} = VALUES(${field})`);
 
-  return `STATS ${aggs.join(',')} BY ${source.identity_fields.join(',')}`;
+  if (source.timestamp_field) {
+    aggs.push(`entity.last_seen_timestamp = MAX(${source.timestamp_field})`);
+  }
+
+  if (source.display_name) {
+    // ideally we want the latest value but there's no command yet
+    // so we use MAX for now
+    aggs.push(`${source.display_name} = MAX(${source.display_name})`);
+  }
+
+  return `STATS ${aggs.join(', ')} BY ${source.identity_fields.join(', ')}`;
+};
+
+const evalCommand = ({ source }: { source: EntitySource }) => {
+  const id =
+    source.identity_fields.length === 1
+      ? source.identity_fields[0]
+      : `CONCAT(${source.identity_fields.join(', ":", ')})`;
+
+  const displayName = source.display_name
+    ? `COALESCE(${source.display_name}, entity.id)`
+    : 'entity.id';
+
+  return `EVAL ${[
+    `entity.type = "${source.type}"`,
+    `entity.id = ${id}`,
+    `entity.display_name = ${displayName}`,
+  ].join(', ')}`;
+};
+
+const sortCommand = ({ source, sort }: { source: EntitySource; sort?: SortBy }) => {
+  if (sort) {
+    return `SORT ${sort.field} ${sort.direction}`;
+  }
+
+  if (source.timestamp_field) {
+    return `SORT entity.last_seen_timestamp DESC`;
+  }
+
+  return `SORT entity.id ASC`;
 };
 
 export function getEntityInstancesQuery({
@@ -73,19 +112,22 @@ export function getEntityInstancesQuery({
   limit,
   start,
   end,
+  sort,
 }: {
   source: EntitySource;
   limit: number;
   start: string;
   end: string;
+  sort?: SortBy;
 }): string {
   const commands = [
     sourceCommand({ source }),
-    ...filterCommands({ source, start, end }),
+    whereCommand({ source, start, end }),
     statsCommand({ source }),
-    `SORT entity.last_seen_timestamp DESC`,
+    evalCommand({ source }),
+    sortCommand({ source, sort }),
     `LIMIT ${limit}`,
   ];
 
-  return commands.join('|');
+  return commands.join(' | ');
 }

x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { EntitySource } from '.';
 import { mergeEntitiesList } from './utils';
 
 describe('mergeEntitiesList', () => {
@@ -15,20 +16,23 @@ describe('mergeEntitiesList', () => {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
+          'entity.display_name': 'foo',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
+          'entity.display_name': 'foo',
         },
       ];
 
-      const mergedEntities = mergeEntitiesList(entities);
+      const mergedEntities = mergeEntitiesList([], entities);
       expect(mergedEntities.length).toEqual(1);
       expect(mergedEntities[0]).toEqual({
         'entity.id': 'foo',
         'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
         'entity.type': 'service',
+        'entity.display_name': 'foo',
       });
     });
 
@@ -38,33 +42,46 @@ describe('mergeEntitiesList', () => {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-1',
-          'metadata.agent.name': 'agent-1',
-          'metadata.service.environment': ['dev', 'staging'],
-          'metadata.only_in_record_1': 'foo',
+          'entity.display_name': 'foo',
+          'host.name': 'host-1',
+          'agent.name': 'agent-1',
+          'service.environment': ['dev', 'staging'],
+          only_in_record_1: 'foo',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': ['host-2', 'host-3'],
-          'metadata.agent.name': 'agent-2',
-          'metadata.service.environment': 'prod',
-          'metadata.only_in_record_2': 'bar',
+          'entity.display_name': 'foo',
+          'host.name': ['host-2', 'host-3'],
+          'agent.name': 'agent-2',
+          'service.environment': 'prod',
+          only_in_record_2: 'bar',
         },
       ];
 
-      const mergedEntities = mergeEntitiesList(entities);
+      const mergedEntities = mergeEntitiesList(
+        [
+          {
+            metadata_fields: ['host.name', 'agent.name', 'service.environment', 'only_in_record_1'],
+          },
+          {
+            metadata_fields: ['host.name', 'agent.name', 'service.environment', 'only_in_record_2'],
+          },
+        ] as EntitySource[],
+        entities
+      );
       expect(mergedEntities.length).toEqual(1);
       expect(mergedEntities[0]).toEqual({
         'entity.id': 'foo',
         'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
         'entity.type': 'service',
-        'metadata.host.name': ['host-1', 'host-2', 'host-3'],
-        'metadata.agent.name': ['agent-1', 'agent-2'],
-        'metadata.service.environment': ['dev', 'staging', 'prod'],
-        'metadata.only_in_record_1': 'foo',
-        'metadata.only_in_record_2': 'bar',
+        'entity.display_name': 'foo',
+        'host.name': ['host-1', 'host-2', 'host-3'],
+        'agent.name': ['agent-1', 'agent-2'],
+        'service.environment': ['dev', 'staging', 'prod'],
+        only_in_record_1: 'foo',
+        only_in_record_2: 'bar',
       });
     });
 
@@ -74,29 +91,78 @@ describe('mergeEntitiesList', () => {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-1',
+          'entity.display_name': 'foo',
+          'host.name': 'host-1',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-2',
+          'entity.display_name': 'foo',
+          'host.name': 'host-2',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-3',
+          'entity.display_name': 'foo',
+          'host.name': 'host-3',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'host.name': 'host-3',
         },
       ];
 
-      const mergedEntities = mergeEntitiesList(entities);
+      const mergedEntities = mergeEntitiesList(
+        [
+          {
+            metadata_fields: ['host.name'],
+          },
+          {
+            metadata_fields: ['host.name'],
+          },
+        ] as EntitySource[],
+        entities
+      );
       expect(mergedEntities.length).toEqual(1);
       expect(mergedEntities[0]).toEqual({
         'entity.id': 'foo',
         'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
         'entity.type': 'service',
-        'metadata.host.name': ['host-1', 'host-2', 'host-3'],
+        'entity.display_name': 'foo',
+        'host.name': ['host-1', 'host-2', 'host-3'],
+      });
+    });
+
+    it('works without entity.last_seen_timestamp', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'host.name': 'host-1',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'host.name': 'host-2',
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList(
+        [{ metadata_fields: ['host.name'] }, { metadata_fields: ['host.name'] }] as EntitySource[],
+        entities
+      );
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.type': 'service',
+        'entity.display_name': 'foo',
+        'host.name': ['host-1', 'host-2'],
       });
     });
 
@@ -106,29 +172,43 @@ describe('mergeEntitiesList', () => {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-1',
+          'entity.display_name': 'foo',
+          'host.name': 'host-1',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': 'host-2',
+          'entity.display_name': 'foo',
+          'host.name': 'host-2',
         },
         {
           'entity.id': 'foo',
           'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z',
           'entity.type': 'service',
-          'metadata.host.name': ['host-1', 'host-2'],
+          'entity.display_name': 'foo',
+          'host.name': ['host-1', 'host-2'],
         },
       ];
 
-      const mergedEntities = mergeEntitiesList(entities);
+      const mergedEntities = mergeEntitiesList(
+        [
+          {
+            metadata_fields: ['host.name'],
+          },
+          {
+            metadata_fields: ['host.name'],
+          },
+        ] as EntitySource[],
+        entities
+      );
       expect(mergedEntities.length).toEqual(1);
       expect(mergedEntities[0]).toEqual({
         'entity.id': 'foo',
         'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z',
         'entity.type': 'service',
-        'metadata.host.name': ['host-1', 'host-2'],
+        'entity.display_name': 'foo',
+        'host.name': ['host-1', 'host-2'],
       });
     });
   });

x-pack/plugins/entity_manager/server/lib/queries/utils.ts

@@ -5,24 +5,33 @@
  * 2.0.
  */
 
+import { compact, uniq } from 'lodash';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { EntityV2 } from '@kbn/entities-schema';
 import { ESQLSearchResponse } from '@kbn/es-types';
-import { uniq } from 'lodash';
+import { EntitySource } from '.';
 
-function mergeEntities(entity1: EntityV2, entity2: EntityV2): EntityV2 {
-  const merged: EntityV2 = {
-    ...entity1,
-    'entity.last_seen_timestamp': new Date(
-      Math.max(
-        Date.parse(entity1['entity.last_seen_timestamp']),
-        Date.parse(entity2['entity.last_seen_timestamp'])
-      )
-    ).toISOString(),
-  };
+function getLatestDate(date1?: string, date2?: string) {
+  if (!date1 && !date2) return;
+
+  return new Date(
+    Math.max(date1 ? Date.parse(date1) : 0, date2 ? Date.parse(date2) : 0)
+  ).toISOString();
+}
+
+function mergeEntities(metadataFields: string[], entity1: EntityV2, entity2: EntityV2): EntityV2 {
+  const merged: EntityV2 = { ...entity1 };
+
+  const latestTimestamp = getLatestDate(
+    entity1['entity.last_seen_timestamp'],
+    entity2['entity.last_seen_timestamp']
+  );
+  if (latestTimestamp) {
+    merged['entity.last_seen_timestamp'] = latestTimestamp;
+  }
 
   for (const [key, value] of Object.entries(entity2).filter(([_key]) =>
-    _key.startsWith('metadata.')
+    metadataFields.includes(_key)
   )) {
     if (merged[key]) {
       merged[key] = uniq([
@@ -36,7 +45,10 @@ function mergeEntities(entity1: EntityV2, entity2: EntityV2): EntityV2 {
   return merged;
 }
 
-export function mergeEntitiesList(entities: EntityV2[]): EntityV2[] {
+export function mergeEntitiesList(sources: EntitySource[], entities: EntityV2[]): EntityV2[] {
+  const metadataFields = uniq(
+    sources.flatMap((source) => compact([source.timestamp_field, ...source.metadata_fields]))
+  );
   const instances: { [key: string]: EntityV2 } = {};
 
   for (let i = 0; i < entities.length; i++) {
@@ -44,7 +56,7 @@ export function mergeEntitiesList(entities: EntityV2[]): EntityV2[] {
     const id = entity['entity.id'];
 
     if (instances[id]) {
-      instances[id] = mergeEntities(instances[id], entity);
+      instances[id] = mergeEntities(metadataFields, instances[id], entity);
     } else {
       instances[id] = entity;
     }

x-pack/plugins/entity_manager/server/routes/v2/search.ts

@@ -22,20 +22,34 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
         .optional(z.string())
         .default(() => moment().subtract(5, 'minutes').toISOString())
         .refine((val) => moment(val).isValid(), {
-          message: 'start should be a date in ISO format',
+          message: '[start] should be a date in ISO format',
         }),
       end: z
         .optional(z.string())
         .default(() => moment().toISOString())
         .refine((val) => moment(val).isValid(), {
-          message: 'start should be a date in ISO format',
+          message: '[end] should be a date in ISO format',
         }),
+      sort: z.optional(
+        z.object({
+          field: z.string(),
+          direction: z.enum(['ASC', 'DESC']),
+        })
+      ),
       limit: z.optional(z.number()).default(10),
     }),
   }),
   handler: async ({ request, response, params, logger, getScopedClient }) => {
     try {
-      const { type, start, end, limit, filters, metadata_fields: metadataFields } = params.body;
+      const {
+        type,
+        start,
+        end,
+        limit,
+        filters,
+        sort,
+        metadata_fields: metadataFields,
+      } = params.body;
 
       const client = await getScopedClient({ request });
       const entities = await client.searchEntities({
@@ -44,6 +58,7 @@ export const searchEntitiesRoute = createEntityManagerServerRoute({
         metadataFields,
         start,
         end,
+        sort,
         limit,
       });
 
@@ -69,25 +84,32 @@ export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({
         .optional(z.string())
         .default(() => moment().subtract(5, 'minutes').toISOString())
         .refine((val) => moment(val).isValid(), {
-          message: 'start should be a date in ISO format',
+          message: '[start] should be a date in ISO format',
         }),
       end: z
         .optional(z.string())
         .default(() => moment().toISOString())
         .refine((val) => moment(val).isValid(), {
-          message: 'start should be a date in ISO format',
+          message: '[end] should be a date in ISO format',
         }),
+      sort: z.optional(
+        z.object({
+          field: z.string(),
+          direction: z.enum(['ASC', 'DESC']),
+        })
+      ),
       limit: z.optional(z.number()).default(10),
     }),
   }),
-  handler: async ({ request, response, params, logger, getScopedClient }) => {
-    const { sources, start, end, limit } = params.body;
+  handler: async ({ request, response, params, getScopedClient }) => {
+    const { sources, start, end, limit, sort } = params.body;
 
     const client = await getScopedClient({ request });
     const entities = await client.searchEntitiesBySources({
       sources,
       start,
       end,
+      sort,
       limit,
     });
 

x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx

@@ -101,17 +101,29 @@ function EntitySourceForm({
           />
         </EuiFormRow>
       </EuiFlexItem>
+
+      <EuiFlexItem>
+        <EuiFormRow label="Display name">
+          <EuiFieldText
+            data-test-subj="entityManagerFormDisplayName"
+            name="display_name"
+            defaultValue={source.display_name}
+            onChange={(e) => onFieldChange(index, 'display_name', e.target.value)}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
     </EuiFlexGroup>
   );
 }
 
 interface EntitySource {
   id: string;
-  index_patterns?: string[];
-  identity_fields?: string[];
-  metadata_fields?: string[];
-  filters?: string[];
+  index_patterns: string[];
+  identity_fields: string[];
+  metadata_fields: string[];
+  filters: string[];
   timestamp_field?: string;
+  display_name?: string;
 }
 
 const newEntitySource = ({
@@ -126,7 +138,7 @@ const newEntitySource = ({
   metadataFields?: string[];
   filters?: string[];
   timestampField?: string;
-}) => ({
+}): EntitySource => ({
   id: uuid(),
   index_patterns: indexPatterns,
   identity_fields: identityFields,
@@ -320,6 +332,10 @@ export function EntityManagerOverviewPage() {
             field: 'entity.id',
             name: 'entity.id',
           },
+          {
+            field: 'entity.display_name',
+            name: 'entity.display_name',
+          },
           {
             field: 'entity.type',
             name: 'entity.type',
@@ -329,16 +345,10 @@ export function EntityManagerOverviewPage() {
             name: 'entity.last_seen_timestamp',
           },
           ...Array.from(new Set(entitySources.flatMap((source) => source.identity_fields))).map(
-            (field) => ({
-              field,
-              name: field,
-            })
+            (field) => ({ field, name: field })
           ),
           ...Array.from(new Set(entitySources.flatMap((source) => source.metadata_fields))).map(
-            (field) => ({
-              field: `metadata.${field}`,
-              name: `metadata.${field}`,
-            })
+            (field) => ({ field, name: field })
           ),
         ]}
       />