[eem] _search accepts kql filters (#203089)

## Summary

`searchEntities` now accepts kql filters instead of esql and translates
that to dsl filters at the query level

x-pack/plugins/entity_manager/server/lib/v2/entity_client.ts

@@ -98,7 +98,7 @@ export class EntityClient {
           );
         }
 
-        const query = getEntityInstancesQuery({
+        const { query, filter } = getEntityInstancesQuery({
           source: {
             ...source,
             metadata_fields: availableMetadataFields,
@@ -109,10 +109,13 @@ export class EntityClient {
           sort,
           limit,
         });
-        this.options.logger.debug(`Entity query: ${query}`);
+        this.options.logger.debug(
+          () => `Entity query: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}`
+        );
 
         const rawEntities = await runESQLQuery<EntityV2>('resolve entities', {
           query,
+          filter,
           esClient: this.options.clusterClient.asCurrentUser,
           logger: this.options.logger,
         });

x-pack/plugins/entity_manager/server/lib/v2/queries/index.test.ts

@@ -10,7 +10,7 @@ import { getEntityInstancesQuery } from '.';
 describe('getEntityInstancesQuery', () => {
   describe('getEntityInstancesQuery', () => {
     it('generates a valid esql query', () => {
-      const query = getEntityInstancesQuery({
+      const { query, filter } = getEntityInstancesQuery({
         source: {
           id: 'service_source',
           type_id: 'service',
@@ -29,14 +29,65 @@ describe('getEntityInstancesQuery', () => {
 
       expect(query).toEqual(
         'FROM logs-*, metrics-* | ' +
-          'WHERE service.name::keyword IS NOT NULL | ' +
-          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z" AND custom_timestamp_field <= "2024-11-20T20:00:00.000Z" | ' +
           'STATS host.name = VALUES(host.name::keyword), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' +
           'RENAME `service.name::keyword` AS service.name | ' +
           'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' +
           'SORT entity.id DESC | ' +
           'LIMIT 5'
       );
+
+      expect(filter).toEqual({
+        bool: {
+          filter: [
+            {
+              bool: {
+                should: [
+                  {
+                    exists: {
+                      field: 'service.name',
+                    },
+                  },
+                ],
+                minimum_should_match: 1,
+              },
+            },
+            {
+              bool: {
+                filter: [
+                  {
+                    bool: {
+                      should: [
+                        {
+                          range: {
+                            custom_timestamp_field: {
+                              gte: '2024-11-20T19:00:00.000Z',
+                            },
+                          },
+                        },
+                      ],
+                      minimum_should_match: 1,
+                    },
+                  },
+                  {
+                    bool: {
+                      should: [
+                        {
+                          range: {
+                            custom_timestamp_field: {
+                              lte: '2024-11-20T20:00:00.000Z',
+                            },
+                          },
+                        },
+                      ],
+                      minimum_should_match: 1,
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      });
     });
   });
 });

x-pack/plugins/entity_manager/server/lib/v2/queries/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 import { asKeyword } from './utils';
 import { EntitySourceDefinition, SortBy } from '../types';
 
@@ -21,7 +22,7 @@ const sourceCommand = ({ source }: { source: EntitySourceDefinition }) => {
   return query;
 };
 
-const whereCommand = ({
+const dslFilter = ({
   source,
   start,
   end,
@@ -30,10 +31,7 @@ const whereCommand = ({
   start: string;
   end: string;
 }) => {
-  const filters = [
-    source.identity_fields.map((field) => `${asKeyword(field)} IS NOT NULL`).join(' AND '),
-    ...source.filters,
-  ];
+  const filters = [...source.filters, ...source.identity_fields.map((field) => `${field}: *`)];
 
   if (source.timestamp_field) {
     filters.push(
@@ -41,7 +39,8 @@ const whereCommand = ({
     );
   }
 
-  return filters.map((filter) => `WHERE ${filter}`).join(' | ');
+  const kuery = filters.map((filter) => '(' + filter + ')').join(' AND ');
+  return toElasticsearchQuery(fromKueryExpression(kuery));
 };
 
 const statsCommand = ({ source }: { source: EntitySourceDefinition }) => {
@@ -108,16 +107,16 @@ export function getEntityInstancesQuery({
   start: string;
   end: string;
   sort?: SortBy;
-}): string {
+}) {
   const commands = [
     sourceCommand({ source }),
-    whereCommand({ source, start, end }),
     statsCommand({ source }),
     renameCommand({ source }),
     evalCommand({ source }),
     sortCommand({ source, sort }),
     `LIMIT ${limit}`,
   ];
+  const filter = dslFilter({ source, start, end });
 
-  return commands.join(' | ');
+  return { query: commands.join(' | '), filter };
 }

x-pack/plugins/entity_manager/server/lib/v2/run_esql_query.ts

@@ -7,6 +7,7 @@
 
 import { withSpan } from '@kbn/apm-utils';
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import { ESQLColumn, ESQLRow, ESQLSearchResponse } from '@kbn/es-types';
 
 export interface SourceAs<T> {
@@ -19,19 +20,24 @@ export async function runESQLQuery<T>(
     esClient,
     logger,
     query,
+    filter,
   }: {
     esClient: ElasticsearchClient;
     logger: Logger;
     query: string;
+    filter?: QueryDslQueryContainer;
   }
 ): Promise<T[]> {
-  logger.trace(() => `Request (${operationName}):\n${query}`);
+  logger.trace(
+    () => `Request (${operationName}):\nquery: ${query}\nfilter: ${JSON.stringify(filter, null, 2)}`
+  );
   return withSpan(
     { name: operationName, labels: { plugin: '@kbn/entityManager-plugin' } },
     async () =>
       esClient.esql.query(
         {
           query,
+          filter,
           format: 'json',
         },
         { querystring: { drop_null_columns: true } }
@@ -62,8 +68,7 @@ function rowToObject(row: ESQLRow, columns: ESQLColumn[]) {
       return object;
     }
 
-    // Removes the type suffix from the column name
-    const name = column.name.replace(/\.(text|keyword)$/, '');
+    const name = column.name;
     if (!object[name]) {
       object[name] = value;
     }

x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx

@@ -70,7 +70,7 @@ function EntitySourceForm({
       </EuiFlexItem>
 
       <EuiFlexItem>
-        <EuiFormRow label="Filters (comma-separated ESQL filters)">
+        <EuiFormRow label="Filters (comma-separated KQL filters)">
           <EuiFieldText
             data-test-subj="entityManagerFormFilters"
             name="filters"