Siem query rule - reduce field_caps usage (#184890)

## Summary

Previously, the siem query rule loaded the full set of fields for an
index pattern when running a query. This could load 5k fields or more.
Now it only loads the fields necessary for the query.

Changes as part of this PR
- The data plugin exports `queryToFields` which takes a query and
returns a list of the fields required to translate the query to ES DSL.
- `queryToFields` properly handles all filter types, previously expected
unified search bar provided filters.
- `createSecurityRuleTypeWrapper` has been modified to skip field
loading for the siem query rule
- `getFilter` takes an optional `loadFields` arguments which loads only
necessary fields
- `getQueryFilterLoadFields` was created - based on `getQueryFilter` but
also loads necessary fields

src/plugins/data/common/search/search_source/index.ts

@@ -15,3 +15,4 @@ export * from './fetch';
 export * from './search_source';
 export * from './search_source_service';
 export * from './types';
+export * from './query_to_fields';

src/plugins/data/common/search/search_source/query_to_fields.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { DataViewLazy } from '@kbn/data-views-plugin/common';
+import { fromKueryExpression, getKqlFieldNames } from '@kbn/es-query';
+import type { SearchRequest } from './fetch';
+import { EsQuerySortValue } from '../..';
+
+export async function queryToFields({
+  dataView,
+  sort,
+  request,
+}: {
+  dataView: DataViewLazy;
+  sort?: EsQuerySortValue | EsQuerySortValue[];
+  request: SearchRequest;
+}) {
+  let fields = dataView.timeFieldName ? [dataView.timeFieldName] : [];
+  if (sort) {
+    const sortArr = Array.isArray(sort) ? sort : [sort];
+    fields.push(...sortArr.flatMap((s) => Object.keys(s)));
+  }
+  for (const query of request.query) {
+    if (query.query) {
+      const nodes = fromKueryExpression(query.query);
+      const queryFields = getKqlFieldNames(nodes);
+      fields = fields.concat(queryFields);
+    }
+  }
+  const filters = request.filters;
+  if (filters) {
+    const filtersArr = Array.isArray(filters) ? filters : [filters];
+    for (const f of filtersArr) {
+      // unified search bar filters have meta object and key (regular filters)
+      // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is)
+      // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all)
+      if (f?.meta?.key && f.meta.disabled !== true) {
+        fields.push(f.meta.key);
+      }
+    }
+  }
+
+  // if source filtering is enabled, we need to fetch all the fields
+  const fieldName =
+    dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length ? ['*'] : fields;
+
+  if (fieldName.length) {
+    return (await dataView.getFields({ fieldName })).getFieldMapSorted();
+  }
+
+  // no fields needed to be loaded for query
+  return {};
+}

src/plugins/data/common/search/search_source/search_source.ts

@@ -77,11 +77,9 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import {
   buildEsQuery,
   Filter,
-  fromKueryExpression,
   isOfQueryType,
   isPhraseFilter,
   isPhrasesFilter,
-  getKqlFieldNames,
 } from '@kbn/es-query';
 import { fieldWildcardFilter } from '@kbn/kibana-utils-plugin/common';
 import { getHighlightRequest } from '@kbn/field-formats-plugin/common';
@@ -95,6 +93,7 @@ import type { ISearchGeneric, IKibanaSearchResponse, IEsSearchResponse } from '@
 import { normalizeSortRequest } from './normalize_sort_request';
 
 import { AggConfigSerialized, DataViewField, SerializedSearchSourceFields } from '../..';
+import { queryToFields } from './query_to_fields';
 
 import { AggConfigs, EsQuerySortValue } from '../..';
 import type {
@@ -778,43 +777,7 @@ export class SearchSource {
 
   public async loadDataViewFields(dataView: DataViewLazy) {
     const request = this.mergeProps(this, { body: {} });
-    let fields = dataView.timeFieldName ? [dataView.timeFieldName] : [];
-    const sort = this.getField('sort');
-    if (sort) {
-      const sortArr = Array.isArray(sort) ? sort : [sort];
-      for (const s of sortArr) {
-        const keys = Object.keys(s);
-        fields = fields.concat(keys);
-      }
-    }
-    for (const query of request.query) {
-      if (query.query) {
-        const nodes = fromKueryExpression(query.query);
-        const queryFields = getKqlFieldNames(nodes);
-        fields = fields.concat(queryFields);
-      }
-    }
-    const filters = request.filters;
-    if (filters) {
-      const filtersArr = Array.isArray(filters) ? filters : [filters];
-      for (const f of filtersArr) {
-        fields = fields.concat(f.meta.key);
-      }
-    }
-    fields = fields.filter((f) => Boolean(f));
-
-    if (dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length) {
-      // if source filtering is enabled, we need to fetch all the fields
-      return (await dataView.getFields({ fieldName: ['*'] })).getFieldMapSorted();
-    } else if (fields.length) {
-      return (
-        await dataView.getFields({
-          fieldName: fields,
-        })
-      ).getFieldMapSorted();
-    }
-    // no fields needed to be loaded for query
-    return {};
+    return await queryToFields({ dataView, request });
   }
 
   private flatten() {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts

@@ -104,6 +104,14 @@ export const createRuleTypeMocks = (
     alertWithPersistence: jest.fn(),
     logger: loggerMock,
     shouldWriteAlerts: () => true,
+    dataViews: {
+      createDataViewLazy: jest.fn().mockResolvedValue({
+        getFields: jest.fn().mockResolvedValue({
+          getFieldMapSorted: jest.fn().mockReturnValue({}),
+        }),
+        getSourceFiltering: jest.fn().mockReturnValue({ excludes: [] }),
+      }),
+    },
   };
 
   return {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -26,6 +26,8 @@ import {
   hasTimestampFields,
   isMachineLearningParams,
   isEsqlParams,
+  isQueryParams,
+  isEqlParams,
   getDisabledActionsWarningText,
 } from './utils/utils';
 import { DEFAULT_MAX_SIGNALS, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants';
@@ -341,7 +343,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
             });
           }
 
-          if (!isMachineLearningParams(params) && !isEsqlParams(params)) {
+          if (
+            !isMachineLearningParams(params) &&
+            !isEsqlParams(params) &&
+            !isQueryParams(params) &&
+            !isEqlParams(params)
+          ) {
             inputIndexFields = await getFieldsForWildcard({
               index: inputIndex,
               dataViews: services.dataViews,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts

@@ -59,6 +59,7 @@ export const queryExecutor = async ({
       index: runOpts.inputIndex,
       exceptionFilter: runOpts.exceptionFilter,
       fields: runOpts.inputIndexFields,
+      loadFields: true,
     });
 
     const license = await firstValueFrom(licensing.license$);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts

@@ -26,7 +26,8 @@ import type { SavedIdOrUndefined } from '../../../../../common/api/detection_eng
 import type { PartialFilter } from '../../types';
 import { withSecuritySpan } from '../../../../utils/with_security_span';
 import type { ESBoolQuery } from '../../../../../common/typed_json';
-import { getQueryFilter } from './get_query_filter';
+import { getQueryFilter as getQueryFilterNoLoadFields } from './get_query_filter';
+import { getQueryFilterLoadFields } from './get_query_filter_load_fields';
 
 export interface GetFilterArgs {
   type: Type;
@@ -38,6 +39,7 @@ export interface GetFilterArgs {
   index: IndexPatternArray | undefined;
   exceptionFilter: Filter | undefined;
   fields?: DataViewFieldBase[];
+  loadFields?: boolean;
 }
 
 interface QueryAttributes {
@@ -59,7 +61,11 @@ export const getFilter = async ({
   query,
   exceptionFilter,
   fields = [],
+  loadFields = false,
 }: GetFilterArgs): Promise<ESBoolQuery> => {
+  const getQueryFilter = loadFields
+    ? getQueryFilterLoadFields(services.dataViews)
+    : getQueryFilterNoLoadFields;
   const queryFilter = () => {
     if (query != null && language != null && index != null) {
       return getQueryFilter({

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_query_filter_load_fields.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Language } from '@kbn/securitysolution-io-ts-alerting-types';
+import type { Filter, EsQueryConfig, DataViewFieldBase } from '@kbn/es-query';
+import { DataView } from '@kbn/data-views-plugin/server';
+import { queryToFields } from '@kbn/data-plugin/common';
+import type { DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common';
+import { buildEsQuery } from '@kbn/es-query';
+import type { ESBoolQuery } from '../../../../../common/typed_json';
+import { getAllFilters } from './get_query_filter';
+import type {
+  IndexPatternArray,
+  RuleQuery,
+} from '../../../../../common/api/detection_engine/model/rule_schema';
+
+export const getQueryFilterLoadFields =
+  (dataViewsService: DataViewsContract) =>
+  async ({
+    query,
+    language,
+    filters,
+    index,
+    exceptionFilter,
+  }: {
+    query: RuleQuery;
+    language: Language;
+    filters: unknown;
+    index: IndexPatternArray;
+    exceptionFilter: Filter | undefined;
+    fields?: DataViewFieldBase[];
+  }): Promise<ESBoolQuery> => {
+    const config: EsQueryConfig = {
+      allowLeadingWildcards: true,
+      queryStringOptions: { analyze_wildcard: true },
+      ignoreFilterIfFieldNotInIndex: false,
+      dateFormatTZ: 'Zulu',
+    };
+
+    const initialQuery = { query, language };
+    const allFilters = getAllFilters(filters as Filter[], exceptionFilter);
+
+    const title = (index ?? []).join();
+
+    const dataViewLazy = await dataViewsService.createDataViewLazy({ title });
+
+    const flds = await queryToFields({
+      dataView: dataViewLazy,
+      request: { query: [initialQuery], filters: allFilters },
+    });
+
+    const dataViewLimitedFields = new DataView({
+      spec: { title },
+      fieldFormats: {} as unknown as FieldFormatsStartCommon,
+      shortDotsEnable: false,
+      metaFields: [],
+    });
+
+    dataViewLimitedFields.fields.replaceAll(Object.values(flds).map((fld) => fld.toSpec()));
+
+    return buildEsQuery(dataViewLimitedFields, initialQuery, allFilters, config);
+  };