[data.search] Improve SearchSource types (part 1) (#184531)

## Summary

Minor improvements to types in `SearchSource`.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

examples/embeddable_examples/public/react_embeddables/data_table/data_table_queries.ts

@@ -40,8 +40,7 @@ export const initializeDataTableQueries = async (
 
   // set up search source
   let abortController: AbortController | undefined;
-  const fields: Record<string, string> = { field: '*', include_unmapped: 'true' };
-  searchSource.setField('fields', [fields]);
+  searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
   searchSource.setField('size', 50);
 
   // initialize state for API.

packages/kbn-es-types/index.ts

@@ -22,4 +22,5 @@ export type {
   ESQLRow,
   ESQLSearchReponse,
   ESQLSearchParams,
+  SearchField,
 } from './src';

packages/kbn-es-types/src/index.ts

@@ -6,6 +6,10 @@
  * Side Public License, v 1.
  */
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  Field,
+  QueryDslFieldAndFormat,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import {
   InferSearchResponseOf,
   AggregateOf as AggregationResultOf,
@@ -39,6 +43,9 @@ export type ESSearchResponse<
   TOptions extends { restTotalHitsAsInt: boolean } = { restTotalHitsAsInt: false }
 > = InferSearchResponseOf<TDocument, TSearchRequest, TOptions>;
 
+// `fields` parameter from a search request (estypes.SearchRequest)
+export type SearchField = QueryDslFieldAndFormat | Field;
+
 export type {
   InferSearchResponseOf,
   AggregationResultOf,

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -11,7 +11,7 @@ import type { DataView } from '@kbn/data-views-plugin/common';
 import { buildExpression, ExpressionAstExpression } from '@kbn/expressions-plugin/common';
 import type { MockedKeys } from '@kbn/utility-types-jest';
 import type { ISearchGeneric } from '@kbn/search-types';
-import { SearchSource, SearchSourceDependencies, SortDirection } from '.';
+import { SearchFieldValue, SearchSource, SearchSourceDependencies, SortDirection } from '.';
 import { AggConfigs, AggTypesRegistryStart } from '../..';
 import { mockAggTypesRegistry } from '../aggs/test_helpers';
 import { RequestAdapter, RequestResponder } from '@kbn/inspector-plugin/common';
@@ -462,7 +462,11 @@ describe('SearchSource', () => {
             runtimeFields: {},
           }),
         } as unknown as DataView);
-        searchSource.setField('fields', ['hello', 'a', { foo: 'c' }]);
+        searchSource.setField('fields', [
+          'hello',
+          'a',
+          { foo: 'c' } as unknown as SearchFieldValue,
+        ]);
 
         const request = searchSource.getSearchRequestBody();
         expect(request.script_fields).toEqual({ hello: {} });
@@ -625,7 +629,7 @@ describe('SearchSource', () => {
             runtimeFields: {},
           }),
         } as unknown as DataView);
-        searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+        searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
 
         const request = searchSource.getSearchRequestBody();
         expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);
@@ -641,7 +645,7 @@ describe('SearchSource', () => {
             runtimeFields: {},
           }),
         } as unknown as DataView);
-        searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+        searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
 
         const request = searchSource.getSearchRequestBody();
         expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]);

src/plugins/data/common/search/search_source/search_source.ts

@@ -629,14 +629,14 @@ export class SearchSource {
     val = typeof val === 'function' ? val(this) : val;
     if (val == null || !key) return;
 
-    const addToRoot = (rootKey: string, value: any) => {
+    const addToRoot = (rootKey: string, value: unknown) => {
       data[rootKey] = value;
     };
 
     /**
      * Add the key and val to the body of the request
      */
-    const addToBody = (bodyKey: string, value: any) => {
+    const addToBody = (bodyKey: string, value: unknown) => {
       // ignore if we already have a value
       if (data.body[bodyKey] == null) {
         data.body[bodyKey] = value;
@@ -717,7 +717,7 @@ export class SearchSource {
   private getFieldsWithoutSourceFilters(
     index: DataView | undefined,
     bodyFields: SearchFieldValue[]
-  ) {
+  ): SearchFieldValue[] {
     if (!index) {
       return bodyFields;
     }
@@ -727,9 +727,7 @@ export class SearchSource {
       return bodyFields;
     }
     const sourceFiltersValues = sourceFilters.excludes;
-    const wildcardField = bodyFields.find(
-      (el: SearchFieldValue) => el === '*' || (el as Record<string, string>).field === '*'
-    );
+    const wildcardField = bodyFields.find((el) => this.getFieldName(el) === '*');
     const filter = fieldWildcardFilter(
       sourceFiltersValues,
       this.dependencies.getConfig(UI_SETTINGS.META_FIELDS)
@@ -748,7 +746,7 @@ export class SearchSource {
   }
 
   private getFieldFromDocValueFieldsOrIndexPattern(
-    docvaluesIndex: Record<string, object>,
+    docvaluesIndex: Record<string, SearchFieldValue>,
     fld: SearchFieldValue,
     index?: DataView
   ) {
@@ -756,10 +754,7 @@ export class SearchSource {
       return fld;
     }
     const fieldName = this.getFieldName(fld);
-    const field = {
-      ...docvaluesIndex[fieldName],
-      ...fld,
-    };
+    const field = Object.assign({}, docvaluesIndex[fieldName], fld);
     if (!index) {
       return field;
     }
@@ -894,7 +889,6 @@ export class SearchSource {
         fields,
         docvalueFields: body.docvalue_fields,
         fieldsFromSource,
-        // @ts-expect-error - Needs closer look to fix
         filteredDocvalueFields,
         metaFields,
         fieldListProvided,
@@ -930,7 +924,7 @@ export class SearchSource {
     fields: SearchFieldValue[];
     fieldsFromSource: SearchFieldValue[];
   }) {
-    const bodyFieldNames = fields.map((field: SearchFieldValue) => this.getFieldName(field));
+    const bodyFieldNames = fields.map((field) => this.getFieldName(field));
     return [...new Set([...bodyFieldNames, ...fieldsFromSource])];
   }
 
@@ -997,7 +991,7 @@ export class SearchSource {
     fields: SearchFieldValue[];
     docvalueFields: Array<{ field: string; format: string }>;
     fieldsFromSource: SearchFieldValue[];
-    filteredDocvalueFields: Array<{ field: string; format: string }>;
+    filteredDocvalueFields: SearchFieldValue[];
     metaFields: string[];
     fieldListProvided: boolean;
     sourceFieldsProvided: boolean;
@@ -1016,13 +1010,11 @@ export class SearchSource {
       }
       return [
         ...fields,
-        ...filteredDocvalueFields.filter((fld: SearchFieldValue) => {
+        ...filteredDocvalueFields.filter((fld) => {
           const fldName = this.getFieldName(fld);
           return (
             fieldsFromSource.includes(fldName) &&
-            !(docvalueFields || [])
-              .map((d: string | Record<string, SearchFieldValue>) => this.getFieldName(d))
-              .includes(fldName)
+            !(docvalueFields || []).map((d) => this.getFieldName(d)).includes(fldName)
           );
         }),
       ];
@@ -1040,7 +1032,7 @@ export class SearchSource {
     index?: DataView;
     fields: SearchFieldValue[];
     metaFields: string[];
-    filteredDocvalueFields: Array<{ field: string; format: string }>;
+    filteredDocvalueFields: SearchFieldValue[];
   }) {
     const bodyFields = this.getFieldsWithoutSourceFilters(index, fields);
     // if items that are in the docvalueFields are provided, we should

src/plugins/data/common/search/search_source/types.ts

@@ -8,12 +8,12 @@
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { RequestAdapter } from '@kbn/inspector-plugin/common';
-import { Query, AggregateQuery } from '@kbn/es-query';
-import { SerializableRecord } from '@kbn/utility-types';
-import { PersistableStateService } from '@kbn/kibana-utils-plugin/common';
-import type { Filter } from '@kbn/es-query';
-import { ISearchOptions } from '@kbn/search-types';
+import type { AggregateQuery, Filter, Query } from '@kbn/es-query';
+import type { Serializable, SerializableRecord } from '@kbn/utility-types';
+import type { PersistableStateService } from '@kbn/kibana-utils-plugin/common';
+import type { ISearchOptions } from '@kbn/search-types';
 import type { DataView, DataViewSpec } from '@kbn/data-views-plugin/common';
+import type { SearchField } from '@kbn/es-types';
 import type { AggConfigSerialized, IAggConfigs } from '../../../public';
 import type { SearchSource } from './search_source';
 
@@ -62,12 +62,7 @@ export type EsQuerySortValue = Record<
   SortDirection | SortDirectionNumeric | SortDirectionFormat
 >;
 
-interface SearchField {
-  [key: string]: SearchFieldValue;
-}
-
-// @internal
-export type SearchFieldValue = string | SearchField;
+export type SearchFieldValue = SearchField & Serializable;
 
 /**
  * search source fields

src/plugins/discover/public/application/context/services/anchor.test.ts

@@ -257,7 +257,7 @@ describe('context app', function () {
         const removeFieldsSpy = searchSourceStub.removeField.withArgs('fieldsFromSource');
         expect(setFieldsSpy.calledOnce).toBe(true);
         expect(removeFieldsSpy.calledOnce).toBe(true);
-        expect(setFieldsSpy.firstCall.args[1]).toEqual([{ field: '*', include_unmapped: 'true' }]);
+        expect(setFieldsSpy.firstCall.args[1]).toEqual([{ field: '*', include_unmapped: true }]);
       });
     });
   });

src/plugins/discover/public/application/context/services/anchor.ts

@@ -88,7 +88,7 @@ export function updateSearchSource(
     .setField('trackTotalHits', false);
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+    searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
   }
   return searchSource;
 }

src/plugins/discover/public/application/context/services/context.ts

@@ -135,7 +135,7 @@ export function updateSearchSource(
 ) {
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]);
+    searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
   }
   return searchSource
     .setParent(undefined)

src/plugins/discover/public/application/main/data_fetching/update_search_source.test.ts

@@ -42,7 +42,7 @@ describe('updateVolatileSearchSource', () => {
       sort: [] as SortOrder[],
       customFilters: [],
     });
-    expect(searchSource.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]);
+    expect(searchSource.getField('fields')).toEqual([{ field: '*', include_unmapped: true }]);
     expect(searchSource.getField('fieldsFromSource')).toBe(undefined);
   });
 
@@ -56,7 +56,7 @@ describe('updateVolatileSearchSource', () => {
       customFilters: [],
     });
     expect(volatileSearchSourceMock.getField('fields')).toEqual([
-      { field: '*', include_unmapped: 'true' },
+      { field: '*', include_unmapped: true },
     ]);
     expect(volatileSearchSourceMock.getField('fieldsFromSource')).toBe(undefined);
   });

src/plugins/discover/public/application/main/data_fetching/update_search_source.ts

@@ -56,11 +56,7 @@ export function updateVolatileSearchSource(
 
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    const fields: Record<string, string> = { field: '*' };
-
-    fields.include_unmapped = 'true';
-
-    searchSource.setField('fields', [fields]);
+    searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
   } else {
     searchSource.removeField('fields');
   }

src/plugins/discover/public/embeddable/utils/update_search_source.test.ts

@@ -53,7 +53,7 @@ describe('updateSearchSource', () => {
       true,
       defaults
     );
-    expect(searchSource.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]);
+    expect(searchSource.getField('fields')).toEqual([{ field: '*', include_unmapped: true }]);
     expect(searchSource.getField('fieldsFromSource')).toBe(undefined);
     expect(searchSource.getField('size')).toEqual(customSampleSize);
   });

src/plugins/discover/public/embeddable/utils/update_search_source.ts

@@ -33,8 +33,7 @@ export const updateSearchSource = (
   );
   if (useNewFieldsApi) {
     searchSource.removeField('fieldsFromSource');
-    const fields: Record<string, string> = { field: '*', include_unmapped: 'true' };
-    searchSource.setField('fields', [fields]);
+    searchSource.setField('fields', [{ field: '*', include_unmapped: true }]);
   } else {
     searchSource.removeField('fields');
   }

src/plugins/discover/public/utils/get_sharing_data.test.ts

@@ -83,7 +83,7 @@ describe('getSharingData', () => {
         "fields": Array [
           Object {
             "field": "*",
-            "include_unmapped": "true",
+            "include_unmapped": true,
           },
         ],
         "index": "the-data-view-id",
@@ -154,13 +154,13 @@ describe('getSharingData', () => {
       services
     );
     expect(getSearchSource({}).fields).toStrictEqual([
-      { field: 'cool-timefield', include_unmapped: 'true' },
-      { field: 'cool-field-1', include_unmapped: 'true' },
-      { field: 'cool-field-2', include_unmapped: 'true' },
-      { field: 'cool-field-3', include_unmapped: 'true' },
-      { field: 'cool-field-4', include_unmapped: 'true' },
-      { field: 'cool-field-5', include_unmapped: 'true' },
-      { field: 'cool-field-6', include_unmapped: 'true' },
+      { field: 'cool-timefield', include_unmapped: true },
+      { field: 'cool-field-1', include_unmapped: true },
+      { field: 'cool-field-2', include_unmapped: true },
+      { field: 'cool-field-3', include_unmapped: true },
+      { field: 'cool-field-4', include_unmapped: true },
+      { field: 'cool-field-5', include_unmapped: true },
+      { field: 'cool-field-6', include_unmapped: true },
     ]);
   });
 
@@ -190,9 +190,9 @@ describe('getSharingData', () => {
       services
     );
     expect(getSearchSource({}).fields).toStrictEqual([
-      { field: 'cool-timefield', include_unmapped: 'true' },
-      { field: 'cool-field-1', include_unmapped: 'true' },
-      { field: 'cool-field-2.*', include_unmapped: 'true' },
+      { field: 'cool-timefield', include_unmapped: true },
+      { field: 'cool-field-1', include_unmapped: true },
+      { field: 'cool-field-2.*', include_unmapped: true },
     ]);
   });
 

src/plugins/discover/public/utils/get_sharing_data.ts

@@ -125,9 +125,9 @@ export async function getSharingData(
                 field = `${column}.*`;
               }
 
-              return { field, include_unmapped: 'true' };
+              return { field, include_unmapped: true };
             })
-          : [{ field: '*', include_unmapped: 'true' }];
+          : [{ field: '*', include_unmapped: true }];
 
         searchSourceUpdated.setField('fields', fields);
       }

src/plugins/unified_doc_viewer/public/hooks/use_es_doc_search.test.tsx

@@ -87,7 +87,7 @@ describe('Test of <Doc /> helper / hook', () => {
           "fields": Array [
             Object {
               "field": "*",
-              "include_unmapped": "true",
+              "include_unmapped": true,
             },
           ],
           "query": Object {
@@ -131,7 +131,7 @@ describe('Test of <Doc /> helper / hook', () => {
           "fields": Array [
             Object {
               "field": "*",
-              "include_unmapped": "true",
+              "include_unmapped": true,
             },
           ],
           "query": Object {
@@ -185,7 +185,7 @@ describe('Test of <Doc /> helper / hook', () => {
           "fields": Array [
             Object {
               "field": "*",
-              "include_unmapped": "true",
+              "include_unmapped": true,
             },
           ],
           "query": Object {

src/plugins/unified_doc_viewer/public/hooks/use_es_doc_search.ts

@@ -140,8 +140,7 @@ export function buildSearchBody(
     return undefined;
   }
   if (useNewFieldsApi) {
-    // @ts-expect-error
-    request.body.fields = [{ field: '*', include_unmapped: 'true' }];
+    request.body.fields = [{ field: '*', include_unmapped: true }];
     request.body.runtime_mappings = runtimeFields ? runtimeFields : {};
     if (requestAllFields) {
       request.body._source = true;

x-pack/test/reporting_api_integration/reporting_and_security/generate_csv_discover.ts

@@ -109,14 +109,14 @@ export default function ({ getService }: FtrProviderContext) {
             objectType: 'search',
             searchSource: {
               fields: [
-                { field: 'order_date', include_unmapped: 'true' },
-                { field: 'category', include_unmapped: 'true' },
-                { field: 'currency', include_unmapped: 'true' },
-                { field: 'customer_id', include_unmapped: 'true' },
-                { field: 'order_id', include_unmapped: 'true' },
-                { field: 'day_of_week_i', include_unmapped: 'true' },
-                { field: 'products.created_on', include_unmapped: 'true' },
-                { field: 'sku', include_unmapped: 'true' },
+                { field: 'order_date', include_unmapped: true },
+                { field: 'category', include_unmapped: true },
+                { field: 'currency', include_unmapped: true },
+                { field: 'customer_id', include_unmapped: true },
+                { field: 'order_id', include_unmapped: true },
+                { field: 'day_of_week_i', include_unmapped: true },
+                { field: 'products.created_on', include_unmapped: true },
+                { field: 'sku', include_unmapped: true },
               ],
               filter: [
                 {
@@ -183,7 +183,7 @@ export default function ({ getService }: FtrProviderContext) {
             version: true,
             query: { query: '', language: 'kuery' },
             index: '5c620ea0-dc4f-11ec-972a-bf98ce1eebd7',
-            fields: fields.map((field) => ({ field, include_unmapped: 'true' })),
+            fields: fields.map((field) => ({ field, include_unmapped: true })),
             filter: [],
           },
         });
@@ -621,7 +621,7 @@ export default function ({ getService }: FtrProviderContext) {
             title: 'A Big Integer for _id CSV Report',
             searchSource: {
               query: { query: '', language: 'kuery' },
-              fields: [{ field: '*', include_unmapped: 'true' }],
+              fields: [{ field: '*', include_unmapped: true }],
               index: 'c424ce04-f440-4f48-aa0c-534da84d06f6',
               sort: [{ timestamp: 'desc' as SortDirection }],
               filter: [

x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts

@@ -149,7 +149,7 @@ export default function ({ getService }: FtrProviderContext) {
             objectType: 'search',
             searchSource: {
               version: true,
-              fields: [{ field: '*', include_unmapped: 'true' }],
+              fields: [{ field: '*', include_unmapped: true }],
               index: '5193f870-d861-11e9-a311-0fa548c5f953',
             } as unknown as SerializedSearchSourceFields,
             columns: [],

x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts

@@ -108,14 +108,14 @@ export default function ({ getService }: FtrProviderContext) {
             objectType: 'search',
             searchSource: {
               fields: [
-                { field: 'order_date', include_unmapped: 'true' },
-                { field: 'category', include_unmapped: 'true' },
-                { field: 'currency', include_unmapped: 'true' },
-                { field: 'customer_id', include_unmapped: 'true' },
-                { field: 'order_id', include_unmapped: 'true' },
-                { field: 'day_of_week_i', include_unmapped: 'true' },
-                { field: 'products.created_on', include_unmapped: 'true' },
-                { field: 'sku', include_unmapped: 'true' },
+                { field: 'order_date', include_unmapped: true },
+                { field: 'category', include_unmapped: true },
+                { field: 'currency', include_unmapped: true },
+                { field: 'customer_id', include_unmapped: true },
+                { field: 'order_id', include_unmapped: true },
+                { field: 'day_of_week_i', include_unmapped: true },
+                { field: 'products.created_on', include_unmapped: true },
+                { field: 'sku', include_unmapped: true },
               ],
               filter: [
                 {
@@ -181,7 +181,7 @@ export default function ({ getService }: FtrProviderContext) {
               version: true,
               index: '5c620ea0-dc4f-11ec-972a-bf98ce1eebd7',
               query: { language: 'kuery', query: '' },
-              fields: fields.map((field) => ({ field, include_unmapped: 'true' })),
+              fields: fields.map((field) => ({ field, include_unmapped: true })),
               filter: [],
               sort: [{ text: 'asc' as SortDirection }],
             },
@@ -621,7 +621,7 @@ export default function ({ getService }: FtrProviderContext) {
             title: 'A Big Integer for _id CSV Report',
             searchSource: {
               query: { query: '', language: 'kuery' },
-              fields: [{ field: '*', include_unmapped: 'true' }],
+              fields: [{ field: '*', include_unmapped: true }],
               index: 'c424ce04-f440-4f48-aa0c-534da84d06f6',
               sort: [{ timestamp: 'desc' as SortDirection }],
               filter: [