[UnifiedFieldList] Add new fields ingested in background with valid mappings (#172329)

Improves UnifiedFieldList by adding a newly ingested field to the list with the right type. On top of
that, it triggers a refresh of the selected DataView fields, so the new field be available by consumers of the DataView.

Co-authored-by: Julia Rechkunova <julia.rechkunova@elastic.co>

packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx

@@ -22,7 +22,7 @@ import {
   useEuiTheme,
 } from '@elastic/eui';
 import { ToolbarButton } from '@kbn/shared-ux-button-toolbar';
-import { type DataViewField } from '@kbn/data-views-plugin/public';
+import { DataViewField, type FieldSpec } from '@kbn/data-views-plugin/common';
 import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils';
 import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils';
 import { FieldList } from '../../components/field_list';
@@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
     onSelectedFieldFilter,
   ]);
 
-  useEffect(() => {
-    if (
-      searchMode !== 'documents' ||
-      !useNewFieldsApi ||
-      stateService.creationOptions.disableMultiFieldsGroupingByParent
-    ) {
-      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
-    } else {
-      setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap));
-    }
-  }, [
-    stateService.creationOptions.disableMultiFieldsGroupingByParent,
-    selectedFieldsState.selectedFieldsMap,
-    allFields,
-    useNewFieldsApi,
-    setMultiFieldsMap,
-    searchMode,
-  ]);
-
   const popularFieldsLimit = useMemo(
     () => core.uiSettings.get(FIELDS_LIMIT_SETTING),
     [core.uiSettings]
@@ -226,24 +207,47 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
       [searchMode, stateService.creationOptions.disableMultiFieldsGroupingByParent]
     );
 
-  const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields<DataViewField>({
-    dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries
-    allFields,
-    popularFieldsLimit:
-      searchMode !== 'documents' || stateService.creationOptions.disablePopularFields
-        ? 0
-        : popularFieldsLimit,
-    isAffectedByGlobalFilter,
-    services: {
-      dataViews,
-      core,
-    },
-    sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields,
-    onSelectedFieldFilter,
-    onSupportedFieldFilter:
-      stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter,
-    onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails,
-  });
+  const { fieldListFiltersProps, fieldListGroupedProps, allFieldsModified } =
+    useGroupedFields<DataViewField>({
+      dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries
+      allFields,
+      popularFieldsLimit:
+        searchMode !== 'documents' || stateService.creationOptions.disablePopularFields
+          ? 0
+          : popularFieldsLimit,
+      isAffectedByGlobalFilter,
+      services: {
+        dataViews,
+        core,
+      },
+      sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields,
+      onSelectedFieldFilter,
+      onSupportedFieldFilter:
+        stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter,
+      onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails,
+      getNewFieldsBySpec,
+    });
+
+  useEffect(() => {
+    if (
+      searchMode !== 'documents' ||
+      !useNewFieldsApi ||
+      stateService.creationOptions.disableMultiFieldsGroupingByParent
+    ) {
+      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
+    } else {
+      setMultiFieldsMap(
+        calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap)
+      );
+    }
+  }, [
+    stateService.creationOptions.disableMultiFieldsGroupingByParent,
+    selectedFieldsState.selectedFieldsMap,
+    allFieldsModified,
+    useNewFieldsApi,
+    setMultiFieldsMap,
+    searchMode,
+  ]);
 
   const renderFieldItem: FieldListGroupedProps<DataViewField>['renderFieldItem'] = useCallback(
     ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
@@ -456,3 +460,7 @@ function calculateMultiFields(
   });
   return map;
 }
+
+function getNewFieldsBySpec(fieldSpecArr: FieldSpec[]): DataViewField[] {
+  return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec));
+}

packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx

@@ -125,6 +125,7 @@ describe('UnifiedFieldList useExistingFields', () => {
     expect(hookReader.result.current.getFieldsExistenceStatus(dataViewId)).toBe(
       ExistenceFetchStatus.succeeded
     );
+    expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]);
 
     // does not have existence info => works less restrictive
     const anotherDataViewId = 'test-id';
@@ -140,6 +141,7 @@ describe('UnifiedFieldList useExistingFields', () => {
     expect(hookReader.result.current.getFieldsExistenceStatus(anotherDataViewId)).toBe(
       ExistenceFetchStatus.unknown
     );
+    expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]);
   });
 
   it('should work correctly with multiple readers', async () => {
@@ -217,6 +219,7 @@ describe('UnifiedFieldList useExistingFields', () => {
     expect(currentResult.isFieldsExistenceInfoUnavailable(dataViewId)).toBe(true);
     expect(currentResult.hasFieldData(dataViewId, dataView.fields[0].name)).toBe(true);
     expect(currentResult.getFieldsExistenceStatus(dataViewId)).toBe(ExistenceFetchStatus.failed);
+    expect(currentResult.getNewFields(dataViewId)).toStrictEqual([]);
   });
 
   it('should work correctly for multiple data views', async () => {
@@ -533,4 +536,49 @@ describe('UnifiedFieldList useExistingFields', () => {
 
     expect(params.onNoData).toHaveBeenCalledTimes(1); // still 1 time
   });
+
+  it('should include newFields', async () => {
+    const newFields = [{ name: 'test', type: 'keyword', searchable: true, aggregatable: true }];
+
+    (ExistingFieldsServiceApi.loadFieldExisting as jest.Mock).mockImplementation(
+      async ({ dataView: currentDataView }) => {
+        return {
+          existingFieldNames: [currentDataView.fields[0].name],
+          newFields,
+        };
+      }
+    );
+
+    const params: ExistingFieldsFetcherParams = {
+      dataViews: [dataView],
+      services: mockedServices,
+      fromDate: '2019-01-01',
+      toDate: '2020-01-01',
+      query: { query: '', language: 'lucene' },
+      filters: [],
+    };
+    const hookFetcher = renderHook(useExistingFieldsFetcher, {
+      initialProps: params,
+    });
+
+    const hookReader = renderHook(useExistingFieldsReader);
+    await hookFetcher.waitForNextUpdate();
+
+    expect(ExistingFieldsServiceApi.loadFieldExisting).toHaveBeenCalledWith(
+      expect.objectContaining({
+        fromDate: '2019-01-01',
+        toDate: '2020-01-01',
+        dslQuery,
+        dataView,
+        timeFieldName: dataView.timeFieldName,
+      })
+    );
+
+    expect(hookReader.result.current.getFieldsExistenceStatus(dataView.id!)).toBe(
+      ExistenceFetchStatus.succeeded
+    );
+
+    expect(hookReader.result.current.getNewFields(dataView.id!)).toBe(newFields);
+    expect(hookReader.result.current.getNewFields('another-id')).toStrictEqual([]);
+  });
 });

packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts

@@ -12,7 +12,7 @@ import { BehaviorSubject } from 'rxjs';
 import type { CoreStart } from '@kbn/core/public';
 import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query';
 import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common';
 import { getEsQueryConfig } from '@kbn/data-service/src/es_query';
 import { reportPerformanceMetricEvent } from '@kbn/ebt-tools';
 import { loadFieldExisting } from '../services/field_existing';
@@ -20,10 +20,12 @@ import { ExistenceFetchStatus } from '../types';
 
 const getBuildEsQueryAsync = async () => (await import('@kbn/es-query')).buildEsQuery;
 const generateId = htmlIdGenerator();
+const DEFAULT_EMPTY_NEW_FIELDS: FieldSpec[] = [];
 
 export interface ExistingFieldsInfo {
   fetchStatus: ExistenceFetchStatus;
   existingFieldsByFieldNameMap: Record<string, boolean>;
+  newFields?: FieldSpec[];
   numberOfFetches: number;
   hasDataViewRestrictions?: boolean;
 }
@@ -54,6 +56,7 @@ export interface ExistingFieldsReader {
   hasFieldData: (dataViewId: string, fieldName: string) => boolean;
   getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus;
   isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean;
+  getNewFields: (dataViewId: string) => FieldSpec[];
 }
 
 const initialData: ExistingFieldsByDataViewMap = {};
@@ -157,6 +160,7 @@ export const useExistingFieldsFetcher = (
           }
 
           info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames);
+          info.newFields = result.newFields;
           info.fetchStatus = ExistenceFetchStatus.succeeded;
         } catch (error) {
           info.fetchStatus = ExistenceFetchStatus.failed;
@@ -286,6 +290,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
     [existingFieldsByDataViewMap]
   );
 
+  const getNewFields = useCallback(
+    (dataViewId: string) => {
+      const info = existingFieldsByDataViewMap[dataViewId];
+
+      if (info?.fetchStatus === ExistenceFetchStatus.succeeded) {
+        return info?.newFields ?? DEFAULT_EMPTY_NEW_FIELDS;
+      }
+
+      return DEFAULT_EMPTY_NEW_FIELDS;
+    },
+    [existingFieldsByDataViewMap]
+  );
+
   const getFieldsExistenceInfo = useCallback(
     (dataViewId: string) => {
       return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo;
@@ -321,8 +338,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
       hasFieldData,
       getFieldsExistenceStatus,
       isFieldsExistenceInfoUnavailable,
+      getNewFields,
     }),
-    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable]
+    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields]
   );
 };
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx

@@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -185,6 +187,8 @@ describe('UnifiedFieldList useGroupedFields()', () => {
 
     expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded);
     expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true);
+    expect(result.current.allFieldsModified).toBe(allFields);
+    expect(result.current.hasNewFields).toBe(false);
 
     rerender({
       ...props,
@@ -198,6 +202,82 @@ describe('UnifiedFieldList useGroupedFields()', () => {
     expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe(
       scrollToTopResetCounter1
     );
+    expect(result.current.allFieldsModified).toBe(allFields);
+    expect(result.current.hasNewFields).toBe(false);
+
+    (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore();
+  });
+
+  it('should work correctly with new fields', async () => {
+    const props: GroupedFieldsParams<DataViewField> = {
+      dataViewId: dataView.id!,
+      allFields,
+      services: mockedServices,
+      getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)),
+    };
+
+    const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true };
+
+    jest.spyOn(ExistenceApi, 'useExistingFieldsReader').mockImplementation(
+      (): ExistingFieldsReader => ({
+        hasFieldData: (dataViewId) => {
+          return dataViewId === props.dataViewId;
+        },
+        getFieldsExistenceStatus: (dataViewId) =>
+          dataViewId === props.dataViewId
+            ? ExistenceFetchStatus.succeeded
+            : ExistenceFetchStatus.unknown,
+        isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [newField],
+      })
+    );
+
+    const { result, waitForNextUpdate, rerender } = renderHook(useGroupedFields, {
+      initialProps: props,
+    });
+
+    await waitForNextUpdate();
+
+    let fieldListGroupedProps = result.current.fieldListGroupedProps;
+    const fieldGroups = fieldListGroupedProps.fieldGroups;
+    const scrollToTopResetCounter1 = fieldListGroupedProps.scrollToTopResetCounter;
+
+    expect(
+      Object.keys(fieldGroups!).map(
+        (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}`
+      )
+    ).toStrictEqual([
+      'SpecialFields-0',
+      'SelectedFields-0',
+      'PopularFields-0',
+      'AvailableFields-25',
+      'UnmappedFields-1',
+      'EmptyFields-0',
+      'MetaFields-3',
+    ]);
+
+    expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded);
+    expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true);
+    expect(result.current.allFieldsModified).toStrictEqual([
+      ...allFields,
+      new DataViewField(newField),
+    ]);
+    expect(result.current.hasNewFields).toBe(true);
+
+    rerender({
+      ...props,
+      dataViewId: null, // for text-based queries
+      allFields,
+    });
+
+    fieldListGroupedProps = result.current.fieldListGroupedProps;
+    expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded);
+    expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true);
+    expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe(
+      scrollToTopResetCounter1
+    );
+    expect(result.current.allFieldsModified).toBe(allFields);
+    expect(result.current.hasNewFields).toBe(false);
 
     (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore();
   });
@@ -438,6 +518,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId,
+        getNewFields: () => [],
       })
     );
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts

@@ -10,8 +10,9 @@ import { groupBy } from 'lodash';
 import { useEffect, useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { type CoreStart } from '@kbn/core-lifecycle-browser';
-import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
 import { type DataViewsContract } from '@kbn/data-views-plugin/public';
+import { type UseNewFieldsParams, useNewFields } from './use_new_fields';
 import {
   type FieldListGroups,
   type FieldsGroup,
@@ -41,6 +42,7 @@ export interface GroupedFieldsParams<T extends FieldListItem> {
   onOverrideFieldGroupDetails?: OverrideFieldGroupDetails;
   onSupportedFieldFilter?: (field: T) => boolean;
   onSelectedFieldFilter?: (field: T) => boolean;
+  getNewFieldsBySpec?: UseNewFieldsParams<T>['getNewFieldsBySpec'];
 }
 
 export interface GroupedFieldsResult<T extends FieldListItem> {
@@ -52,6 +54,8 @@ export interface GroupedFieldsResult<T extends FieldListItem> {
     fieldsExistInIndex: boolean;
     screenReaderDescriptionId?: string;
   };
+  allFieldsModified: T[] | null; // `null` is for loading indicator
+  hasNewFields: boolean;
 }
 
 export function useGroupedFields<T extends FieldListItem = DataViewField>({
@@ -65,6 +69,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   onOverrideFieldGroupDetails,
   onSupportedFieldFilter,
   onSelectedFieldFilter,
+  getNewFieldsBySpec,
 }: GroupedFieldsParams<T>): GroupedFieldsResult<T> {
   const fieldsExistenceReader = useExistingFieldsReader();
   const fieldListFilters = useFieldFilters<T>({
@@ -73,6 +78,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     getCustomFieldType,
     onSupportedFieldFilter,
   });
+
   const onFilterFieldList = fieldListFilters.onFilterField;
   const [dataView, setDataView] = useState<DataView | null>(null);
   const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName);
@@ -101,6 +107,13 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     // if field existence information changed, reload the data view too
   }, [dataViewId, services.dataViews, setDataView, hasFieldDataHandler]);
 
+  const { allFieldsModified, hasNewFields } = useNewFields<T>({
+    dataView,
+    allFields,
+    getNewFieldsBySpec,
+    fieldsExistenceReader,
+  });
+
   // important when switching from a known dataViewId to no data view (like in text-based queries)
   useEffect(() => {
     if (dataView && !dataViewId) {
@@ -120,13 +133,16 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     };
 
     const selectedFields = sortedSelectedFields || [];
-    const sortedFields = [...(allFields || [])].sort(sortFields);
+
+    const sortedFields = [...(allFieldsModified || [])].sort(sortFields);
+
     const groupedFields = {
       ...getDefaultFieldGroups(),
       ...groupBy(sortedFields, (field) => {
         if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) {
           selectedFields.push(field);
         }
+
         if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) {
           return 'skippedFields';
         }
@@ -311,7 +327,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
 
     return fieldGroupDefinitions;
   }, [
-    allFields,
+    allFieldsModified,
     onSupportedFieldFilter,
     onSelectedFieldFilter,
     onOverrideFieldGroupDetails,
@@ -381,6 +397,8 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   return {
     fieldListGroupedProps,
     fieldListFiltersProps: fieldListFilters.fieldListFiltersProps,
+    allFieldsModified,
+    hasNewFields,
   };
 }
 

packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { renderHook } from '@testing-library/react-hooks';
+import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub';
+import { DataViewField } from '@kbn/data-views-plugin/common';
+import { useNewFields, type UseNewFieldsParams } from './use_new_fields';
+import { type ExistingFieldsReader } from './use_existing_fields';
+import { ExistenceFetchStatus } from '../types';
+
+const fieldsExistenceReader: ExistingFieldsReader = {
+  hasFieldData: (dataViewId) => {
+    return dataViewId === dataView.id;
+  },
+  getFieldsExistenceStatus: (dataViewId) =>
+    dataViewId === dataView.id ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown,
+  isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== dataView.id,
+  getNewFields: () => [],
+};
+
+describe('UnifiedFieldList useNewFields()', () => {
+  const allFields = dataView.fields;
+
+  it('should work correctly in loading state', async () => {
+    const props: UseNewFieldsParams<DataViewField> = {
+      dataView,
+      allFields: null,
+      fieldsExistenceReader,
+    };
+    const { result } = renderHook(useNewFields, {
+      initialProps: props,
+    });
+
+    expect(result.current.allFieldsModified).toBe(null);
+    expect(result.current.hasNewFields).toBe(false);
+  });
+
+  it('should work correctly with empty new fields', async () => {
+    const props: UseNewFieldsParams<DataViewField> = {
+      dataView,
+      allFields,
+      fieldsExistenceReader,
+    };
+    const { result } = renderHook(useNewFields, {
+      initialProps: props,
+    });
+
+    expect(result.current.allFieldsModified).toBe(allFields);
+    expect(result.current.hasNewFields).toBe(false);
+  });
+
+  it('should work correctly with new fields', async () => {
+    const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true };
+    const newField2 = { ...newField, name: 'test2' };
+    const props: UseNewFieldsParams<DataViewField> = {
+      dataView,
+      allFields,
+      fieldsExistenceReader: {
+        ...fieldsExistenceReader,
+        getNewFields: () => [newField, newField2],
+      },
+      getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)),
+    };
+    const { result } = renderHook(useNewFields, {
+      initialProps: props,
+    });
+
+    expect(result.current.allFieldsModified).toStrictEqual([
+      ...allFields,
+      new DataViewField(newField),
+      new DataViewField(newField2),
+    ]);
+    expect(result.current.hasNewFields).toBe(true);
+  });
+});

packages/kbn-unified-field-list/src/hooks/use_new_fields.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { useMemo } from 'react';
+import type { FieldSpec } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
+import type { FieldListItem } from '../types';
+import type { ExistingFieldsReader } from './use_existing_fields';
+
+export interface UseNewFieldsParams<T extends FieldListItem> {
+  dataView?: DataView | null;
+  allFields: T[] | null; // `null` is for loading indicator
+  getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[];
+  fieldsExistenceReader: ExistingFieldsReader;
+}
+
+export interface UseNewFieldsResult<T extends FieldListItem> {
+  allFieldsModified: T[] | null;
+  hasNewFields: boolean;
+}
+
+/**
+ * This hook is used to get the new fields of previous fields for wildcards request, and merges those
+ * with the existing fields.
+ */
+export function useNewFields<T extends FieldListItem = DataViewField>({
+  dataView,
+  allFields,
+  getNewFieldsBySpec,
+  fieldsExistenceReader,
+}: UseNewFieldsParams<T>): UseNewFieldsResult<T> {
+  const dataViewId = dataView?.id;
+
+  const newFields = useMemo(() => {
+    const newLoadedFields =
+      allFields && dataView?.id && getNewFieldsBySpec
+        ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView)
+        : null;
+
+    return newLoadedFields?.length ? newLoadedFields : null;
+  }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]);
+
+  const hasNewFields = Boolean(allFields && newFields && newFields.length > 0);
+
+  const allFieldsModified = useMemo(() => {
+    if (!allFields || !newFields?.length || !dataViewId) return allFields;
+    // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request
+    // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way
+    const allFieldsExlNew = allFields.filter(
+      (field) => !newFields.some((newField) => newField.name === field.name)
+    );
+
+    return [...allFieldsExlNew, ...newFields];
+  }, [newFields, allFields, dataViewId]);
+
+  return { allFieldsModified, hasNewFields };
+}

packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts

@@ -7,7 +7,7 @@
  */
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { RuntimeField } from '@kbn/data-views-plugin/common';
+import type { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common';
 import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common';
 import type { IKibanaSearchRequest } from '@kbn/data-plugin/common';
 
@@ -49,15 +49,25 @@ export async function fetchFieldExistence({
   metaFields: string[];
   dataViewsService: DataViewsContract;
 }) {
-  const allFields = buildFieldList(dataView, metaFields);
   const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
     // filled in by data views service
     pattern: '',
     indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
   });
+
+  // take care of fields of existingFieldList, that are not yet available
+  // in the given data view. Those fields we consider as new fields,
+  // that were ingested after the data view was loaded
+  const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name));
+  // refresh the data view in case there are new fields
+  if (newFields.length) {
+    await dataViewsService.refreshFields(dataView, false);
+  }
+  const allFields = buildFieldList(dataView, metaFields);
   return {
-    indexPatternTitle: dataView.title,
+    indexPatternTitle: dataView.getIndexPattern(),
     existingFieldNames: existingFields(existingFieldList, allFields),
+    newFields,
   };
 }
 
@@ -66,19 +76,23 @@ export async function fetchFieldExistence({
  */
 export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
   return indexPattern.fields.map((field) => {
-    return {
-      name: field.name,
-      isScript: !!field.scripted,
-      lang: field.lang,
-      script: field.script,
-      // id is a special case - it doesn't show up in the meta field list,
-      // but as it's not part of source, it has to be handled separately.
-      isMeta: metaFields?.includes(field.name) || field.name === '_id',
-      runtimeField: !field.isMapped ? field.runtimeField : undefined,
-    };
+    return buildField(field, metaFields);
   });
 }
 
+export function buildField(field: DataViewField, metaFields: string[]): Field {
+  return {
+    name: field.name,
+    isScript: !!field.scripted,
+    lang: field.lang,
+    script: field.script,
+    // id is a special case - it doesn't show up in the meta field list,
+    // but as it's not part of source, it has to be handled separately.
+    isMeta: metaFields?.includes(field.name) || field.name === '_id',
+    runtimeField: !field.isMapped ? field.runtimeField : undefined,
+  };
+}
+
 function toQuery(
   timeFieldName: string | undefined,
   fromDate: string | undefined,

packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts

@@ -9,7 +9,7 @@
 import { IUiSettingsClient } from '@kbn/core/public';
 import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
 import { UI_SETTINGS } from '@kbn/data-service/src/constants';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common';
 import { lastValueFrom } from 'rxjs';
 import { fetchFieldExistence } from './field_existing_utils';
 
@@ -27,6 +27,7 @@ interface FetchFieldExistenceParams {
 export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{
   existingFieldNames: string[];
   indexPatternTitle: string;
+  newFields?: FieldSpec[];
 }>;
 
 export const loadFieldExisting: LoadFieldExistingHandler = async ({

src/plugins/unified_search/public/dataview_picker/change_dataview.tsx

@@ -307,12 +307,6 @@ export function ChangeDataView({
           isTextBasedLangSelected={isTextBasedLangSelected}
           setPopoverIsOpen={setPopoverIsOpen}
           onChangeDataView={async (newId) => {
-            try {
-              // refreshing the field list
-              await dataViews.get(newId, false, true);
-            } catch (e) {
-              //
-            }
             setSelectedDataViewId(newId);
             setPopoverIsOpen(false);
             if (isTextBasedLangSelected && !isTextLangTransitionModalDismissed) {

test/functional/apps/discover/group4/_field_list_new_fields.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const security = getService('security');
+  const es = getService('es');
+  const retry = getService('retry');
+  const queryBar = getService('queryBar');
+  const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'unifiedFieldList']);
+
+  describe('Field list new fields in background handling', function () {
+    before(async () => {
+      await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
+      await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json');
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setCommonlyUsedTime('This_week');
+    });
+
+    after(async () => {
+      await kibanaServer.savedObjects.cleanStandardList();
+      await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional');
+      await es.transport.request({
+        path: '/my-index-000001',
+        method: 'DELETE',
+      });
+    });
+
+    it('Check that new ingested fields are added to the available fields section', async function () {
+      const initialPattern = 'my-index-';
+      await es.transport.request({
+        path: '/my-index-000001/_doc',
+        method: 'POST',
+        body: {
+          '@timestamp': new Date().toISOString(),
+          a: 'GET /search HTTP/1.1 200 1070000',
+        },
+      });
+
+      await PageObjects.discover.createAdHocDataView(initialPattern, true);
+
+      await retry.waitFor('current data view to get updated', async () => {
+        return (await PageObjects.discover.getCurrentlySelectedDataView()) === `${initialPattern}*`;
+      });
+      await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
+
+      expect(await PageObjects.discover.getHitCountInt()).to.be(1);
+      expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([
+        '@timestamp',
+        'a',
+      ]);
+
+      await es.transport.request({
+        path: '/my-index-000001/_doc',
+        method: 'POST',
+        body: {
+          '@timestamp': new Date().toISOString(),
+          b: 'GET /search HTTP/1.1 200 1070000',
+        },
+      });
+
+      await retry.waitFor('the new record was found', async () => {
+        await queryBar.submitQuery();
+        await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
+        return (await PageObjects.discover.getHitCountInt()) === 2;
+      });
+
+      expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([
+        '@timestamp',
+        'a',
+        'b',
+      ]);
+    });
+  });
+}

test/functional/apps/discover/group4/index.ts

@@ -33,5 +33,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./_context_encoded_url_params'));
     loadTestFile(require.resolve('./_hide_announcements'));
     loadTestFile(require.resolve('./_data_view_edit'));
+    loadTestFile(require.resolve('./_field_list_new_fields'));
   });
 }

x-pack/plugins/lens/public/data_views_service/loader.test.ts

@@ -5,8 +5,13 @@
  * 2.0.
  */
 
-import { DataViewsContract } from '@kbn/data-views-plugin/public';
-import { ensureIndexPattern, loadIndexPatternRefs, loadIndexPatterns } from './loader';
+import { DataViewsContract, DataViewField } from '@kbn/data-views-plugin/public';
+import {
+  ensureIndexPattern,
+  loadIndexPatternRefs,
+  loadIndexPatterns,
+  buildIndexPatternField,
+} from './loader';
 import { sampleIndexPatterns, mockDataViewsService } from './mocks';
 import { documentField } from '../datasources/form_based/document_field';
 
@@ -313,4 +318,32 @@ describe('loader', () => {
       expect(onError).not.toHaveBeenCalled();
     });
   });
+
+  describe('buildIndexPatternField', () => {
+    it('should return a field with the correct name and derived parameters', async () => {
+      const field = buildIndexPatternField({
+        name: 'foo',
+        displayName: 'Foo',
+        type: 'string',
+        aggregatable: true,
+        searchable: true,
+      } as DataViewField);
+      expect(field.name).toEqual('foo');
+      expect(field.meta).toEqual(false);
+      expect(field.runtime).toEqual(false);
+    });
+    it('should return return the right meta field value', async () => {
+      const field = buildIndexPatternField(
+        {
+          name: 'meta',
+          displayName: 'Meta',
+          type: 'string',
+          aggregatable: true,
+          searchable: true,
+        } as DataViewField,
+        new Set(['meta'])
+      );
+      expect(field.meta).toEqual(true);
+    });
+  });
 });

x-pack/plugins/lens/public/data_views_service/loader.ts

@@ -6,7 +6,12 @@
  */
 
 import { isFieldLensCompatible } from '@kbn/visualization-ui-components';
-import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public';
+import {
+  DataViewsContract,
+  DataView,
+  DataViewSpec,
+  DataViewField,
+} from '@kbn/data-views-plugin/public';
 import { keyBy } from 'lodash';
 import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types';
 import { documentField } from '../datasources/form_based/document_field';
@@ -32,46 +37,7 @@ export function convertDataViewIntoLensIndexPattern(
   const metaKeys = new Set(dataView.metaFields);
   const newFields = dataView.fields
     .filter(isFieldLensCompatible)
-    .map((field): IndexPatternField => {
-      // Convert the getters on the index pattern service into plain JSON
-      const base = {
-        name: field.name,
-        displayName: field.displayName,
-        type: field.type,
-        aggregatable: field.aggregatable,
-        filterable: field.filterable,
-        searchable: field.searchable,
-        meta: metaKeys.has(field.name),
-        esTypes: field.esTypes,
-        scripted: field.scripted,
-        isMapped: field.isMapped,
-        customLabel: field.customLabel,
-        runtimeField: field.runtimeField,
-        runtime: Boolean(field.runtimeField),
-        timeSeriesDimension: field.timeSeriesDimension,
-        timeSeriesMetric: field.timeSeriesMetric,
-        timeSeriesRollup: field.isRolledUpField,
-        partiallyApplicableFunctions: field.isRolledUpField
-          ? {
-              percentile: true,
-              percentile_rank: true,
-              median: true,
-              last_value: true,
-              unique_count: true,
-              standard_deviation: true,
-            }
-          : undefined,
-      };
-
-      // Simplifies tests by hiding optional properties instead of undefined
-      return base.scripted
-        ? {
-            ...base,
-            lang: field.lang,
-            script: field.script,
-          }
-        : base;
-    })
+    .map((field) => buildIndexPatternField(field, metaKeys))
     .concat(documentField);
 
   const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView;
@@ -113,6 +79,51 @@ export function convertDataViewIntoLensIndexPattern(
   };
 }
 
+export function buildIndexPatternField(
+  field: DataViewField,
+  metaKeys?: Set<string>
+): IndexPatternField {
+  const meta = metaKeys ? metaKeys.has(field.name) : false;
+  // Convert the getters on the index pattern service into plain JSON
+  const base = {
+    name: field.name,
+    displayName: field.displayName,
+    type: field.type,
+    aggregatable: field.aggregatable,
+    filterable: field.filterable,
+    searchable: field.searchable,
+    meta,
+    esTypes: field.esTypes,
+    scripted: field.scripted,
+    isMapped: field.isMapped,
+    customLabel: field.customLabel,
+    runtimeField: field.runtimeField,
+    runtime: Boolean(field.runtimeField),
+    timeSeriesDimension: field.timeSeriesDimension,
+    timeSeriesMetric: field.timeSeriesMetric,
+    timeSeriesRollup: field.isRolledUpField,
+    partiallyApplicableFunctions: field.isRolledUpField
+      ? {
+          percentile: true,
+          percentile_rank: true,
+          median: true,
+          last_value: true,
+          unique_count: true,
+          standard_deviation: true,
+        }
+      : undefined,
+  };
+
+  // Simplifies tests by hiding optional properties instead of undefined
+  return base.scripted
+    ? {
+        ...base,
+        lang: field.lang,
+        script: field.script,
+      }
+    : base;
+}
+
 export async function loadIndexPatternRefs(
   dataViews: MinimalDataViewsContract
 ): Promise<IndexPatternRef[]> {

x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx

@@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { CoreStart } from '@kbn/core/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
-import { type DataView } from '@kbn/data-plugin/common';
+import { type DataView, DataViewField, FieldSpec } from '@kbn/data-plugin/common';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public';
 import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public';
@@ -28,6 +28,9 @@ import {
   useGroupedFields,
 } from '@kbn/unified-field-list';
 import { ChartsPluginSetup } from '@kbn/charts-plugin/public';
+import useLatest from 'react-use/lib/useLatest';
+import { isFieldLensCompatible } from '@kbn/visualization-ui-components';
+import { buildIndexPatternField } from '../../data_views_service/loader';
 import type {
   DatasourceDataPanelProps,
   FramePublicAPI,
@@ -249,18 +252,20 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
     }
   }, []);
 
-  const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields<IndexPatternField>({
-    dataViewId: currentIndexPatternId,
-    allFields,
-    services: {
-      dataViews,
-      core,
-    },
-    isAffectedByGlobalFilter: Boolean(filters.length),
-    onSupportedFieldFilter,
-    onSelectedFieldFilter,
-    onOverrideFieldGroupDetails,
-  });
+  const { fieldListFiltersProps, fieldListGroupedProps, hasNewFields } =
+    useGroupedFields<IndexPatternField>({
+      dataViewId: currentIndexPatternId,
+      allFields,
+      services: {
+        dataViews,
+        core,
+      },
+      isAffectedByGlobalFilter: Boolean(filters.length),
+      onSupportedFieldFilter,
+      onSelectedFieldFilter,
+      onOverrideFieldGroupDetails,
+      getNewFieldsBySpec,
+    });
 
   const closeFieldEditor = useRef<() => void | undefined>();
 
@@ -273,7 +278,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
     };
   }, []);
 
-  const refreshFieldList = useCallback(async () => {
+  const refreshFieldList = useLatest(async () => {
     if (currentIndexPattern) {
       const newlyMappedIndexPattern = await indexPatternService.loadIndexPatterns({
         patterns: [currentIndexPattern.id],
@@ -289,13 +294,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
     }
     // start a new session so all charts are refreshed
     data.search.session.start();
-  }, [
-    indexPatternService,
-    currentIndexPattern,
-    onIndexPatternRefresh,
-    frame.dataViews.indexPatterns,
-    data.search.session,
-  ]);
+  });
+
+  useEffect(() => {
+    if (hasNewFields) {
+      refreshFieldList.current();
+    }
+  }, [hasNewFields, refreshFieldList]);
 
   const editField = useMemo(
     () =>
@@ -309,7 +314,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
               fieldName,
               onSave: () => {
                 if (indexPatternInstance.isPersisted()) {
-                  refreshFieldList();
+                  refreshFieldList.current();
                   refetchFieldsExistenceInfo(indexPatternInstance.id);
                 } else {
                   indexPatternService.replaceDataViewId(indexPatternInstance);
@@ -341,7 +346,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
               fieldName,
               onDelete: () => {
                 if (indexPatternInstance.isPersisted()) {
-                  refreshFieldList();
+                  refreshFieldList.current();
                   refetchFieldsExistenceInfo(indexPatternInstance.id);
                 } else {
                   indexPatternService.replaceDataViewId(indexPatternInstance);
@@ -408,4 +413,16 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({
   );
 };
 
+function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) {
+  const metaKeys = dataView ? new Set(dataView.metaFields) : undefined;
+
+  return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => {
+    const field = new DataViewField(fieldSpec);
+    if (isFieldLensCompatible(field)) {
+      result.push(buildIndexPatternField(field, metaKeys));
+    }
+    return result;
+  }, []);
+}
+
 export const MemoizedDataPanel = memo(InnerFormBasedDataPanel);

x-pack/test/functional/apps/lens/group2/fields_list.ts

@@ -9,13 +9,15 @@ import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
-  const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']);
+  const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header', 'timePicker']);
   const find = getService('find');
   const log = getService('log');
   const testSubjects = getService('testSubjects');
   const filterBar = getService('filterBar');
   const fieldEditor = getService('fieldEditor');
   const retry = getService('retry');
+  const es = getService('es');
+  const queryBar = getService('queryBar');
 
   describe('lens fields list tests', () => {
     for (const datasourceType of ['form-based', 'ad-hoc', 'ad-hoc-no-timefield']) {
@@ -48,7 +50,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
             await PageObjects.header.waitUntilLoadingHasFinished();
           });
         });
-
         it('should show all fields as available', async () => {
           expect(
             await (await testSubjects.find('lnsIndexPatternAvailableFields-count')).getVisibleText()
@@ -231,5 +232,50 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         }
       });
     }
+
+    describe(`update field list test`, () => {
+      before(async () => {
+        await es.transport.request({
+          path: '/field-update-test/_doc',
+          method: 'POST',
+          body: {
+            '@timestamp': new Date().toISOString(),
+            oldField: 10,
+          },
+        });
+        await PageObjects.visualize.navigateToNewVisualization();
+        await PageObjects.visualize.clickVisType('lens');
+        await PageObjects.timePicker.setCommonlyUsedTime('This_week');
+
+        await PageObjects.lens.createAdHocDataView('field-update-test', true);
+        await retry.try(async () => {
+          const selectedPattern = await PageObjects.lens.getDataPanelIndexPattern();
+          expect(selectedPattern).to.eql('field-update-test*');
+        });
+      });
+      after(async () => {
+        await es.transport.request({
+          path: '/field-update-test',
+          method: 'DELETE',
+        });
+      });
+
+      it('should show new fields Available fields', async () => {
+        await es.transport.request({
+          path: '/field-update-test/_doc',
+          method: 'POST',
+          body: {
+            '@timestamp': new Date().toISOString(),
+            oldField: 10,
+            newField: 20,
+          },
+        });
+        await PageObjects.lens.waitForField('oldField');
+        await queryBar.setQuery('oldField: 10');
+        await queryBar.submitQuery();
+        await PageObjects.header.waitUntilLoadingHasFinished();
+        await PageObjects.lens.waitForField('newField');
+      });
+    });
   });
 }