[8.0][Discover] Only fetch field statistics for available fields (#125385)

* Resolve conflicts

* Revert unrelated ml change

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/discover/public/application/apps/main/components/layout/discover_layout.test.tsx

@@ -21,6 +21,7 @@ import { indexPatternWithTimefieldMock } from '../../../../../__mocks__/index_pa
 import { GetStateReturn } from '../../services/discover_state';
 import { DiscoverLayoutProps } from './types';
 import {
+  AvailableFields$,
   DataCharts$,
   DataDocuments$,
   DataMain$,
@@ -74,6 +75,11 @@ function getProps(indexPattern: IndexPattern, wasSidebarClosed?: boolean): Disco
     result: esHits as ElasticSearchHit[],
   }) as DataDocuments$;
 
+  const availableFields$ = new BehaviorSubject({
+    fetchStatus: FetchStatus.COMPLETE,
+    fields: [] as string[],
+  }) as AvailableFields$;
+
   const totalHits$ = new BehaviorSubject({
     fetchStatus: FetchStatus.COMPLETE,
     result: Number(esHits.length),
@@ -133,6 +139,7 @@ function getProps(indexPattern: IndexPattern, wasSidebarClosed?: boolean): Disco
     documents$,
     totalHits$,
     charts$,
+    availableFields$,
   };
 
   return {

src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx

@@ -44,7 +44,7 @@ import {
   useSavedSearchAliasMatchRedirect,
 } from '../../../../../saved_searches';
 import { VIEW_MODE } from '../view_mode_toggle';
-import { DataViewType } from '../../../../../../../data_views/common';
+import { DataViewType, DataView } from '../../../../../../../data_views/common';
 import {
   DOCUMENTS_VIEW_CLICK,
   FIELD_STATISTICS_VIEW_CLICK,
@@ -204,6 +204,14 @@ export function DiscoverLayout({
   }, [isSidebarClosed, storage]);
 
   const contentCentered = resultState === 'uninitialized' || resultState === 'none';
+  const onDataViewCreated = useCallback(
+    (ip: DataView) => {
+      if (ip.id) {
+        onChangeIndexPattern(ip.id);
+      }
+    },
+    [onChangeIndexPattern]
+  );
 
   return (
     <EuiPage className="dscPage" data-fetch-counter={fetchCounter.current}>
@@ -247,6 +255,8 @@ export function DiscoverLayout({
               useNewFieldsApi={useNewFieldsApi}
               onEditRuntimeField={onEditRuntimeField}
               viewMode={viewMode}
+              onDataViewCreated={onDataViewCreated}
+              availableFields$={savedSearchData$.availableFields$}
             />
           </EuiFlexItem>
           <EuiHideFor sizes={['xs', 's']}>
@@ -334,6 +344,7 @@ export function DiscoverLayout({
                     />
                   ) : (
                     <FieldStatisticsTableMemoized
+                      availableFields$={savedSearchData$.availableFields$}
                       savedSearch={savedSearch}
                       services={services}
                       indexPattern={indexPattern}

src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx

@@ -23,6 +23,9 @@ import { ElasticSearchHit } from '../../../../doc_views/doc_views_types';
 import { discoverServiceMock as mockDiscoverServices } from '../../../../../__mocks__/services';
 import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs';
 import { VIEW_MODE } from '../view_mode_toggle';
+import { BehaviorSubject } from 'rxjs';
+import { AvailableFields$ } from '../../services/use_saved_search';
+import { FetchStatus } from '../../../../types';
 
 jest.mock('../../../../../kibana_services', () => ({
   getServices: () => mockDiscoverServices,
@@ -49,6 +52,11 @@ function getCompProps(): DiscoverSidebarProps {
       fieldCounts[key] = (fieldCounts[key] || 0) + 1;
     }
   }
+  const availableFields$ = new BehaviorSubject({
+    fetchStatus: FetchStatus.COMPLETE,
+    fields: [] as string[],
+  }) as AvailableFields$;
+
   return {
     columns: ['extension'],
     fieldCounts,
@@ -67,6 +75,8 @@ function getCompProps(): DiscoverSidebarProps {
     onEditRuntimeField: jest.fn(),
     editField: jest.fn(),
     viewMode: VIEW_MODE.DOCUMENT_LEVEL,
+    onDataViewCreated: jest.fn(),
+    availableFields$,
   };
 }
 

src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.test.tsx

@@ -24,7 +24,7 @@ import {
 import { DiscoverServices } from '../../../../../build_services';
 import { ElasticSearchHit } from '../../../../doc_views/doc_views_types';
 import { FetchStatus } from '../../../../types';
-import { DataDocuments$ } from '../../services/use_saved_search';
+import { AvailableFields$, DataDocuments$ } from '../../services/use_saved_search';
 import { stubLogstashIndexPattern } from '../../../../../../../data/common/stubs';
 import { VIEW_MODE } from '../view_mode_toggle';
 
@@ -94,6 +94,10 @@ function getCompProps(): DiscoverSidebarResponsiveProps {
       fetchStatus: FetchStatus.COMPLETE,
       result: hits as ElasticSearchHit[],
     }) as DataDocuments$,
+    availableFields$: new BehaviorSubject({
+      fetchStatus: FetchStatus.COMPLETE,
+      fields: [] as string[],
+    }) as AvailableFields$,
     indexPatternList,
     onChangeIndexPattern: jest.fn(),
     onAddFilter: jest.fn(),
@@ -105,6 +109,7 @@ function getCompProps(): DiscoverSidebarResponsiveProps {
     trackUiMetric: jest.fn(),
     onEditRuntimeField: jest.fn(),
     viewMode: VIEW_MODE.DOCUMENT_LEVEL,
+    onDataViewCreated: jest.fn(),
   };
 }
 

src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar_responsive.tsx

@@ -35,9 +35,10 @@ import { DiscoverSidebar } from './discover_sidebar';
 import { DiscoverServices } from '../../../../../build_services';
 import { AppState } from '../../services/discover_state';
 import { DiscoverIndexPatternManagement } from './discover_index_pattern_management';
-import { DataDocuments$ } from '../../services/use_saved_search';
+import { AvailableFields$, DataDocuments$ } from '../../services/use_saved_search';
 import { calcFieldCounts } from '../../utils/calc_field_counts';
 import { VIEW_MODE } from '../view_mode_toggle';
+import { FetchStatus } from '../../../../types';
 
 export interface DiscoverSidebarResponsiveProps {
   /**
@@ -107,10 +108,18 @@ export interface DiscoverSidebarResponsiveProps {
    * callback to execute on edit runtime field
    */
   onEditRuntimeField: () => void;
+  /**
+   * callback to execute on create dataview
+   */
+  onDataViewCreated: (dataView: IndexPattern) => void;
   /**
    * Discover view mode
    */
   viewMode: VIEW_MODE;
+  /**
+   * list of available fields fetched from ES
+   */
+  availableFields$: AvailableFields$;
 }
 
 /**
@@ -181,6 +190,32 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
   }, []);
 
   const { indexPatternFieldEditor } = props.services;
+  const { availableFields$ } = props;
+
+  useEffect(
+    () => {
+      // For an external embeddable like the Field stats
+      // it is useful to know what fields are populated in the docs fetched
+      // or what fields are selected by the user
+
+      const fieldCnts = fieldCounts.current ?? {};
+
+      const availableFields = props.columns.length > 0 ? props.columns : Object.keys(fieldCnts);
+      availableFields$.next({
+        fetchStatus: FetchStatus.COMPLETE,
+        fields: availableFields,
+      });
+    },
+    // Using columns.length here instead of columns to avoid array reference changing
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [
+      selectedIndexPattern,
+      availableFields$,
+      fieldCounts.current,
+      documentState.result,
+      props.columns.length,
+    ]
+  );
 
   const editField = useCallback(
     (fieldName?: string) => {

src/plugins/discover/public/application/apps/main/services/use_saved_search.ts

@@ -29,6 +29,7 @@ export interface SavedSearchData {
   documents$: DataDocuments$;
   totalHits$: DataTotalHits$;
   charts$: DataCharts$;
+  availableFields$: AvailableFields$;
 }
 
 export interface TimechartBucketInterval {
@@ -41,6 +42,7 @@ export type DataMain$ = BehaviorSubject<DataMainMsg>;
 export type DataDocuments$ = BehaviorSubject<DataDocumentsMsg>;
 export type DataTotalHits$ = BehaviorSubject<DataTotalHitsMsg>;
 export type DataCharts$ = BehaviorSubject<DataChartsMessage>;
+export type AvailableFields$ = BehaviorSubject<DataAvailableFieldsMsg>;
 
 export type DataRefetch$ = Subject<DataRefetchMsg>;
 
@@ -77,6 +79,10 @@ export interface DataChartsMessage extends DataMsg {
   chartData?: Chart;
 }
 
+export interface DataAvailableFieldsMsg extends DataMsg {
+  fields?: string[];
+}
+
 /**
  * This hook return 2 observables, refetch$ allows to trigger data fetching, data$ to subscribe
  * to the data fetching
@@ -113,14 +119,19 @@ export const useSavedSearch = ({
 
   const charts$: DataCharts$ = useBehaviorSubject({ fetchStatus: initialFetchStatus });
 
+  const availableFields$: AvailableFields$ = useBehaviorSubject({
+    fetchStatus: initialFetchStatus,
+  });
+
   const dataSubjects = useMemo(() => {
     return {
       main$,
       documents$,
       totalHits$,
       charts$,
+      availableFields$,
     };
-  }, [main$, charts$, documents$, totalHits$]);
+  }, [main$, charts$, documents$, totalHits$, availableFields$]);
 
   /**
    * The observable to trigger data fetching in UI

src/plugins/discover/public/application/apps/main/utils/fetch_all.test.ts

@@ -1,59 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-import { FetchStatus } from '../../../types';
-import { BehaviorSubject } from 'rxjs';
-import { RequestAdapter } from '../../../../../../inspector';
-import { savedSearchMock } from '../../../../__mocks__/saved_search';
-import { ReduxLikeStateContainer } from '../../../../../../kibana_utils/common';
-import { AppState } from '../services/discover_state';
-import { discoverServiceMock } from '../../../../__mocks__/services';
-import { fetchAll } from './fetch_all';
-
-describe('test fetchAll', () => {
-  test('changes of fetchStatus when starting with FetchStatus.UNINITIALIZED', async (done) => {
-    const subjects = {
-      main$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
-      documents$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
-      totalHits$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
-      charts$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
-    };
-    const deps = {
-      appStateContainer: {
-        getState: () => {
-          return { interval: 'auto' };
-        },
-      } as ReduxLikeStateContainer<AppState>,
-      abortController: new AbortController(),
-      data: discoverServiceMock.data,
-      inspectorAdapters: { requests: new RequestAdapter() },
-      onResults: jest.fn(),
-      searchSessionId: '123',
-      initialFetchStatus: FetchStatus.UNINITIALIZED,
-      useNewFieldsApi: true,
-      services: discoverServiceMock,
-    };
-
-    const stateArr: FetchStatus[] = [];
-
-    subjects.main$.subscribe((value) => stateArr.push(value.fetchStatus));
-
-    const parentSearchSource = savedSearchMock.searchSource;
-    const childSearchSource = parentSearchSource.createChild();
-
-    fetchAll(subjects, childSearchSource, false, deps).subscribe({
-      complete: () => {
-        expect(stateArr).toEqual([
-          FetchStatus.UNINITIALIZED,
-          FetchStatus.LOADING,
-          FetchStatus.COMPLETE,
-        ]);
-        done();
-      },
-    });
-  });
-});

src/plugins/discover/public/application/apps/main/utils/fetch_chart.test.ts

@@ -15,6 +15,7 @@ import { AppState } from '../services/discover_state';
 import { discoverServiceMock } from '../../../../__mocks__/services';
 import { calculateBounds, IKibanaSearchResponse } from '../../../../../../data/common';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { AvailableFields$ } from '../services/use_saved_search';
 
 function getDataSubjects() {
   return {
@@ -22,6 +23,9 @@ function getDataSubjects() {
     documents$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     totalHits$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     charts$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
+    availableFields$: new BehaviorSubject({
+      fetchStatus: FetchStatus.UNINITIALIZED,
+    }) as AvailableFields$,
   };
 }
 

src/plugins/discover/public/application/apps/main/utils/fetch_documents.test.ts

@@ -11,6 +11,7 @@ import { BehaviorSubject, throwError as throwErrorRx } from 'rxjs';
 import { RequestAdapter } from '../../../../../../inspector';
 import { savedSearchMock } from '../../../../__mocks__/saved_search';
 import { discoverServiceMock } from '../../../../__mocks__/services';
+import { AvailableFields$ } from '../services/use_saved_search';
 
 function getDataSubjects() {
   return {
@@ -18,6 +19,9 @@ function getDataSubjects() {
     documents$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     totalHits$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     charts$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
+    availableFields$: new BehaviorSubject({
+      fetchStatus: FetchStatus.UNINITIALIZED,
+    }) as AvailableFields$,
   };
 }
 

src/plugins/discover/public/application/apps/main/utils/fetch_total_hits.test.ts

@@ -11,6 +11,7 @@ import { RequestAdapter } from '../../../../../../inspector';
 import { savedSearchMock } from '../../../../__mocks__/saved_search';
 import { fetchTotalHits } from './fetch_total_hits';
 import { discoverServiceMock } from '../../../../__mocks__/services';
+import { AvailableFields$ } from '../services/use_saved_search';
 
 function getDataSubjects() {
   return {
@@ -18,6 +19,9 @@ function getDataSubjects() {
     documents$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     totalHits$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
     charts$: new BehaviorSubject({ fetchStatus: FetchStatus.UNINITIALIZED }),
+    availableFields$: new BehaviorSubject({
+      fetchStatus: FetchStatus.UNINITIALIZED,
+    }) as AvailableFields$,
   };
 }
 

src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx

@@ -21,7 +21,7 @@ import {
 import { FIELD_STATISTICS_LOADED } from './constants';
 import { SavedSearch } from '../../../saved_searches';
 import { GetStateReturn } from '../../apps/main/services/discover_state';
-import { DataRefetch$ } from '../../apps/main/services/use_saved_search';
+import { AvailableFields$, DataRefetch$ } from '../../apps/main/services/use_saved_search';
 
 export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   indexPattern: IndexPattern;
@@ -34,6 +34,8 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
    * Callback to add a filter to filter bar
    */
   onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void;
+  sessionId?: string;
+  fieldsToFetch?: string[];
 }
 export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput {
   showDistributions?: boolean;
@@ -87,11 +89,14 @@ export interface FieldStatisticsTableProps {
    */
   trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void;
   savedSearchRefetch$?: DataRefetch$;
+  availableFields$?: AvailableFields$;
+  searchSessionId?: string;
 }
 
 export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
   const {
     services,
+    availableFields$,
     indexPattern,
     savedSearch,
     query,
@@ -101,8 +106,8 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
     onAddFilter,
     trackUiMetric,
     savedSearchRefetch$,
+    searchSessionId,
   } = props;
-  const { uiSettings } = services;
   const [embeddable, setEmbeddable] = useState<
     | ErrorEmbeddable
     | IEmbeddable<DataVisualizerGridEmbeddableInput, DataVisualizerGridEmbeddableOutput>
@@ -128,11 +133,19 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
         embeddable.updateInput({ lastReloadRequestTime: Date.now() });
       }
     });
+
+    const fields = availableFields$?.subscribe(() => {
+      if (embeddable && !isErrorEmbeddable(embeddable) && !availableFields$?.getValue().error) {
+        embeddable.updateInput({ fieldsToFetch: availableFields$?.getValue().fields });
+      }
+    });
+
     return () => {
       sub?.unsubscribe();
       refetch?.unsubscribe();
+      fields?.unsubscribe();
     };
-  }, [embeddable, stateContainer, savedSearchRefetch$]);
+  }, [embeddable, stateContainer, savedSearchRefetch$, availableFields$]);
 
   useEffect(() => {
     if (embeddable && !isErrorEmbeddable(embeddable)) {
@@ -144,10 +157,22 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
         filters,
         visibleFieldNames: columns,
         onAddFilter,
+        sessionId: searchSessionId,
+        fieldsToFetch: availableFields$?.getValue().fields,
       });
       embeddable.reload();
     }
-  }, [embeddable, indexPattern, savedSearch, query, columns, filters, onAddFilter]);
+  }, [
+    embeddable,
+    indexPattern,
+    savedSearch,
+    query,
+    columns,
+    filters,
+    onAddFilter,
+    searchSessionId,
+    availableFields$,
+  ]);
 
   useEffect(() => {
     if (showPreviewByDefault && embeddable && !isErrorEmbeddable(embeddable)) {
@@ -158,7 +183,7 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
 
       embeddable.reload();
     }
-  }, [showPreviewByDefault, uiSettings, embeddable]);
+  }, [showPreviewByDefault, embeddable]);
 
   useEffect(() => {
     let unmounted = false;
@@ -203,7 +228,7 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
       // Clean up embeddable upon unmounting
       embeddable?.destroy();
     };
-  }, [embeddable, embeddableRoot, uiSettings, trackUiMetric]);
+  }, [embeddable, embeddableRoot, trackUiMetric]);
 
   return (
     <div