[Discover] Refactor text based query language state transformation (#140169)

* Extract useTextBasedQueryLanguage hook

* Add unit tests

* Add comments

* Refactor data view list handling

src/plugins/discover/public/__mocks__/data_views.ts

@@ -8,6 +8,8 @@
 
 import { DataViewsContract } from '@kbn/data-views-plugin/public';
 import { dataViewMock } from './data_view';
+import { dataViewComplexMock } from './data_view_complex';
+import { dataViewWithTimefieldMock } from './data_view_with_timefield';
 
 export const dataViewsMock = {
   getCache: async () => {
@@ -21,4 +23,7 @@ export const dataViewsMock = {
     }
   },
   updateSavedObject: jest.fn(),
+  getIdsWithTitle: jest.fn(() => {
+    return Promise.resolve([dataViewMock, dataViewComplexMock, dataViewWithTimefieldMock]);
+  }),
 } as unknown as jest.Mocked<DataViewsContract>;

src/plugins/discover/public/__mocks__/services.ts

@@ -25,6 +25,7 @@ import { TopNavMenu } from '@kbn/navigation-plugin/public';
 import { FORMATS_UI_SETTINGS } from '@kbn/field-formats-plugin/common';
 import { LocalStorageMock } from './local_storage_mock';
 import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks';
+import { dataViewsMock } from './data_views';
 const dataPlugin = dataPluginMock.createStartContract();
 const expressionsPlugin = expressionsPluginMock.createStartContract();
 
@@ -114,4 +115,5 @@ export const discoverServiceMock = {
   },
   expressions: expressionsPlugin,
   savedObjectsTagging: {},
+  dataViews: dataViewsMock,
 } as unknown as DiscoverServices;

src/plugins/discover/public/application/main/components/layout/discover_layout.test.tsx

@@ -16,8 +16,7 @@ import { esHits } from '../../../../__mocks__/es_hits';
 import { dataViewMock } from '../../../../__mocks__/data_view';
 import { savedSearchMock } from '../../../../__mocks__/saved_search';
 import { createSearchSourceMock } from '@kbn/data-plugin/common/search/search_source/mocks';
-import type { DataView, DataViewAttributes } from '@kbn/data-views-plugin/public';
-import { SavedObject } from '@kbn/core/types';
+import type { DataView } from '@kbn/data-views-plugin/public';
 import { dataViewWithTimefieldMock } from '../../../../__mocks__/data_view_with_timefield';
 import { GetStateReturn } from '../../services/discover_state';
 import { DiscoverLayoutProps } from './types';
@@ -59,9 +58,7 @@ function mountComponent(
     return { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' };
   };
 
-  const dataViewList = [dataView].map((ip) => {
-    return { ...ip, ...{ attributes: { title: ip.title } } };
-  }) as unknown as Array<SavedObject<DataViewAttributes>>;
+  const dataViewList = [dataView];
 
   const main$ = new BehaviorSubject({
     fetchStatus: FetchStatus.COMPLETE,

src/plugins/discover/public/application/main/components/layout/types.ts

@@ -7,9 +7,8 @@
  */
 
 import type { Query, TimeRange, AggregateQuery } from '@kbn/es-query';
-import type { SavedObject } from '@kbn/data-plugin/public';
-import type { DataView, DataViewAttributes } from '@kbn/data-views-plugin/public';
-import { ISearchSource } from '@kbn/data-plugin/public';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { DataViewListItem, ISearchSource } from '@kbn/data-plugin/public';
 import { RequestAdapter } from '@kbn/inspector-plugin/common';
 import { SavedSearch } from '@kbn/saved-search-plugin/public';
 import { DataTableRecord } from '../../../../types';
@@ -18,7 +17,7 @@ import { DataRefetch$, SavedSearchData } from '../../hooks/use_saved_search';
 
 export interface DiscoverLayoutProps {
   dataView: DataView;
-  dataViewList: Array<SavedObject<DataViewAttributes>>;
+  dataViewList: DataViewListItem[];
   inspectorAdapters: { requests: RequestAdapter };
   navigateTo: (url: string) => void;
   onChangeDataView: (id: string) => void;

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx

@@ -12,8 +12,8 @@ import { getDataTableRecords } from '../../../../__fixtures__/real_hits';
 import { mountWithIntl } from '@kbn/test-jest-helpers';
 import React from 'react';
 import { DiscoverSidebarProps } from './discover_sidebar';
-import { DataViewAttributes } from '@kbn/data-views-plugin/public';
-import { SavedObject } from '@kbn/core/types';
+import { DataViewListItem } from '@kbn/data-views-plugin/public';
+
 import { getDefaultFieldFilter } from './lib/field_filter';
 import { DiscoverSidebarComponent as DiscoverSidebar } from './discover_sidebar';
 import { discoverServiceMock as mockDiscoverServices } from '../../../../__mocks__/services';
@@ -39,9 +39,9 @@ function getCompProps(): DiscoverSidebarProps {
   const hits = getDataTableRecords(dataView);
 
   const dataViewList = [
-    { id: '0', attributes: { title: 'b' } } as SavedObject<DataViewAttributes>,
-    { id: '1', attributes: { title: 'a' } } as SavedObject<DataViewAttributes>,
-    { id: '2', attributes: { title: 'c' } } as SavedObject<DataViewAttributes>,
+    { id: '0', title: 'b' } as DataViewListItem,
+    { id: '1', title: 'a' } as DataViewListItem,
+    { id: '2', title: 'c' } as DataViewListItem,
   ];
 
   const fieldCounts: Record<string, number> = {};

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx

@@ -13,8 +13,7 @@ import { getDataTableRecords } from '../../../../__fixtures__/real_hits';
 import { act } from 'react-dom/test-utils';
 import { mountWithIntl } from '@kbn/test-jest-helpers';
 import React from 'react';
-import { DataViewAttributes } from '@kbn/data-views-plugin/public';
-import { SavedObject } from '@kbn/core/types';
+import { DataViewListItem } from '@kbn/data-views-plugin/public';
 import {
   DiscoverSidebarResponsive,
   DiscoverSidebarResponsiveProps,
@@ -78,9 +77,9 @@ function getCompProps(): DiscoverSidebarResponsiveProps {
   const hits = getDataTableRecords(dataView);
 
   const dataViewList = [
-    { id: '0', attributes: { title: 'b' } } as SavedObject<DataViewAttributes>,
-    { id: '1', attributes: { title: 'a' } } as SavedObject<DataViewAttributes>,
-    { id: '2', attributes: { title: 'c' } } as SavedObject<DataViewAttributes>,
+    { id: '0', title: 'b' } as DataViewListItem,
+    { id: '1', title: 'a' } as DataViewListItem,
+    { id: '2', title: 'c' } as DataViewListItem,
   ];
 
   for (const hit of hits) {

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx

@@ -22,8 +22,7 @@ import {
   EuiShowFor,
   EuiTitle,
 } from '@elastic/eui';
-import type { DataView, DataViewAttributes, DataViewField } from '@kbn/data-views-plugin/public';
-import { SavedObject } from '@kbn/core/types';
+import type { DataView, DataViewField, DataViewListItem } from '@kbn/data-views-plugin/public';
 import { useDiscoverServices } from '../../../../hooks/use_discover_services';
 import { getDefaultFieldFilter } from './lib/field_filter';
 import { DiscoverSidebar } from './discover_sidebar';
@@ -51,7 +50,7 @@ export interface DiscoverSidebarResponsiveProps {
   /**
    * List of available data views
    */
-  dataViewList: Array<SavedObject<DataViewAttributes>>;
+  dataViewList: DataViewListItem[];
   /**
    * Has been toggled closed
    */

src/plugins/discover/public/application/main/discover_main_app.test.tsx

@@ -7,12 +7,11 @@
  */
 import React from 'react';
 import { mountWithIntl } from '@kbn/test-jest-helpers';
+import { DataViewListItem } from '@kbn/data-views-plugin/public';
 import { dataViewMock } from '../../__mocks__/data_view';
 import { DiscoverMainApp } from './discover_main_app';
 import { DiscoverTopNav } from './components/top_nav/discover_topnav';
 import { savedSearchMock } from '../../__mocks__/saved_search';
-import { SavedObject } from '@kbn/core/types';
-import type { DataViewAttributes } from '@kbn/data-views-plugin/public';
 import { setHeaderActionMenuMounter } from '../../kibana_services';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { discoverServiceMock } from '../../__mocks__/services';
@@ -25,7 +24,7 @@ describe('DiscoverMainApp', () => {
   test('renders', () => {
     const dataViewList = [dataViewMock].map((ip) => {
       return { ...ip, ...{ attributes: { title: ip.title } } };
-    }) as unknown as Array<SavedObject<DataViewAttributes>>;
+    }) as unknown as DataViewListItem[];
     const props = {
       dataViewList,
       savedSearch: savedSearchMock,

src/plugins/discover/public/application/main/discover_main_app.tsx

@@ -7,9 +7,8 @@
  */
 import React, { useCallback, useEffect, useState } from 'react';
 import { useHistory } from 'react-router-dom';
-import type { DataViewAttributes } from '@kbn/data-views-plugin/public';
-import type { SavedObject } from '@kbn/data-plugin/public';
 import { SavedSearch } from '@kbn/saved-search-plugin/public';
+import { DataViewListItem } from '@kbn/data-views-plugin/public';
 import { DiscoverLayout } from './components/layout';
 import { setBreadcrumbsTitle } from '../../utils/breadcrumbs';
 import { addHelpMenuToAppChrome } from '../../components/help_menu/help_menu_util';
@@ -25,7 +24,7 @@ export interface DiscoverMainProps {
   /**
    * List of available data views
    */
-  dataViewList: Array<SavedObject<DataViewAttributes>>;
+  dataViewList: DataViewListItem[];
   /**
    * Current instance of SavedSearch
    */
@@ -35,7 +34,7 @@ export interface DiscoverMainProps {
 export function DiscoverMainApp(props: DiscoverMainProps) {
   const { savedSearch, dataViewList } = props;
   const services = useDiscoverServices();
-  const { chrome, docLinks, uiSettings: config, data, spaces, history } = services;
+  const { chrome, docLinks, data, spaces, history } = services;
   const usedHistory = useHistory();
   const [expandedDoc, setExpandedDoc] = useState<DataTableRecord | undefined>(undefined);
   const navigateTo = useCallback(
@@ -64,6 +63,7 @@ export function DiscoverMainApp(props: DiscoverMainProps) {
     history: usedHistory,
     savedSearch,
     setExpandedDoc,
+    dataViewList,
   });
 
   /**
@@ -81,7 +81,7 @@ export function DiscoverMainApp(props: DiscoverMainProps) {
     return () => {
       data.search.session.clear();
     };
-  }, [savedSearch, chrome, docLinks, refetch$, stateContainer, data, config]);
+  }, [savedSearch, chrome, data]);
 
   /**
    * Initializing syncing with state and help menu

src/plugins/discover/public/application/main/discover_main_route.tsx

@@ -7,12 +7,9 @@
  */
 import React, { useEffect, useState, memo, useCallback } from 'react';
 import { useParams, useHistory } from 'react-router-dom';
-import { SavedObject } from '@kbn/data-plugin/public';
+import { DataViewListItem } from '@kbn/data-plugin/public';
 import { ISearchSource } from '@kbn/data-plugin/public';
-import {
-  DataViewAttributes,
-  DataViewSavedObjectConflictError,
-} from '@kbn/data-views-plugin/public';
+import { DataViewSavedObjectConflictError } from '@kbn/data-views-plugin/public';
 import { redirectWhenMissing } from '@kbn/kibana-utils-plugin/public';
 import { useExecutionContext } from '@kbn/kibana-react-plugin/public';
 import {
@@ -60,7 +57,7 @@ export function DiscoverMainRoute(props: Props) {
   const [error, setError] = useState<Error>();
   const [savedSearch, setSavedSearch] = useState<SavedSearch>();
   const dataView = savedSearch?.searchSource?.getField('index');
-  const [dataViewList, setDataViewList] = useState<Array<SavedObject<DataViewAttributes>>>([]);
+  const [dataViewList, setDataViewList] = useState<DataViewListItem[]>([]);
   const [hasESData, setHasESData] = useState(false);
   const [hasUserDataView, setHasUserDataView] = useState(false);
   const [showNoDataPage, setShowNoDataPage] = useState<boolean>(false);
@@ -99,7 +96,7 @@ export function DiscoverMainRoute(props: Props) {
         const { index } = appStateContainer.getState();
         const ip = await loadDataView(index || '', data.dataViews, config);
 
-        const ipList = ip.list as Array<SavedObject<DataViewAttributes>>;
+        const ipList = ip.list;
         const dataViewData = resolveDataView(ip, searchSource, toastNotifications);
         await data.dataViews.refreshFields(dataViewData);
         setDataViewList(ipList);

src/plugins/discover/public/application/main/hooks/use_discover_state.test.ts

@@ -7,28 +7,14 @@
  */
 
 import { renderHook } from '@testing-library/react-hooks';
+import { DataViewListItem, SearchSource } from '@kbn/data-plugin/public';
 import { createSearchSessionMock } from '../../../__mocks__/search_session';
 import { discoverServiceMock } from '../../../__mocks__/services';
 import { savedSearchMock } from '../../../__mocks__/saved_search';
 import { useDiscoverState } from './use_discover_state';
 import { dataViewMock } from '../../../__mocks__/data_view';
-import { SearchSource } from '@kbn/data-plugin/public';
 
 describe('test useDiscoverState', () => {
-  const originalSavedObjectsClient = discoverServiceMock.core.savedObjects.client;
-
-  beforeAll(() => {
-    discoverServiceMock.core.savedObjects.client.resolve = jest.fn().mockReturnValue({
-      saved_object: {
-        attributes: {},
-      },
-    });
-  });
-
-  afterAll(() => {
-    discoverServiceMock.core.savedObjects.client = originalSavedObjectsClient;
-  });
-
   test('return is valid', async () => {
     const { history } = createSearchSessionMock();
 
@@ -38,6 +24,7 @@ describe('test useDiscoverState', () => {
         history,
         savedSearch: savedSearchMock,
         setExpandedDoc: jest.fn(),
+        dataViewList: [dataViewMock as DataViewListItem],
       });
     });
     expect(result.current.state.index).toBe(dataViewMock.id);

src/plugins/discover/public/application/main/hooks/use_discover_state.ts

@@ -6,23 +6,17 @@
  * Side Public License, v 1.
  */
 import { useMemo, useEffect, useState, useCallback } from 'react';
-import usePrevious from 'react-use/lib/usePrevious';
 import { isEqual } from 'lodash';
 import { History } from 'history';
-import { DataViewType } from '@kbn/data-views-plugin/public';
-import {
-  isOfAggregateQueryType,
-  getIndexPatternFromSQLQuery,
-  AggregateQuery,
-  Query,
-} from '@kbn/es-query';
+import { DataViewListItem, DataViewType } from '@kbn/data-views-plugin/public';
 import { SavedSearch, getSavedSearch } from '@kbn/saved-search-plugin/public';
 import type { SortOrder } from '@kbn/saved-search-plugin/public';
+import { useTextBasedQueryLanguage } from './use_text_based_query_language';
 import { getState } from '../services/discover_state';
 import { getStateDefaults } from '../utils/get_state_defaults';
 import { DiscoverServices } from '../../../build_services';
 import { loadDataView } from '../utils/resolve_data_view';
-import { useSavedSearch as useSavedSearchData, DataDocumentsMsg } from './use_saved_search';
+import { useSavedSearch as useSavedSearchData } from './use_saved_search';
 import {
   MODIFY_COLUMNS_ON_SWITCH,
   SEARCH_FIELDS_FROM_SOURCE,
@@ -30,27 +24,26 @@ import {
   SORT_DEFAULT_ORDER_SETTING,
 } from '../../../../common';
 import { useSearchSession } from './use_search_session';
-import { useDataState } from './use_data_state';
 import { FetchStatus } from '../../types';
 import { getDataViewAppState } from '../utils/get_switch_data_view_app_state';
 import { DataTableRecord } from '../../../types';
 import { restoreStateFromSavedSearch } from '../../../services/saved_searches/restore_from_saved_search';
 
-const MAX_NUM_OF_COLUMNS = 50;
-
 export function useDiscoverState({
   services,
   history,
   savedSearch,
   setExpandedDoc,
+  dataViewList,
 }: {
   services: DiscoverServices;
   savedSearch: SavedSearch;
   history: History;
   setExpandedDoc: (doc?: DataTableRecord) => void;
+  dataViewList: DataViewListItem[];
 }) {
-  const { uiSettings: config, data, filterManager, dataViews, storage } = services;
-  const useNewFieldsApi = useMemo(() => !config.get(SEARCH_FIELDS_FROM_SOURCE), [config]);
+  const { uiSettings, data, filterManager, dataViews, storage } = services;
+  const useNewFieldsApi = useMemo(() => !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE), [uiSettings]);
   const { timefilter } = data.query.timefilter;
 
   const dataView = savedSearch.searchSource.getField('index')!;
@@ -65,25 +58,22 @@ export function useDiscoverState({
       getState({
         getStateDefaults: () =>
           getStateDefaults({
-            config,
+            config: uiSettings,
             data,
             savedSearch,
             storage,
           }),
-        storeInSessionStorage: config.get('state:storeInSessionStorage'),
+        storeInSessionStorage: uiSettings.get('state:storeInSessionStorage'),
         history,
         toasts: services.core.notifications.toasts,
-        uiSettings: config,
+        uiSettings,
       }),
-    [config, data, history, savedSearch, services.core.notifications.toasts, storage]
+    [uiSettings, data, history, savedSearch, services.core.notifications.toasts, storage]
   );
 
   const { appStateContainer } = stateContainer;
 
   const [state, setState] = useState(appStateContainer.getState());
-  const [documentStateCols, setDocumentStateCols] = useState<string[]>([]);
-  const [sqlQuery] = useState<AggregateQuery | Query | undefined>(state.query);
-  const prevQuery = usePrevious(state.query);
 
   /**
    * Search session logic
@@ -94,12 +84,12 @@ export function useDiscoverState({
     // A saved search is created on every page load, so we check the ID to see if we're loading a
     // previously saved search or if it is just transient
     const shouldSearchOnPageLoad =
-      config.get<boolean>(SEARCH_ON_PAGE_LOAD_SETTING) ||
+      uiSettings.get<boolean>(SEARCH_ON_PAGE_LOAD_SETTING) ||
       savedSearch.id !== undefined ||
       timefilter.getRefreshInterval().pause === false ||
       searchSessionManager.hasSearchSessionIdInURL();
     return shouldSearchOnPageLoad ? FetchStatus.LOADING : FetchStatus.UNINITIALIZED;
-  }, [config, savedSearch.id, searchSessionManager, timefilter]);
+  }, [uiSettings, savedSearch.id, searchSessionManager, timefilter]);
 
   /**
    * Data fetching logic
@@ -113,8 +103,16 @@ export function useDiscoverState({
     stateContainer,
     useNewFieldsApi,
   });
-
-  const documentState: DataDocumentsMsg = useDataState(data$.documents$);
+  /**
+   * State changes (data view, columns), when a text base query result is returned
+   */
+  useTextBasedQueryLanguage({
+    documents$: data$.documents$,
+    dataViews,
+    stateContainer,
+    dataViewList,
+    savedSearch,
+  });
 
   /**
    * Reset to display loading spinner when savedSearch is changing
@@ -151,7 +149,11 @@ export function useDiscoverState({
          *  That's because appState is updated before savedSearchData$
          *  The following line of code catches this, but should be improved
          */
-        const nextDataView = await loadDataView(nextState.index, dataViews, config);
+        const nextDataView = await loadDataView(
+          nextState.index,
+          services.dataViews,
+          services.uiSettings
+        );
         savedSearch.searchSource.setField('index', nextDataView.loaded);
 
         reset();
@@ -163,17 +165,7 @@ export function useDiscoverState({
       setState(nextState);
     });
     return () => unsubscribe();
-  }, [
-    config,
-    dataViews,
-    appStateContainer,
-    setState,
-    state,
-    refetch$,
-    data$,
-    reset,
-    savedSearch.searchSource,
-  ]);
+  }, [services, appStateContainer, state, refetch$, data$, reset, savedSearch.searchSource]);
 
   /**
    * function to revert any changes to a given saved search
@@ -190,7 +182,7 @@ export function useDiscoverState({
       const newDataView = newSavedSearch.searchSource.getField('index') || dataView;
       newSavedSearch.searchSource.setField('index', newDataView);
       const newAppState = getStateDefaults({
-        config,
+        config: uiSettings,
         data,
         savedSearch: newSavedSearch,
         storage,
@@ -204,7 +196,7 @@ export function useDiscoverState({
       await stateContainer.replaceUrlAppState(newAppState);
       setState(newAppState);
     },
-    [services, dataView, config, data, storage, stateContainer]
+    [services, dataView, uiSettings, data, storage, stateContainer]
   );
 
   /**
@@ -219,8 +211,8 @@ export function useDiscoverState({
           nextDataView,
           state.columns || [],
           (state.sort || []) as SortOrder[],
-          config.get(MODIFY_COLUMNS_ON_SWITCH),
-          config.get(SORT_DEFAULT_ORDER_SETTING),
+          uiSettings.get(MODIFY_COLUMNS_ON_SWITCH),
+          uiSettings.get(SORT_DEFAULT_ORDER_SETTING),
           state.query
         );
         stateContainer.setAppState(nextAppState);
@@ -228,7 +220,7 @@ export function useDiscoverState({
       setExpandedDoc(undefined);
     },
     [
-      config,
+      uiSettings,
       dataView,
       dataViews,
       setExpandedDoc,
@@ -254,12 +246,6 @@ export function useDiscoverState({
   /**
    * Trigger data fetching on dataView or savedSearch changes
    */
-  useEffect(() => {
-    if (!isEqual(state.query, prevQuery)) {
-      setDocumentStateCols([]);
-    }
-  }, [state.query, prevQuery]);
-
   useEffect(() => {
     if (dataView) {
       refetch$.next(undefined);
@@ -276,41 +262,6 @@ export function useDiscoverState({
     }
   }, [dataView, stateContainer]);
 
-  const getResultColumns = useCallback(() => {
-    if (documentState.result?.length && documentState.fetchStatus === FetchStatus.COMPLETE) {
-      const firstRow = documentState.result[0];
-      const columns = Object.keys(firstRow.raw).slice(0, MAX_NUM_OF_COLUMNS);
-      if (!isEqual(columns, documentStateCols) && !isEqual(state.query, sqlQuery)) {
-        return columns;
-      }
-      return [];
-    }
-    return [];
-  }, [documentState, documentStateCols, sqlQuery, state.query]);
-
-  useEffect(() => {
-    async function fetchDataview() {
-      if (state.query && isOfAggregateQueryType(state.query) && 'sql' in state.query) {
-        const indexPatternFromQuery = getIndexPatternFromSQLQuery(state.query.sql);
-        const idsTitles = await dataViews.getIdsWithTitle();
-        const dataViewObj = idsTitles.find(({ title }) => title === indexPatternFromQuery);
-        if (dataViewObj) {
-          const columns = getResultColumns();
-          if (columns.length) {
-            setDocumentStateCols(columns);
-          }
-          const nextState = {
-            index: dataViewObj.id,
-            ...(columns.length && { columns }),
-          };
-          stateContainer.replaceUrlAppState(nextState);
-        }
-      }
-    }
-    fetchDataview();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [config, documentState, dataViews]);
-
   return {
     data$,
     dataView,

src/plugins/discover/public/application/main/hooks/use_saved_search.test.ts

@@ -15,6 +15,8 @@ import { getState, AppState } from '../services/discover_state';
 import { uiSettingsMock } from '../../../__mocks__/ui_settings';
 import { useDiscoverState } from './use_discover_state';
 import { FetchStatus } from '../../types';
+import { dataViewMock } from '../../../__mocks__/data_view';
+import { DataViewListItem } from '@kbn/data-views-plugin/common';
 
 describe('test useSavedSearch', () => {
   test('useSavedSearch return is valid', async () => {
@@ -61,6 +63,7 @@ describe('test useSavedSearch', () => {
         history,
         savedSearch: savedSearchMock,
         setExpandedDoc: jest.fn(),
+        dataViewList: [dataViewMock as DataViewListItem],
       });
     });
 
@@ -104,6 +107,7 @@ describe('test useSavedSearch', () => {
         history,
         savedSearch: savedSearchMock,
         setExpandedDoc: jest.fn(),
+        dataViewList: [dataViewMock as DataViewListItem],
       });
     });
 

src/plugins/discover/public/application/main/hooks/use_saved_search.ts

@@ -11,6 +11,7 @@ import type { AutoRefreshDoneFn } from '@kbn/data-plugin/public';
 import { ISearchSource } from '@kbn/data-plugin/public';
 import { RequestAdapter } from '@kbn/inspector-plugin/public';
 import { SavedSearch } from '@kbn/saved-search-plugin/public';
+import { AggregateQuery, Query } from '@kbn/es-query';
 import { getRawRecordType } from '../utils/get_raw_record_type';
 import { DiscoverServices } from '../../../build_services';
 import { DiscoverSearchSessionManager } from '../services/discover_search_session';
@@ -71,6 +72,7 @@ export interface DataMsg {
   fetchStatus: FetchStatus;
   error?: Error;
   recordRawType?: RecordRawType;
+  query?: AggregateQuery | Query | undefined;
 }
 
 export interface DataMainMsg extends DataMsg {

src/plugins/discover/public/application/main/hooks/use_saved_search_messages.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { AggregateQuery, Query } from '@kbn/es-query';
 import { FetchStatus } from '../../types';
 import {
   DataCharts$,
@@ -61,12 +62,14 @@ export function sendPartialMsg(main$: DataMain$) {
  */
 export function sendLoadingMsg(
   data$: DataMain$ | DataDocuments$ | DataTotalHits$ | DataCharts$,
-  recordRawType: RecordRawType
+  recordRawType: RecordRawType,
+  query?: AggregateQuery | Query
 ) {
   if (data$.getValue().fetchStatus !== FetchStatus.LOADING) {
     data$.next({
       fetchStatus: FetchStatus.LOADING,
       recordRawType,
+      query,
     });
   }
 }

src/plugins/discover/public/application/main/hooks/use_test_based_query_language.test.ts

@@ -0,0 +1,249 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { renderHook } from '@testing-library/react-hooks';
+import { waitFor } from '@testing-library/react';
+import { discoverServiceMock } from '../../../__mocks__/services';
+import { useTextBasedQueryLanguage } from './use_text_based_query_language';
+import { AppState, GetStateReturn } from '../services/discover_state';
+import { BehaviorSubject } from 'rxjs';
+import { FetchStatus } from '../../types';
+import { DataDocuments$, RecordRawType } from './use_saved_search';
+import { DataTableRecord } from '../../../types';
+import { AggregateQuery, Query } from '@kbn/es-query';
+import { dataViewMock } from '../../../__mocks__/data_view';
+import { DataViewListItem } from '@kbn/data-views-plugin/common';
+import { savedSearchMock } from '../../../__mocks__/saved_search';
+
+function getHookProps(
+  replaceUrlAppState: (newState: Partial<AppState>) => Promise<void>,
+  query: AggregateQuery | Query | undefined
+) {
+  const stateContainer = {
+    replaceUrlAppState,
+    appStateContainer: {
+      getState: () => {
+        return [];
+      },
+    },
+  } as unknown as GetStateReturn;
+
+  const msgLoading = {
+    recordRawType: RecordRawType.PLAIN,
+    fetchStatus: FetchStatus.LOADING,
+    query,
+  };
+
+  const documents$ = new BehaviorSubject(msgLoading) as DataDocuments$;
+
+  return {
+    documents$,
+    dataViews: discoverServiceMock.dataViews,
+    stateContainer,
+    dataViewList: [dataViewMock as DataViewListItem],
+    savedSearch: savedSearchMock,
+  };
+}
+const query = { sql: 'SELECT * from the-data-view-title' };
+const msgComplete = {
+  recordRawType: RecordRawType.PLAIN,
+  fetchStatus: FetchStatus.COMPLETE,
+  result: [
+    {
+      id: '1',
+      raw: { field1: 1, field2: 2 },
+      flattened: { field1: 1, field2: 2 },
+    } as unknown as DataTableRecord,
+  ],
+  query,
+};
+
+describe('useTextBasedQueryLanguage', () => {
+  test('a text based query should change state when loading and finished', async () => {
+    const replaceUrlAppState = jest.fn();
+    const props = getHookProps(replaceUrlAppState, query);
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+    expect(replaceUrlAppState).toHaveBeenCalledWith({ index: 'the-data-view-id' });
+
+    replaceUrlAppState.mockReset();
+
+    documents$.next(msgComplete);
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+
+    await waitFor(() => {
+      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        index: 'the-data-view-id',
+        columns: ['field1', 'field2'],
+      });
+    });
+  });
+  test('changing a text based query with different result columns should change state when loading and finished', async () => {
+    const replaceUrlAppState = jest.fn();
+    const props = getHookProps(replaceUrlAppState, query);
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+
+    documents$.next(msgComplete);
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(2));
+    replaceUrlAppState.mockReset();
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title' },
+    });
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+
+    await waitFor(() => {
+      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        index: 'the-data-view-id',
+        columns: ['field1'],
+      });
+    });
+  });
+  test('only changing a text based query with same result columns should not change columns', async () => {
+    const replaceUrlAppState = jest.fn();
+    const props = getHookProps(replaceUrlAppState, query);
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+
+    documents$.next(msgComplete);
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(2));
+    replaceUrlAppState.mockReset();
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title' },
+    });
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+    replaceUrlAppState.mockReset();
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title WHERE field1=1' },
+    });
+
+    await waitFor(() => {
+      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        index: 'the-data-view-id',
+      });
+    });
+  });
+  test('if its not a text based query coming along, it should be ignored', async () => {
+    const replaceUrlAppState = jest.fn();
+    const props = getHookProps(replaceUrlAppState, query);
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+
+    documents$.next(msgComplete);
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(2));
+    replaceUrlAppState.mockReset();
+
+    documents$.next({
+      recordRawType: RecordRawType.DOCUMENT,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+    });
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title WHERE field1=1' },
+    });
+
+    await waitFor(() => {
+      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        index: 'the-data-view-id',
+        columns: ['field1'],
+      });
+    });
+  });
+
+  test('it should not overwrite existing state columns on initial fetch', async () => {
+    const replaceUrlAppState = jest.fn();
+    const props = getHookProps(replaceUrlAppState, query);
+    props.stateContainer.appStateContainer.getState = jest.fn(() => {
+      return { columns: ['field1'], index: 'the-data-view-id' };
+    });
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1, field2: 2 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title WHERE field1=1' },
+    });
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-title' },
+    });
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+    expect(replaceUrlAppState).toHaveBeenCalledWith({
+      columns: ['field1'],
+    });
+  });
+});

src/plugins/discover/public/application/main/hooks/use_text_based_query_language.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { isEqual } from 'lodash';
+import {
+  isOfAggregateQueryType,
+  getIndexPatternFromSQLQuery,
+  AggregateQuery,
+  Query,
+} from '@kbn/es-query';
+import { useCallback, useEffect, useRef } from 'react';
+import { DataViewListItem, DataViewsContract } from '@kbn/data-views-plugin/public';
+import { SavedSearch } from '@kbn/saved-search-plugin/public';
+import type { GetStateReturn } from '../services/discover_state';
+import type { DataDocuments$ } from './use_saved_search';
+import { FetchStatus } from '../../types';
+
+const MAX_NUM_OF_COLUMNS = 50;
+
+/**
+ * Hook to take care of text based query language state transformations when a new result is returned
+ * If necessary this is setting displayed columns and selected data view
+ */
+export function useTextBasedQueryLanguage({
+  documents$,
+  dataViews,
+  stateContainer,
+  dataViewList,
+  savedSearch,
+}: {
+  documents$: DataDocuments$;
+  stateContainer: GetStateReturn;
+  dataViews: DataViewsContract;
+  dataViewList: DataViewListItem[];
+  savedSearch: SavedSearch;
+}) {
+  const prev = useRef<{ query: AggregateQuery | Query | undefined; columns: string[] }>({
+    columns: [],
+    query: undefined,
+  });
+
+  const cleanup = useCallback(() => {
+    if (prev.current.query) {
+      // cleanup when it's not a text based query lang
+      prev.current = {
+        columns: [],
+        query: undefined,
+      };
+    }
+  }, []);
+
+  useEffect(() => {
+    const subscription = documents$.subscribe(async (next) => {
+      const { query } = next;
+      const { columns: stateColumns, index } = stateContainer.appStateContainer.getState();
+      let nextColumns: string[] = [];
+      const isTextBasedQueryLang =
+        next.recordRawType === 'plain' && query && isOfAggregateQueryType(query) && 'sql' in query;
+      const hasResults = next.result?.length && next.fetchStatus === FetchStatus.COMPLETE;
+      const initialFetch = !prev.current.columns.length;
+
+      if (isTextBasedQueryLang) {
+        if (hasResults) {
+          // check if state needs to contain column transformation due to a different columns in the resultset
+          const firstRow = next.result![0];
+          const firstRowColumns = Object.keys(firstRow.raw).slice(0, MAX_NUM_OF_COLUMNS);
+          if (
+            !isEqual(firstRowColumns, prev.current.columns) &&
+            !isEqual(query, prev.current.query)
+          ) {
+            nextColumns = firstRowColumns;
+            prev.current = { columns: nextColumns, query };
+          }
+          if (firstRowColumns && initialFetch) {
+            prev.current = { columns: firstRowColumns, query };
+          }
+        }
+        const indexPatternFromQuery = getIndexPatternFromSQLQuery(query.sql);
+        const dataViewObj = dataViewList.find(({ title }) => title === indexPatternFromQuery);
+
+        if (dataViewObj) {
+          // don't set the columns on initial fetch, to prevent overwriting existing state
+          const addColumnsToState = Boolean(
+            nextColumns.length && (!initialFetch || !stateColumns?.length)
+          );
+          // no need to reset index to state if it hasn't changed
+          const addDataViewToState = Boolean(dataViewObj.id !== index);
+          if (!addColumnsToState && !addDataViewToState) {
+            return;
+          }
+
+          const nextState = {
+            ...(addDataViewToState && { index: dataViewObj.id }),
+            ...(addColumnsToState && { columns: nextColumns }),
+          };
+          stateContainer.replaceUrlAppState(nextState);
+        }
+      } else {
+        // cleanup for a "regular" query
+        cleanup();
+      }
+    });
+    return () => {
+      // cleanup for e.g. when savedSearch is switched
+      cleanup();
+      subscription.unsubscribe();
+    };
+  }, [documents$, dataViews, stateContainer, dataViewList, savedSearch, cleanup]);
+}

src/plugins/discover/public/application/main/utils/fetch_all.test.ts

@@ -242,10 +242,11 @@ describe('test fetchAll', () => {
     ];
     const documents = hits.map((hit) => buildDataTableRecord(hit, dataViewMock));
     mockFetchSQL.mockResolvedValue(documents);
+    const query = { sql: 'SELECT * from foo' };
     deps = {
       appStateContainer: {
         getState: () => {
-          return { interval: 'auto', query: { sql: 'SELECT * from foo' } };
+          return { interval: 'auto', query };
         },
       } as unknown as ReduxLikeStateContainer<AppState>,
       abortController: new AbortController(),
@@ -260,11 +261,12 @@ describe('test fetchAll', () => {
     await fetchAll(subjects, searchSource, false, deps);
     expect(await collect()).toEqual([
       { fetchStatus: FetchStatus.UNINITIALIZED },
-      { fetchStatus: FetchStatus.LOADING, recordRawType: 'plain' },
+      { fetchStatus: FetchStatus.LOADING, recordRawType: 'plain', query },
       {
         fetchStatus: FetchStatus.COMPLETE,
         recordRawType: 'plain',
         result: documents,
+        query,
       },
     ]);
   });

src/plugins/discover/public/application/main/utils/fetch_all.ts

@@ -65,7 +65,7 @@ export function fetchAll(
   const { initialFetchStatus, appStateContainer, services, useNewFieldsApi, data } = fetchDeps;
 
   /**
-   * Method to create a an error handler that will forward the received error
+   * Method to create an error handler that will forward the received error
    * to the specified subjects. It will ignore AbortErrors and will use the data
    * plugin to show a toast for the error (e.g. allowing better insights into shard failures).
    */
@@ -103,7 +103,7 @@ export function fetchAll(
 
     // Mark all subjects as loading
     sendLoadingMsg(dataSubjects.main$, recordRawType);
-    sendLoadingMsg(dataSubjects.documents$, recordRawType);
+    sendLoadingMsg(dataSubjects.documents$, recordRawType, query);
     sendLoadingMsg(dataSubjects.totalHits$, recordRawType);
     sendLoadingMsg(dataSubjects.charts$, recordRawType);
 
@@ -152,6 +152,7 @@ export function fetchAll(
           fetchStatus: FetchStatus.COMPLETE,
           result: docs,
           recordRawType,
+          query,
         });
 
         checkHitCount(docs.length);

src/plugins/discover/public/application/main/utils/resolve_data_view.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { loadDataView, getFallbackDataViewId, DataViewSavedObject } from './resolve_data_view';
+import { loadDataView, getFallbackDataViewId } from './resolve_data_view';
 import { dataViewsMock } from '../../../__mocks__/data_views';
 import { dataViewMock } from '../../../__mocks__/data_view';
 import { configMock } from '../../../__mocks__/config';
@@ -31,8 +31,8 @@ describe('Resolve data view tests', () => {
     expect(result).toBe('');
   });
   test('getFallbackDataViewId with an dataViews array', async () => {
-    const list = await dataViewsMock.getCache();
-    const result = await getFallbackDataViewId(list as unknown as DataViewSavedObject[], '');
+    const list = await dataViewsMock.getIdsWithTitle();
+    const result = await getFallbackDataViewId(list, '');
     expect(result).toBe('the-data-view-id');
   });
 });

src/plugins/discover/public/application/main/utils/resolve_data_view.ts

@@ -7,16 +7,14 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public';
+import type { DataView, DataViewListItem, DataViewsContract } from '@kbn/data-views-plugin/public';
 import type { ISearchSource } from '@kbn/data-plugin/public';
-import type { IUiSettingsClient, SavedObject, ToastsStart } from '@kbn/core/public';
-export type DataViewSavedObject = SavedObject & { title: string };
-
+import type { IUiSettingsClient, ToastsStart } from '@kbn/core/public';
 interface DataViewData {
   /**
    * List of existing data views
    */
-  list: DataViewSavedObject[];
+  list: DataViewListItem[];
   /**
    * Loaded data view (might be default data view if requested was not found)
    */
@@ -32,9 +30,9 @@ interface DataViewData {
 }
 
 export function findDataViewById(
-  dataViews: DataViewSavedObject[],
+  dataViews: DataViewListItem[],
   id: string
-): DataViewSavedObject | undefined {
+): DataViewListItem | undefined {
   if (!Array.isArray(dataViews) || !id) {
     return;
   }
@@ -46,7 +44,7 @@ export function findDataViewById(
  * the first available data view id if not
  */
 export function getFallbackDataViewId(
-  dataViews: DataViewSavedObject[],
+  dataViews: DataViewListItem[],
   defaultIndex: string = ''
 ): string {
   if (defaultIndex && findDataViewById(dataViews, defaultIndex)) {
@@ -62,7 +60,7 @@ export function getFallbackDataViewId(
  */
 export function getDataViewId(
   id: string = '',
-  dataViews: DataViewSavedObject[] = [],
+  dataViews: DataViewListItem[] = [],
   defaultIndex: string = ''
 ): string {
   if (!id || !findDataViewById(dataViews, id)) {
@@ -79,7 +77,7 @@ export async function loadDataView(
   dataViews: DataViewsContract,
   config: IUiSettingsClient
 ): Promise<DataViewData> {
-  const dataViewList = (await dataViews.getCache()) as unknown as DataViewSavedObject[];
+  const dataViewList = await dataViews.getIdsWithTitle();
 
   const actualId = getDataViewId(id, dataViewList, config.get('defaultIndex'));
   return {