[Text based languages] Enable index patterns (#147423)

## Summary Closes https://github.com/elastic/kibana/issues/144295 Closes https://github.com/elastic/kibana/issues/143623 1. Enables adhoc dataviews to work with text based languages. Write now if a user has created an adhoc dataview and switch to text based mode it fails. This PR fixes it. 2. Enables queries referencing index patterns. Until now we allow only queries that reference existing dataviews. This PR extends that to work with all valid index patterns. ### How it works It creates an adhoc dataview and try to find: - if a @timestamp field exists, it uses that - If not it uses the first date field if it exists - If the field doesn't exist, it hides the time picker (no time filter) This was decided after discussing this with the ESQL WG. Having only one date field and in preference the @timestamp field is the recommended way here and we feel that this will be the correct approach for the first milestones. ![sql](https://user-images.githubusercontent.com/17003240/208081884-76215970-5b74-4e4a-9667-81346b34432f.gif) ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
2025-04-24 17:59:23 -04:00 · 2022-12-21 12:28:18 +02:00 · 2022-12-21 12:28:18 +02:00 · 9f54ad46c6
commit 9f54ad46c6
parent 3aff6fad86
30 changed files with 478 additions and 230 deletions
--- a/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.test.ts
+++ b/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.test.ts
@ -5,12 +5,11 @@
 * in compliance with, at your election, the Elastic License 2.0 or the Server
 * Side Public License, v 1.
 */
-import { DataViewsContract } from '@kbn/data-views-plugin/common';
+import { createStubDataView } from '@kbn/data-views-plugin/common/mocks';
 import { textBasedQueryStateToAstWithValidation } from './text_based_query_state_to_ast_with_validation';

 describe('textBasedQueryStateToAstWithValidation', () => {
  it('returns undefined for a non text based query', async () => {
-    const dataViewsService = {} as unknown as DataViewsContract;
    const actual = await textBasedQueryStateToAstWithValidation({
      filters: [],
      query: { language: 'lucene', query: '' },
@ -18,30 +17,19 @@ describe('textBasedQueryStateToAstWithValidation', () => {
        from: 'now',
        to: 'now+7d',
      },
-      dataViewsService,
    });

    expect(actual).toBeUndefined();
  });

  it('returns an object with the correct structure for an SQL query with existing dataview', async () => {
-    const dataViewsService = {
-      getIdsWithTitle: jest.fn(() => {
-        return [
-          {
-            title: 'foo',
-            id: 'bar',
-          },
-        ];
-      }),
-      get: jest.fn(() => {
-        return {
-          title: 'foo',
-          id: 'bar',
-          timeFieldName: 'baz',
-        };
-      }),
-    } as unknown as DataViewsContract;
+    const dataView = createStubDataView({
+      spec: {
+        id: 'foo',
+        title: 'foo',
+        timeFieldName: '@timestamp',
+      },
+    });
    const actual = await textBasedQueryStateToAstWithValidation({
      filters: [],
      query: { sql: 'SELECT * FROM foo' },
@ -49,7 +37,7 @@ describe('textBasedQueryStateToAstWithValidation', () => {
        from: 'now',
        to: 'now+7d',
      },
-      dataViewsService,
+      dataView,
    });

    expect(actual).toHaveProperty(
@ -68,35 +56,20 @@ describe('textBasedQueryStateToAstWithValidation', () => {
    );
  });

-  it('returns an error for text based language with non existing dataview', async () => {
-    const dataViewsService = {
-      getIdsWithTitle: jest.fn(() => {
-        return [
-          {
-            title: 'foo',
-            id: 'bar',
-          },
-        ];
-      }),
-      get: jest.fn(() => {
-        return {
-          title: 'foo',
-          id: 'bar',
-          timeFieldName: 'baz',
-        };
-      }),
-    } as unknown as DataViewsContract;
-
-    await expect(
-      textBasedQueryStateToAstWithValidation({
-        filters: [],
-        query: { sql: 'SELECT * FROM another_dataview' },
-        time: {
-          from: 'now',
-          to: 'now+7d',
-        },
-        dataViewsService,
+  it('returns an object with the correct structure for text based language with non existing dataview', async () => {
+    const actual = await textBasedQueryStateToAstWithValidation({
+      filters: [],
+      query: { sql: 'SELECT * FROM index_pattern_with_no_data_view' },
+      time: {
+        from: 'now',
+        to: 'now+7d',
+      },
+    });
+    expect(actual).toHaveProperty(
+      'chain.2.arguments',
+      expect.objectContaining({
+        query: ['SELECT * FROM index_pattern_with_no_data_view'],
      })
-    ).rejects.toThrow('No data view found for index pattern another_dataview');
+    );
  });
 });
--- a/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.ts
+++ b/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.ts
@ -5,27 +5,17 @@
 * in compliance with, at your election, the Elastic License 2.0 or the Server
 * Side Public License, v 1.
 */
-import {
-  isOfAggregateQueryType,
-  getIndexPatternFromSQLQuery,
-  Query,
-  AggregateQuery,
-} from '@kbn/es-query';
-import type { DataViewsContract } from '@kbn/data-views-plugin/common';
+import { isOfAggregateQueryType, Query } from '@kbn/es-query';
+import type { DataView } from '@kbn/data-views-plugin/common';
 import type { QueryState } from '..';
 import { textBasedQueryStateToExpressionAst } from './text_based_query_state_to_ast';

 interface Args extends QueryState {
-  dataViewsService: DataViewsContract;
+  dataView?: DataView;
  inputQuery?: Query;
+  timeFieldName?: string;
 }

-const getIndexPatternFromAggregateQuery = (query: AggregateQuery) => {
-  if ('sql' in query) {
-    return getIndexPatternFromSQLQuery(query.sql);
-  }
-};
-
 /**
 * Converts QueryState to expression AST
 * @param filters array of kibana filters
@ -37,29 +27,17 @@ export async function textBasedQueryStateToAstWithValidation({
  query,
  inputQuery,
  time,
-  dataViewsService,
+  dataView,
 }: Args) {
  let ast;
  if (query && isOfAggregateQueryType(query)) {
-    // sql query
-    const idxPattern = getIndexPatternFromAggregateQuery(query);
-    const idsTitles = await dataViewsService.getIdsWithTitle();
-    const dataViewIdTitle = idsTitles.find(({ title }) => title === idxPattern);
-
-    if (dataViewIdTitle) {
-      const dataView = await dataViewsService.get(dataViewIdTitle.id);
-      const timeFieldName = dataView.timeFieldName;
-
-      ast = textBasedQueryStateToExpressionAst({
-        filters,
-        query,
-        inputQuery,
-        time,
-        timeFieldName,
-      });
-    } else {
-      throw new Error(`No data view found for index pattern ${idxPattern}`);
-    }
+    ast = textBasedQueryStateToExpressionAst({
+      filters,
+      query,
+      inputQuery,
+      time,
+      timeFieldName: dataView?.timeFieldName,
+    });
  }
  return ast;
 }
--- a/src/plugins/discover/public/mocks/data_view.ts
+++ b/src/plugins/discover/public/mocks/data_view.ts
@ -63,6 +63,14 @@ const fields = [
    filterable: true,
    aggregatable: true,
  },
+  {
+    name: '@timestamp',
+    type: 'date',
+    displayName: '@timestamp',
+    scripted: false,
+    filterable: true,
+    aggregatable: true,
+  },
 ] as DataView['fields'];

 export const buildDataViewMock = ({
--- a/src/plugins/discover/public/application/context/services/snapshots/context.test.ts.snap
+++ b/src/plugins/discover/public/application/context/services/snapshots/context.test.ts.snap
@ -46,6 +46,9 @@ Object {
    Object {
      "field": "object.value",
    },
+    Object {
+      "field": "@timestamp",
+    },
  ],
  "query": Object {
    "bool": Object {
--- a/src/plugins/discover/public/application/main/discover_main_route.tsx
+++ b/src/plugins/discover/public/application/main/discover_main_route.tsx
@ -8,6 +8,7 @@
 import React, { useEffect, useState, memo, useCallback, useMemo } from 'react';
 import { useParams, useHistory } from 'react-router-dom';
 import { DataViewListItem } from '@kbn/data-plugin/public';
+import { isOfAggregateQueryType } from '@kbn/es-query';
 import { DataViewSavedObjectConflictError } from '@kbn/data-views-plugin/public';
 import { redirectWhenMissing } from '@kbn/kibana-utils-plugin/public';
 import { useExecutionContext } from '@kbn/kibana-react-plugin/public';
@ -101,7 +102,7 @@ export function DiscoverMainRoute(props: Props) {
        }

        const { appStateContainer } = getState({ history, savedSearch: nextSavedSearch, services });
-        const { index } = appStateContainer.getState();
+        const { index, query } = appStateContainer.getState();
        const ip = await loadDataView(
          data.dataViews,
          config,
@ -110,7 +111,13 @@ export function DiscoverMainRoute(props: Props) {
        );

        const ipList = ip.list;
-        const dataViewData = resolveDataView(ip, nextSavedSearch.searchSource, toastNotifications);
+        const isTextBasedQuery = query && isOfAggregateQueryType(query);
+        const dataViewData = resolveDataView(
+          ip,
+          nextSavedSearch.searchSource,
+          toastNotifications,
+          isTextBasedQuery
+        );
        await data.dataViews.refreshFields(dataViewData);
        setDataViewList(ipList);

--- a/src/plugins/discover/public/application/main/hooks/use_adhoc_data_views.test.ts
+++ b/src/plugins/discover/public/application/main/hooks/use_adhoc_data_views.test.ts
@ -132,4 +132,29 @@ describe('useAdHocDataViews', () => {
    });
    expect(updatedDataView!.id).toEqual('updated-mock-id');
  });
+
+  it('should update the adHocList correctly for text based mode', async () => {
+    const hook = renderHook((d: DataView) =>
+      useAdHocDataViews({
+        dataView: mockDataView,
+        savedSearch: savedSearchMock,
+        stateContainer: {
+          appStateContainer: { getState: jest.fn().mockReturnValue({}) },
+          replaceUrlAppState: jest.fn(),
+          kbnUrlStateStorage: {
+            kbnUrlControls: { flush: jest.fn() },
+          },
+        } as unknown as GetStateReturn,
+        setUrlTracking: jest.fn(),
+        dataViews: mockDiscoverServices.dataViews,
+        filterManager: mockDiscoverServices.filterManager,
+        toastNotifications: mockDiscoverServices.toastNotifications,
+        isTextBasedMode: true,
+      })
+    );
+
+    const adHocList = await hook.result.current.adHocDataViewList;
+    expect(adHocList.length).toBe(1);
+    expect(adHocList[0].id).toEqual('mock-id');
+  });
 });
--- a/src/plugins/discover/public/application/main/hooks/use_adhoc_data_views.ts
+++ b/src/plugins/discover/public/application/main/hooks/use_adhoc_data_views.ts
@ -32,6 +32,7 @@ export const useAdHocDataViews = ({
  dataViews,
  toastNotifications,
  trackUiMetric,
+  isTextBasedMode,
 }: {
  dataView: DataView;
  savedSearch: SavedSearch;
@ -41,6 +42,7 @@ export const useAdHocDataViews = ({
  filterManager: FilterManager;
  toastNotifications: ToastsStart;
  trackUiMetric?: (metricType: string, eventName: string | string[], count?: number) => void;
+  isTextBasedMode?: boolean;
 }) => {
  const [adHocDataViewList, setAdHocDataViewList] = useState<DataView[]>(
    !dataView.isPersisted() ? [dataView] : []
@ -50,11 +52,14 @@ export const useAdHocDataViews = ({
    if (!dataView.isPersisted()) {
      setAdHocDataViewList((prev) => {
        const existing = prev.find((prevDataView) => prevDataView.id === dataView.id);
-        return existing ? prev : [...prev, dataView];
+        return existing ? prev : isTextBasedMode ? [dataView] : [...prev, dataView];
      });
-      trackUiMetric?.(METRIC_TYPE.COUNT, ADHOC_DATA_VIEW_RENDER_EVENT);
+      // increase the counter only for dataview mode
+      if (!isTextBasedMode) {
+        trackUiMetric?.(METRIC_TYPE.COUNT, ADHOC_DATA_VIEW_RENDER_EVENT);
+      }
    }
-  }, [dataView, trackUiMetric]);
+  }, [dataView, isTextBasedMode, trackUiMetric]);

  /**
   * Takes care of checking data view id references in filters
@ -125,5 +130,6 @@ export const useAdHocDataViews = ({
    persistDataView,
    updateAdHocDataViewId,
    onAddAdHocDataViews,
+    setAdHocDataViewList,
  };
 };
--- a/src/plugins/discover/public/application/main/hooks/use_discover_state.ts
+++ b/src/plugins/discover/public/application/main/hooks/use_discover_state.ts
@ -8,6 +8,7 @@
 import { useMemo, useEffect, useState, useCallback } from 'react';
 import { isEqual } from 'lodash';
 import { History } from 'history';
+import { isOfAggregateQueryType } from '@kbn/es-query';
 import { type DataViewListItem, type DataView, DataViewType } from '@kbn/data-views-plugin/public';
 import { SavedSearch, getSavedSearch } from '@kbn/saved-search-plugin/public';
 import type { SortOrder } from '@kbn/saved-search-plugin/public';
@ -125,6 +126,7 @@ export function useDiscoverState({
  /**
   * Adhoc data views functionality
   */
+  const isTextBasedMode = state?.query && isOfAggregateQueryType(state?.query);
  const { adHocDataViewList, persistDataView, updateAdHocDataViewId, onAddAdHocDataViews } =
    useAdHocDataViews({
      dataView,
@ -135,6 +137,7 @@ export function useDiscoverState({
      filterManager,
      toastNotifications,
      trackUiMetric,
+      isTextBasedMode,
    });

  const [savedDataViewList, setSavedDataViewList] = useState(initialDataViewList);
@ -169,7 +172,7 @@ export function useDiscoverState({
    documents$: data$.documents$,
    dataViews,
    stateContainer,
-    dataViewList: savedDataViewList,
+    dataViewList: [...savedDataViewList, ...adHocDataViewList],
    savedSearch,
  });

--- a/src/plugins/discover/public/application/main/hooks/use_test_based_query_language.test.ts
+++ b/src/plugins/discover/public/application/main/hooks/use_test_based_query_language.test.ts
@ -8,6 +8,7 @@

 import { renderHook } from '@testing-library/react-hooks';
 import { waitFor } from '@testing-library/react';
+import { DataViewsContract } from '@kbn/data-plugin/public';
 import { discoverServiceMock } from '../../../__mocks__/services';
 import { useTextBasedQueryLanguage } from './use_text_based_query_language';
 import { AppState, GetStateReturn } from '../services/discover_state';
@ -22,7 +23,8 @@ import { savedSearchMock } from '../../../__mocks__/saved_search';

 function getHookProps(
  replaceUrlAppState: (newState: Partial<AppState>) => Promise<void>,
-  query: AggregateQuery | Query | undefined
+  query: AggregateQuery | Query | undefined,
+  dataViewsService?: DataViewsContract
 ) {
  const stateContainer = {
    replaceUrlAppState,
@ -43,7 +45,7 @@ function getHookProps(

  return {
    documents$,
-    dataViews: discoverServiceMock.dataViews,
+    dataViews: dataViewsService ?? discoverServiceMock.dataViews,
    stateContainer,
    dataViewList: [dataViewMock as DataViewListItem],
    savedSearch: savedSearchMock,
@ -72,7 +74,7 @@ describe('useTextBasedQueryLanguage', () => {
    renderHook(() => useTextBasedQueryLanguage(props));

    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
-    expect(replaceUrlAppState).toHaveBeenCalledWith({ index: 'the-data-view-id' });
+    expect(replaceUrlAppState).toHaveBeenCalledWith({ columns: [], index: 'the-data-view-id' });

    replaceUrlAppState.mockReset();

@ -159,6 +161,7 @@ describe('useTextBasedQueryLanguage', () => {

    await waitFor(() => {
      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        columns: [],
        index: 'the-data-view-id',
      });
    });
@ -315,4 +318,45 @@ describe('useTextBasedQueryLanguage', () => {
      columns: ['field1'],
    });
  });
+
+  test('changing a text based query with an index pattern that not corresponds to a dataview should return results', async () => {
+    const replaceUrlAppState = jest.fn();
+    const dataViewsCreateMock = discoverServiceMock.dataViews.create as jest.Mock;
+    dataViewsCreateMock.mockImplementation(() => ({
+      ...dataViewMock,
+    }));
+    const dataViewsService = {
+      ...discoverServiceMock.dataViews,
+      create: dataViewsCreateMock,
+    };
+    const props = getHookProps(replaceUrlAppState, query, dataViewsService);
+    const { documents$ } = props;
+
+    renderHook(() => useTextBasedQueryLanguage(props));
+
+    documents$.next(msgComplete);
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(2));
+    replaceUrlAppState.mockReset();
+
+    documents$.next({
+      recordRawType: RecordRawType.PLAIN,
+      fetchStatus: FetchStatus.COMPLETE,
+      result: [
+        {
+          id: '1',
+          raw: { field1: 1 },
+          flattened: { field1: 1 },
+        } as unknown as DataTableRecord,
+      ],
+      query: { sql: 'SELECT field1 from the-data-view-*' },
+    });
+    await waitFor(() => expect(replaceUrlAppState).toHaveBeenCalledTimes(1));
+
+    await waitFor(() => {
+      expect(replaceUrlAppState).toHaveBeenCalledWith({
+        index: 'the-data-view-id',
+        columns: ['field1'],
+      });
+    });
+  });
 });
--- a/src/plugins/discover/public/application/main/hooks/use_text_based_query_language.ts
+++ b/src/plugins/discover/public/application/main/hooks/use_text_based_query_language.ts
@ -13,7 +13,7 @@ import {
  Query,
 } from '@kbn/es-query';
 import { useCallback, useEffect, useRef } from 'react';
-import { DataViewListItem, DataViewsContract } from '@kbn/data-views-plugin/public';
+import type { DataViewListItem, DataViewsContract, DataView } from '@kbn/data-views-plugin/public';
 import { SavedSearch } from '@kbn/saved-search-plugin/public';
 import type { GetStateReturn } from '../services/discover_state';
 import type { DataDocuments$ } from './use_saved_search';
@ -35,7 +35,7 @@ export function useTextBasedQueryLanguage({
  documents$: DataDocuments$;
  stateContainer: GetStateReturn;
  dataViews: DataViewsContract;
-  dataViewList: DataViewListItem[];
+  dataViewList: Array<DataViewListItem | DataView>;
  savedSearch: SavedSearch;
 }) {
  const prev = useRef<{ query: AggregateQuery | Query | undefined; columns: string[] }>({
@ -78,30 +78,44 @@ export function useTextBasedQueryLanguage({
            prev.current = { columns: firstRowColumns, query };
            nextColumns = firstRowColumns;
          }
+
          if (firstRowColumns && initialFetch) {
            prev.current = { columns: firstRowColumns, query };
          }
        }
        const indexPatternFromQuery = getIndexPatternFromSQLQuery(query.sql);
-        const dataViewObj = dataViewList.find(({ title }) => title === indexPatternFromQuery);
+        let dataViewObj = dataViewList.find(({ title }) => title === indexPatternFromQuery);

-        if (dataViewObj) {
-          // don't set the columns on initial fetch, to prevent overwriting existing state
-          const addColumnsToState = Boolean(
-            nextColumns.length && (!initialFetch || !stateColumns?.length)
-          );
-          // no need to reset index to state if it hasn't changed
-          const addDataViewToState = Boolean(dataViewObj.id !== index);
-          if (!addColumnsToState && !addDataViewToState) {
-            return;
+        // no dataview found but the index pattern is valid
+        // create an adhoc instance instead
+        if (!dataViewObj) {
+          dataViewObj = await dataViews.create({
+            title: indexPatternFromQuery,
+          });
+
+          if (dataViewObj.fields.getByName('@timestamp')?.type === 'date') {
+            dataViewObj.timeFieldName = '@timestamp';
+          } else if (dataViewObj.fields.getByType('date')?.length) {
+            const dateFields = dataViewObj.fields.getByType('date');
+            dataViewObj.timeFieldName = dateFields[0].name;
          }
-
-          const nextState = {
-            ...(addDataViewToState && { index: dataViewObj.id }),
-            ...(addColumnsToState && { columns: nextColumns }),
-          };
-          stateContainer.replaceUrlAppState(nextState);
        }
+
+        // don't set the columns on initial fetch, to prevent overwriting existing state
+        const addColumnsToState = Boolean(
+          nextColumns.length && (!initialFetch || !stateColumns?.length)
+        );
+        // no need to reset index to state if it hasn't changed
+        const addDataViewToState = Boolean(dataViewObj.id !== index);
+        if (!addColumnsToState && !addDataViewToState) {
+          return;
+        }
+
+        const nextState = {
+          ...(addDataViewToState && { index: dataViewObj.id }),
+          columns: nextColumns,
+        };
+        stateContainer.replaceUrlAppState(nextState);
      } else {
        // cleanup for a "regular" query
        cleanup();
--- a/src/plugins/discover/public/application/main/utils/fetch_all.ts
+++ b/src/plugins/discover/public/application/main/utils/fetch_all.ts
@ -88,7 +88,7 @@ export function fetchAll(
    // Start fetching all required requests
    const documents =
      useSql && query
-        ? fetchSql(query, services.dataViews, data, services.expressions, inspectorAdapters)
+        ? fetchSql(query, dataView, data, services.expressions, inspectorAdapters)
        : fetchDocuments(searchSource.createCopy(), fetchDeps);

    // Handle results of the individual queries and forward the results to the corresponding dataSubjects
--- a/src/plugins/discover/public/application/main/utils/fetch_sql.ts
+++ b/src/plugins/discover/public/application/main/utils/fetch_sql.ts
@ -12,7 +12,7 @@ import type { Adapters } from '@kbn/inspector-plugin/common';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { Datatable } from '@kbn/expressions-plugin/public';
-import type { DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView } from '@kbn/data-views-plugin/common';
 import { textBasedQueryStateToAstWithValidation } from '@kbn/data-plugin/common';
 import { DataTableRecord } from '../../../types';

@ -25,7 +25,7 @@ interface SQLErrorResponse {

 export function fetchSql(
  query: Query | AggregateQuery,
-  dataViewsService: DataViewsContract,
+  dataView: DataView,
  data: DataPublicPluginStart,
  expressions: ExpressionsStart,
  inspectorAdapters: Adapters,
@ -37,7 +37,7 @@ export function fetchSql(
    filters,
    query,
    time: timeRange,
-    dataViewsService,
+    dataView,
    inputQuery,
  })
    .then((ast) => {
--- a/src/plugins/discover/public/application/main/utils/resolve_data_view.ts
+++ b/src/plugins/discover/public/application/main/utils/resolve_data_view.ts
@ -138,7 +138,8 @@ export async function loadDataView(
 export function resolveDataView(
  ip: DataViewData,
  searchSource: ISearchSource,
-  toastNotifications: ToastsStart
+  toastNotifications: ToastsStart,
+  isTextBasedQuery?: boolean
 ) {
  const { loaded: loadedDataView, stateVal, stateValFound } = ip;

@ -170,19 +171,20 @@ export function resolveDataView(
      });
      return ownDataView;
    }
-
-    toastNotifications.addWarning({
-      title: warningTitle,
-      text: i18n.translate('discover.showingDefaultDataViewWarningDescription', {
-        defaultMessage:
-          'Showing the default data view: "{loadedDataViewTitle}" ({loadedDataViewId})',
-        values: {
-          loadedDataViewTitle: loadedDataView.getIndexPattern(),
-          loadedDataViewId: loadedDataView.id,
-        },
-      }),
-      'data-test-subj': 'dscDataViewNotFoundShowDefaultWarning',
-    });
+    if (!Boolean(isTextBasedQuery)) {
+      toastNotifications.addWarning({
+        title: warningTitle,
+        text: i18n.translate('discover.showingDefaultDataViewWarningDescription', {
+          defaultMessage:
+            'Showing the default data view: "{loadedDataViewTitle}" ({loadedDataViewId})',
+          values: {
+            loadedDataViewTitle: loadedDataView.getIndexPattern(),
+            loadedDataViewId: loadedDataView.id,
+          },
+        }),
+        'data-test-subj': 'dscDataViewNotFoundShowDefaultWarning',
+      });
+    }
  }

  return loadedDataView;
--- a/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx
+++ b/src/plugins/discover/public/embeddable/saved_search_embeddable.tsx
@ -219,6 +219,7 @@ export class SavedSearchEmbeddable
      : child;

    const query = this.savedSearch.searchSource.getField('query');
+    const dataView = this.savedSearch.searchSource.getField('index')!;
    const recordRawType = getRawRecordType(query);
    const useSql = recordRawType === RecordRawType.PLAIN;

@ -227,7 +228,7 @@ export class SavedSearchEmbeddable
      if (useSql && query) {
        const result = await fetchSql(
          this.savedSearch.searchSource.getField('query')!,
-          this.services.dataViews,
+          dataView,
          this.services.data,
          this.services.expressions,
          this.services.inspector,
--- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx
+++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx
@ -116,15 +116,13 @@ export function ChangeDataView({
      setDataViewsList(dataViewsRefs);
    };
    fetchDataViews();
-  }, [data, currentDataViewId, adHocDataViews, savedDataViews]);
+  }, [data, currentDataViewId, adHocDataViews, savedDataViews, isTextBasedLangSelected]);

  useEffect(() => {
-    if (trigger.label) {
-      if (textBasedLanguage) {
-        setTriggerLabel(textBasedLanguage.toUpperCase());
-      } else {
-        setTriggerLabel(trigger.label);
-      }
+    if (textBasedLanguage) {
+      setTriggerLabel(textBasedLanguage.toUpperCase());
+    } else {
+      setTriggerLabel(trigger.label);
    }
  }, [textBasedLanguage, trigger.label]);

@ -157,7 +155,8 @@ export function ChangeDataView({
        {...rest}
      >
        <>
-          {isAdHocSelected && (
+          {/* we don't want to display the adHoc icon on text based mode */}
+          {isAdHocSelected && !isTextBasedLangSelected && (
            <EuiIcon
              type={adhoc}
              color="primary"
--- a/src/plugins/unified_search/public/query_string_input/text_based_languages_editor/helpers.test.ts
+++ b/src/plugins/unified_search/public/query_string_input/text_based_languages_editor/helpers.test.ts
@ -51,42 +51,6 @@ describe('helpers', function () {
      ]);
    });

-    it('should return the correct error object if dataview not found for an one liner query', function () {
-      const error = new Error('No data view found for index pattern kibana_sample_data_ecommerce1');
-      const errors = [error];
-      expect(parseErrors(errors, `SELECT * FROM "kibana_sample_data_ecommerce1"`)).toEqual([
-        {
-          endColumn: 46,
-          endLineNumber: 1,
-          message: 'No data view found for index pattern kibana_sample_data_ecommerce1',
-          severity: 8,
-          startColumn: 10,
-          startLineNumber: 1,
-        },
-      ]);
-    });
-
-    it('should return the correct error object if dataview not found for a multiline query', function () {
-      const error = new Error('No data view found for index pattern kibana_sample_data_ecommerce1');
-      const errors = [error];
-      expect(
-        parseErrors(
-          errors,
-          `SELECT * 
-    from "kibana_sample_data_ecommerce1"`
-        )
-      ).toEqual([
-        {
-          endColumn: 41,
-          endLineNumber: 2,
-          message: 'No data view found for index pattern kibana_sample_data_ecommerce1',
-          severity: 8,
-          startColumn: 5,
-          startLineNumber: 2,
-        },
-      ]);
-    });
-
    it('should return the generic error object for an error of unknown format', function () {
      const error = new Error('I am an unknown error');
      const errors = [error];
--- a/src/plugins/unified_search/public/query_string_input/text_based_languages_editor/helpers.ts
+++ b/src/plugins/unified_search/public/query_string_input/text_based_languages_editor/helpers.ts
@ -9,7 +9,6 @@
 import { useRef } from 'react';
 import useDebounce from 'react-use/lib/useDebounce';
 import { monaco } from '@kbn/monaco';
-import { getIndexPatternFromSQLQuery } from '@kbn/es-query';
 import { i18n } from '@kbn/i18n';

 export const useDebounceWithOptions = (
@ -53,44 +52,6 @@ export const parseErrors = (errors: Error[], code: string) => {
        endLineNumber: Number(lineNumber),
        severity: monaco.MarkerSeverity.Error,
      };
-    } else if (error.message.includes('No data view found')) {
-      const dataviewString = getIndexPatternFromSQLQuery(code);
-      const temp = code.split(dataviewString);
-      const lastChar = temp[0]?.charAt(temp[0]?.length - 1);
-      const additionnalLength = lastChar === '"' || "'" ? 2 : 0;
-      // 5 is the length of FROM + space
-      const errorLength = 5 + dataviewString.length + additionnalLength;
-      // no dataview found error message
-      const hasLines = /\r|\n/.exec(code);
-      if (hasLines) {
-        const linesText = code.split(/\r|\n/);
-        let indexWithError = 1;
-        let lineWithError = '';
-        linesText.forEach((line, index) => {
-          if (line.includes('FROM') || line.includes('from')) {
-            indexWithError = index + 1;
-            lineWithError = line;
-          }
-        });
-        const lineWithErrorUpperCase = lineWithError.toUpperCase();
-        return {
-          message: error.message,
-          startColumn: lineWithErrorUpperCase.indexOf('FROM') + 1,
-          startLineNumber: indexWithError,
-          endColumn: lineWithErrorUpperCase.indexOf('FROM') + 1 + errorLength,
-          endLineNumber: indexWithError,
-          severity: monaco.MarkerSeverity.Error,
-        };
-      } else {
-        return {
-          message: error.message,
-          startColumn: code.toUpperCase().indexOf('FROM') + 1,
-          startLineNumber: 1,
-          endColumn: code.toUpperCase().indexOf('FROM') + 1 + errorLength,
-          endLineNumber: 1,
-          severity: monaco.MarkerSeverity.Error,
-        };
-      }
    } else {
      // unknown error message
      return {
--- a/test/functional/apps/discover/group2/_sql_view.ts
+++ b/test/functional/apps/discover/group2/_sql_view.ts
@ -113,6 +113,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
        cell = await dataGrid.getCellElement(0, 3);
        expect(await cell.getVisibleText()).to.be('2269');
      });
+
+      it('should query an index pattern that doesnt translate to a dataview correctly', async function () {
+        await PageObjects.discover.selectTextBaseLang('SQL');
+        const testQuery = `SELECT "@tags", geo.dest, count(*) occurred FROM "logstash*"
+          GROUP BY "@tags", geo.dest
+          HAVING occurred > 20
+          ORDER BY occurred DESC`;
+
+        await monacoEditor.setCodeEditorValue(testQuery);
+        await testSubjects.click('querySubmitButton');
+        await PageObjects.header.waitUntilLoadingHasFinished();
+
+        const cell = await dataGrid.getCellElement(0, 3);
+        expect(await cell.getVisibleText()).to.be('2269');
+      });
    });
  });
 }
--- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx
+++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx
@ -411,13 +411,20 @@ export const LensTopNavMenu = ({
        const dataViewId = datasourceMap[activeDatasourceId].getUsedDataView(
          datasourceStates[activeDatasourceId].state
        );
-        const dataView = await data.dataViews.get(dataViewId);
+        const dataView = dataViewId ? await data.dataViews.get(dataViewId) : undefined;
        setCurrentIndexPattern(dataView ?? indexPatterns[0]);
      }
    };

    setCurrentPattern();
-  }, [activeDatasourceId, datasourceMap, datasourceStates, indexPatterns, data.dataViews]);
+  }, [
+    activeDatasourceId,
+    datasourceMap,
+    datasourceStates,
+    indexPatterns,
+    data.dataViews,
+    isOnTextBasedMode,
+  ]);

  useEffect(() => {
    if (typeof query === 'object' && query !== null && isOfAggregateQueryType(query)) {
@ -979,6 +986,7 @@ export const LensTopNavMenu = ({
      }
    }
  }
+
  return (
    <AggregateQueryTopNavMenu
      setMenuMountPoint={setHeaderActionMenu}
--- a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx
+++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx
@ -261,7 +261,7 @@ export function getFormBasedDatasource({
          ...state,
          layers: {
            ...newLayers,
-            [layerId]: blankLayer(state.currentIndexPatternId, state.layers[layerId].linkToLayers),
+            [layerId]: blankLayer(state.currentIndexPatternId, state.layers[layerId]?.linkToLayers),
          },
        },
      };
--- a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx
+++ b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx
@ -49,6 +49,7 @@ export function TextBasedDataPanel({
  core,
  data,
  query,
+  frame,
  filters,
  dateRange,
  expressions,
@ -60,12 +61,14 @@ export function TextBasedDataPanel({
  useEffect(() => {
    async function fetchData() {
      if (query && isOfAggregateQueryType(query) && !isEqual(query, prevQuery)) {
+        const frameDataViews = frame.dataViews;
        const stateFromQuery = await getStateFromAggregateQuery(
          state,
          query,
          dataViews,
          data,
-          expressions
+          expressions,
+          frameDataViews
        );

        setDataHasLoaded(true);
@ -73,7 +76,7 @@ export function TextBasedDataPanel({
      }
    }
    fetchData();
-  }, [data, dataViews, expressions, prevQuery, query, setState, state]);
+  }, [data, dataViews, expressions, prevQuery, query, setState, state, frame.dataViews]);

  const { fieldList } = state;

--- a/x-pack/plugins/lens/public/datasources/text_based/fetch_data_from_aggregate_query.ts
+++ b/x-pack/plugins/lens/public/datasources/text_based/fetch_data_from_aggregate_query.ts
@ -10,7 +10,7 @@ import { Query, AggregateQuery, Filter } from '@kbn/es-query';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { Datatable } from '@kbn/expressions-plugin/public';
-import type { DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView } from '@kbn/data-views-plugin/common';
 import { textBasedQueryStateToAstWithValidation } from '@kbn/data-plugin/common';

 interface TextBasedLanguagesErrorResponse {
@ -22,7 +22,7 @@ interface TextBasedLanguagesErrorResponse {

 export function fetchDataFromAggregateQuery(
  query: Query | AggregateQuery,
-  dataViewsService: DataViewsContract,
+  dataView: DataView,
  data: DataPublicPluginStart,
  expressions: ExpressionsStart,
  filters?: Filter[],
@ -33,7 +33,7 @@ export function fetchDataFromAggregateQuery(
    filters,
    query,
    time: timeRange,
-    dataViewsService,
+    dataView,
    inputQuery,
  })
    .then((ast) => {
--- a/x-pack/plugins/lens/public/datasources/text_based/layerpanel.test.tsx
+++ b/x-pack/plugins/lens/public/datasources/text_based/layerpanel.test.tsx
@ -78,7 +78,7 @@ describe('Layer Data Panel', () => {
    expect(instance.find(ChangeIndexPattern).prop('trigger')).toStrictEqual({
      fontWeight: 'normal',
      isDisabled: true,
-      label: 'My fake index pattern',
+      label: 'my-fake-index-pattern',
      size: 's',
      title: 'my-fake-index-pattern',
    });
--- a/x-pack/plugins/lens/public/datasources/text_based/layerpanel.tsx
+++ b/x-pack/plugins/lens/public/datasources/text_based/layerpanel.tsx
@ -18,7 +18,8 @@ export interface TextBasedLayerPanelProps extends DatasourceLayerPanelProps<Text

 export function LayerPanel({ state, layerId, dataViews }: TextBasedLayerPanelProps) {
  const layer = state.layers[layerId];
-  const dataView = dataViews.indexPatternRefs.find((ref) => ref.id === layer.index);
+  const dataView = state.indexPatternRefs.find((ref) => ref.id === layer.index);
+
  const notFoundTitleLabel = i18n.translate('xpack.lens.layerPanel.missingDataView', {
    defaultMessage: 'Data view not found',
  });
--- a/x-pack/plugins/lens/public/datasources/text_based/to_expression.ts
+++ b/x-pack/plugins/lens/public/datasources/text_based/to_expression.ts
@ -34,7 +34,8 @@ function getExpressionForLayer(layer: TextBasedLayer, refs: IndexPatternRef[]):
      };
    }
  });
-  const timeFieldName = refs.find((r) => r.id === layer.index)?.timeField;
+  const timeFieldName = layer.timeField ?? undefined;
+
  const textBasedQueryToAst = textBasedQueryStateToExpressionAst({
    query: layer.query,
    timeFieldName,
--- a/x-pack/plugins/lens/public/datasources/text_based/types.ts
+++ b/x-pack/plugins/lens/public/datasources/text_based/types.ts
@ -43,4 +43,5 @@ export interface IndexPatternRef {
  id: string;
  title: string;
  timeField?: string;
+  name?: string;
 }
--- a/x-pack/plugins/lens/public/datasources/text_based/utils.test.ts
+++ b/x-pack/plugins/lens/public/datasources/text_based/utils.test.ts
@ -189,6 +189,14 @@ describe('Text based languages utils', () => {
              timeFieldName: 'timeField',
            })
          ),
+          create: jest.fn().mockReturnValue(
+            Promise.resolve({
+              id: '1',
+              title: 'my-fake-index-pattern',
+              timeFieldName: 'timeField',
+              isPersisted: () => false,
+            })
+          ),
        },
        dataMock,
        expressionsMock
@ -281,5 +289,162 @@ describe('Text based languages utils', () => {
        },
      });
    });
+
+    it('should return the correct state for not existing dataview', async () => {
+      const state = {
+        layers: {
+          first: {
+            allColumns: [],
+            columns: [],
+            query: undefined,
+            index: '',
+          },
+        },
+        indexPatternRefs: [],
+        fieldList: [],
+        initialContext: {
+          contextualFields: ['bytes', 'dest'],
+          query: { sql: 'SELECT * FROM "foo"' },
+          fieldName: '',
+          dataViewSpec: {
+            title: 'foo',
+            id: '1',
+            name: 'Foo',
+          },
+        },
+      };
+      const dataViewsMock = dataViewPluginMocks.createStartContract();
+      const dataMock = dataPluginMock.createStartContract();
+      const expressionsMock = expressionsPluginMock.createStartContract();
+      const updatedState = await getStateFromAggregateQuery(
+        state,
+        { sql: 'SELECT * FROM my-fake-index-*' },
+        {
+          ...dataViewsMock,
+          getIdsWithTitle: jest.fn().mockReturnValue(
+            Promise.resolve([
+              { id: '1', title: 'my-fake-index-pattern' },
+              { id: '2', title: 'my-fake-restricted-pattern' },
+              { id: '3', title: 'my-compatible-pattern' },
+            ])
+          ),
+          get: jest.fn().mockReturnValue(
+            Promise.resolve({
+              id: '1',
+              title: 'my-fake-index-pattern',
+              timeFieldName: 'timeField',
+            })
+          ),
+          create: jest.fn().mockReturnValue(
+            Promise.resolve({
+              id: 'adHoc-id',
+              title: 'my-fake-index-*',
+              name: 'my-fake-index-*',
+              timeFieldName: 'timeField',
+              isPersisted: () => false,
+              fields: {
+                getByName: jest.fn().mockReturnValue({
+                  type: 'date',
+                }),
+              },
+            })
+          ),
+        },
+        dataMock,
+        expressionsMock
+      );
+
+      expect(updatedState).toStrictEqual({
+        initialContext: {
+          contextualFields: ['bytes', 'dest'],
+          query: { sql: 'SELECT * FROM "foo"' },
+          fieldName: '',
+          dataViewSpec: {
+            title: 'foo',
+            id: '1',
+            name: 'Foo',
+          },
+        },
+        fieldList: [
+          {
+            name: 'timestamp',
+            id: 'timestamp',
+            meta: {
+              type: 'date',
+            },
+          },
+          {
+            name: 'bytes',
+            id: 'bytes',
+            meta: {
+              type: 'number',
+            },
+          },
+          {
+            name: 'memory',
+            id: 'memory',
+            meta: {
+              type: 'number',
+            },
+          },
+        ],
+        indexPatternRefs: [
+          {
+            id: '3',
+            timeField: 'timeField',
+            title: 'my-compatible-pattern',
+          },
+          {
+            id: '1',
+            timeField: 'timeField',
+            title: 'my-fake-index-pattern',
+          },
+          {
+            id: '2',
+            timeField: 'timeField',
+            title: 'my-fake-restricted-pattern',
+          },
+          {
+            id: 'adHoc-id',
+            timeField: '@timestamp',
+            title: 'my-fake-index-*',
+          },
+        ],
+        layers: {
+          first: {
+            allColumns: [
+              {
+                fieldName: 'timestamp',
+                columnId: 'timestamp',
+                meta: {
+                  type: 'date',
+                },
+              },
+              {
+                fieldName: 'bytes',
+                columnId: 'bytes',
+                meta: {
+                  type: 'number',
+                },
+              },
+              {
+                fieldName: 'memory',
+                columnId: 'memory',
+                meta: {
+                  type: 'number',
+                },
+              },
+            ],
+            columns: [],
+            errors: [],
+            index: 'adHoc-id',
+            query: {
+              sql: 'SELECT * FROM my-fake-index-*',
+            },
+            timeField: '@timestamp',
+          },
+        },
+      });
+    });
  });
 });
--- a/x-pack/plugins/lens/public/datasources/text_based/utils.ts
+++ b/x-pack/plugins/lens/public/datasources/text_based/utils.ts
@ -14,6 +14,7 @@ import { generateId } from '../../id_generator';
 import { fetchDataFromAggregateQuery } from './fetch_data_from_aggregate_query';

 import type { IndexPatternRef, TextBasedPrivateState, TextBasedLayerColumn } from './types';
+import type { DataViewsState } from '../../state_management';

 export async function loadIndexPatternRefs(
  indexPatternsService: DataViewsPublicPluginStart
@ -64,9 +65,12 @@ export async function getStateFromAggregateQuery(
  query: AggregateQuery,
  dataViews: DataViewsPublicPluginStart,
  data: DataPublicPluginStart,
-  expressions: ExpressionsStart
+  expressions: ExpressionsStart,
+  frameDataViews?: DataViewsState
 ) {
-  const indexPatternRefs: IndexPatternRef[] = await loadIndexPatternRefs(dataViews);
+  let indexPatternRefs: IndexPatternRef[] = frameDataViews?.indexPatternRefs.length
+    ? frameDataViews.indexPatternRefs
+    : await loadIndexPatternRefs(dataViews);
  const errors: Error[] = [];
  const layerIds = Object.keys(state.layers);
  const context = state.initialContext;
@ -74,14 +78,37 @@ export async function getStateFromAggregateQuery(
  // fetch the pattern from the query
  const indexPattern = getIndexPatternFromTextBasedQuery(query);
  // get the id of the dataview
-  const index = indexPatternRefs.find((r) => r.title === indexPattern)?.id ?? '';
+  let dataViewId = indexPatternRefs.find((r) => r.title === indexPattern)?.id ?? '';
  let columnsFromQuery: DatatableColumn[] = [];
  let allColumns: TextBasedLayerColumn[] = [];
  let timeFieldName;
  try {
-    const table = await fetchDataFromAggregateQuery(query, dataViews, data, expressions);
-    const dataView = await dataViews.get(index);
+    const dataView = dataViewId
+      ? await dataViews.get(dataViewId)
+      : await dataViews.create({
+          title: indexPattern,
+        });
+    if (!dataViewId && !dataView.isPersisted()) {
+      if (dataView && dataView.id) {
+        if (dataView.fields.getByName('@timestamp')?.type === 'date') {
+          dataView.timeFieldName = '@timestamp';
+        } else if (dataView.fields.getByType('date')?.length) {
+          const dateFields = dataView.fields.getByType('date');
+          dataView.timeFieldName = dateFields[0].name;
+        }
+        dataViewId = dataView?.id;
+        indexPatternRefs = [
+          ...indexPatternRefs,
+          {
+            id: dataView.id,
+            title: dataView.name,
+            timeField: dataView.timeFieldName,
+          },
+        ];
+      }
+    }
    timeFieldName = dataView.timeFieldName;
+    const table = await fetchDataFromAggregateQuery(query, dataView, data, expressions);
    columnsFromQuery = table?.columns ?? [];
    allColumns = getAllColumns(state.layers[newLayerId].allColumns, columnsFromQuery);
  } catch (e) {
@ -91,7 +118,7 @@ export async function getStateFromAggregateQuery(
  const tempState = {
    layers: {
      [newLayerId]: {
-        index,
+        index: dataViewId,
        query,
        columns: state.layers[newLayerId].columns ?? [],
        allColumns,
--- a/x-pack/test/functional/apps/discover/visualize_field.ts
+++ b/x-pack/test/functional/apps/discover/visualize_field.ts
@ -147,5 +147,23 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
        expect(await dimensions[1].getVisibleText()).to.be('average');
      });
    });
+
+    it('should visualize correctly text based language queries based on index patterns', async () => {
+      await PageObjects.discover.selectTextBaseLang('SQL');
+      await PageObjects.header.waitUntilLoadingHasFinished();
+      await monacoEditor.setCodeEditorValue(
+        'SELECT extension, AVG("bytes") as average FROM "logstash*" GROUP BY extension'
+      );
+      await testSubjects.click('querySubmitButton');
+      await PageObjects.header.waitUntilLoadingHasFinished();
+
+      await testSubjects.click('textBased-visualize');
+
+      await retry.try(async () => {
+        const dimensions = await testSubjects.findAll('lns-dimensionTrigger-textBased');
+        expect(dimensions).to.have.length(2);
+        expect(await dimensions[1].getVisibleText()).to.be('average');
+      });
+    });
  });
 }
--- a/x-pack/test/functional/apps/lens/group1/text_based_languages.ts
+++ b/x-pack/test/functional/apps/lens/group1/text_based_languages.ts
@ -51,8 +51,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
      await switchToTextBasedLanguage('SQL');
      expect(await testSubjects.exists('showQueryBarMenu')).to.be(false);
      expect(await testSubjects.exists('addFilter')).to.be(false);
+      await testSubjects.click('unifiedTextLangEditor-expand');
      const textBasedQuery = await monacoEditor.getCodeEditorValue();
      expect(textBasedQuery).to.be('SELECT * FROM "log*"');
+      await testSubjects.click('unifiedTextLangEditor-minimize');
    });

    it('should allow adding and using a field', async () => {
@ -137,5 +139,24 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
      expect(await testSubjects.exists('addFilter')).to.be(true);
      expect(await queryBar.getQueryString()).to.be('');
    });
+
+    it('should allow using an index pattern that is not translated to a dataview', async () => {
+      await switchToTextBasedLanguage('SQL');
+      await testSubjects.click('unifiedTextLangEditor-expand');
+      await monacoEditor.setCodeEditorValue(
+        'SELECT extension, AVG("bytes") as average FROM "logstash*" GROUP BY extension'
+      );
+      await testSubjects.click('querySubmitButton');
+      await PageObjects.header.waitUntilLoadingHasFinished();
+      await PageObjects.lens.switchToVisualization('lnsMetric');
+      await PageObjects.lens.configureTextBasedLanguagesDimension({
+        dimension: 'lnsMetric_primaryMetricDimensionPanel > lns-empty-dimension',
+        field: 'average',
+      });
+
+      await PageObjects.lens.waitForVisualization('mtrVis');
+      const metricData = await PageObjects.lens.getMetricVisualizationData();
+      expect(metricData[0].title).to.eql('average');
+    });
  });
 }