[ES|QL] Retrieve the indices from AST parsing (#181271)

## Summary

Closes https://github.com/elastic/kibana/issues/180959

Retrieves the indices from ast parsing. This ensures that the index
patterns we get from the `from` command is always the correct one. I
have replaced it everywhere expect from specific places where I still
use the deprecated function. I am not sure how to test the app and I
don't want to cause regressions so I prefer the responsible teams to do
the migration.


Before

Could not retrieve the index correctly
<img width="1677" alt="image"
src="77cdac00-ffff-4b91-88ba-0fc523c5f54d">

After
Correct retrieval of the index and the @timestamp info

<img width="1067" alt="image"
src="bc14718a-30f5-4f3c-8a56-cf57f69cff14">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: dej611 <dej611@gmail.com>

packages/kbn-esql-ast/src/ast_parser.ts

@@ -12,10 +12,10 @@ import { getParser, ROOT_STATEMENT } from './antlr_facade';
 import { AstListener } from './ast_factory';
 import type { ESQLAst, EditorError } from './types';
 
-export async function getAstAndSyntaxErrors(text: string | undefined): Promise<{
+export function getAstAndSyntaxErrors(text: string | undefined): {
   errors: EditorError[];
   ast: ESQLAst;
-}> {
+} {
   if (text == null) {
     return { ast: [], errors: [] };
   }

packages/kbn-esql-ast/src/types.ts

@@ -113,10 +113,12 @@ export interface ESQLMessage {
   code: string;
 }
 
-export type AstProviderFn = (text: string | undefined) => Promise<{
-  ast: ESQLAst;
-  errors: EditorError[];
-}>;
+export type AstProviderFn = (text: string | undefined) =>
+  | Promise<{
+      ast: ESQLAst;
+      errors: EditorError[];
+    }>
+  | { ast: ESQLAst; errors: EditorError[] };
 
 export interface EditorError {
   startLineNumber: number;

packages/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -69,21 +69,27 @@ describe('sql/esql query helpers', () => {
       expect(idxPattern6).toBe('foo-1,foo-2');
 
       const idxPattern7 = getIndexPatternFromESQLQuery('from foo-1, foo-2 | limit 2');
-      expect(idxPattern7).toBe('foo-1, foo-2');
+      expect(idxPattern7).toBe('foo-1,foo-2');
 
       const idxPattern8 = getIndexPatternFromESQLQuery('FROM foo-1,  foo-2');
-      expect(idxPattern8).toBe('foo-1,  foo-2');
+      expect(idxPattern8).toBe('foo-1,foo-2');
 
       const idxPattern9 = getIndexPatternFromESQLQuery('FROM foo-1, foo-2 [metadata _id]');
-      expect(idxPattern9).toBe('foo-1, foo-2');
+      expect(idxPattern9).toBe('foo-1,foo-2');
 
       const idxPattern10 = getIndexPatternFromESQLQuery('FROM foo-1, remote_cluster:foo-2, foo-3');
-      expect(idxPattern10).toBe('foo-1, remote_cluster:foo-2, foo-3');
+      expect(idxPattern10).toBe('foo-1,remote_cluster:foo-2,foo-3');
 
       const idxPattern11 = getIndexPatternFromESQLQuery(
         'FROM foo-1, foo-2 | where event.reason like "*Disable: changed from [true] to [false]*"'
       );
-      expect(idxPattern11).toBe('foo-1, foo-2');
+      expect(idxPattern11).toBe('foo-1,foo-2');
+
+      const idxPattern12 = getIndexPatternFromESQLQuery('FROM foo-1, foo-2 // from command used');
+      expect(idxPattern12).toBe('foo-1,foo-2');
+
+      const idxPattern13 = getIndexPatternFromESQLQuery('ROW a = 1, b = "two", c = null');
+      expect(idxPattern13).toBe('');
     });
   });
 

packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -5,6 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+import { type ESQLSource, getAstAndSyntaxErrors } from '@kbn/esql-ast';
 
 const DEFAULT_ESQL_LIMIT = 500;
 
@@ -25,22 +26,13 @@ export function getIndexPatternFromSQLQuery(sqlQuery?: string): string {
   return '';
 }
 
-// retrieves the index pattern from the aggregate query for ES|QL
-export function getIndexPatternFromESQLQuery(esql?: string): string {
-  let fromPipe = (esql || '').split('|')[0];
-  const splitFroms = fromPipe?.split(new RegExp(/FROM\s/, 'ig'));
-  const fromsLength = splitFroms?.length ?? 0;
-  if (splitFroms && splitFroms?.length > 2) {
-    fromPipe = `${splitFroms[fromsLength - 2]} FROM ${splitFroms[fromsLength - 1]}`;
-  }
-  const parsedString = fromPipe?.replaceAll('`', '');
-  // case insensitive match for the index pattern
-  const regex = new RegExp(/FROM\s+([(\w*:)?\w*-.!@$^()~;\s]+)/, 'i');
-  const matches = parsedString?.match(regex);
-  if (matches) {
-    return matches[1]?.trim();
-  }
-  return '';
+// retrieves the index pattern from the aggregate query for ES|QL using ast parsing
+export function getIndexPatternFromESQLQuery(esql?: string) {
+  const { ast } = getAstAndSyntaxErrors(esql);
+  const fromCommand = ast.find(({ name }) => name === 'from');
+  const args = (fromCommand?.args ?? []) as ESQLSource[];
+  const indices = args.filter((arg) => arg.sourceType === 'index');
+  return indices?.map((index) => index.text).join(',');
 }
 
 export function getLimitFromESQLQuery(esql: string): number {

packages/kbn-esql-utils/tsconfig.json

@@ -19,5 +19,6 @@
     "@kbn/data-views-plugin",
     "@kbn/crypto-browser",
     "@kbn/data-view-utils",
+    "@kbn/esql-ast",
   ]
 }

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -255,7 +255,7 @@ describe('autocomplete', () => {
           statement,
           offset,
           context,
-          async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
+          async (text) => (text ? getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
           callbackMocks
         );
         const suggestionInertTextSorted = suggestions
@@ -1190,7 +1190,7 @@ describe('autocomplete', () => {
         statement,
         triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
       expect(callbackMocks.getFieldsFor).toHaveBeenCalledWith({
@@ -1206,7 +1206,7 @@ describe('autocomplete', () => {
         statement,
         triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
       expect(callbackMocks.getFieldsFor).toHaveBeenCalledWith({ query: 'from a' });
@@ -1222,7 +1222,7 @@ describe('autocomplete', () => {
         statement,
         triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
     }

packages/kbn-monaco/src/esql/lib/hover/hover.test.ts

@@ -124,7 +124,7 @@ describe('hover', () => {
           model,
           position,
           token,
-          async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
+          async (text) => (text ? getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
           callbackMocks
         );
         expect(contents.map(({ value }) => value)).toEqual(expected);

packages/kbn-monaco/src/esql/worker/esql_worker.ts

@@ -60,8 +60,8 @@ export class ESQLWorker implements BaseWorkerDefinition {
     return [];
   }
 
-  async getAst(text: string | undefined) {
-    const rawAst = await getAstAndSyntaxErrors(text);
+  getAst(text: string | undefined) {
+    const rawAst = getAstAndSyntaxErrors(text);
     return {
       ast: rawAst.ast,
       errors: rawAst.errors.map(inlineToMonacoErrors),

packages/kbn-ui-shared-deps-src/BUILD.bazel

@@ -25,6 +25,7 @@ webpack_cli(
     "//packages/kbn-ui-theme",
     "//packages/kbn-i18n",
     "//packages/kbn-i18n-react",
+    "//packages/kbn-esql-ast",
     "//packages/kbn-monaco",
     "//packages/kbn-datemath",
     "//packages/kbn-analytics",

packages/kbn-ui-shared-deps-src/src/definitions.js

@@ -98,6 +98,7 @@ const externals = {
   '@tanstack/react-query': '__kbnSharedDeps__.ReactQuery',
   '@tanstack/react-query-devtools': '__kbnSharedDeps__.ReactQueryDevtools',
   '@kbn/code-editor': '__kbnSharedDeps__.KbnCodeEditor',
+  '@kbn/esql-ast': '__kbnSharedDeps__.KbnEsqlAst',
 };
 
 module.exports = { distDir, jsFilename, cssDistFilename, externals };

packages/kbn-ui-shared-deps-src/src/entry.js

@@ -74,3 +74,4 @@ export const Classnames = require('classnames');
 export const ReactQuery = require('@tanstack/react-query');
 export const ReactQueryDevtools = require('@tanstack/react-query-devtools');
 export const KbnCodeEditor = require('@kbn/code-editor');
+export const KbnEsqlAst = require('@kbn/esql-ast');