[8.x] [ES|QL] Separate `FROM` autocomplete routine (#210465) (#210944)

# Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] Separate `FROM` autocomplete routine (#210465)](https://github.com/elastic/kibana/pull/210465)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Drew Tate <drew.tate@elastic.co>
2025-04-23 09:19:04 -04:00 · 2025-02-13 13:01:37 +11:00 · 2025-02-13 13:01:37 +11:00 · 1e872526ec
commit 1e872526ec
parent b8c2230bee
16 changed files with 477 additions and 276 deletions
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.from.test.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.from.test.ts
@ -40,12 +40,10 @@ describe('autocomplete.suggest', () => {
        await assertSuggestions('from /index', visibleIndices);
      });

-      test('suggests visible indices on comma', async () => {
+      test("doesn't create suggestions after an open quote", async () => {
        const { assertSuggestions } = await setup();

-        await assertSuggestions('FROM a,/', visibleIndices);
-        await assertSuggestions('FROM a, /', visibleIndices);
-        await assertSuggestions('from *,/', visibleIndices);
+        await assertSuggestions('FROM " /"', []);
      });

      test('can suggest integration data sources', async () => {
@ -72,7 +70,7 @@ describe('autocomplete.suggest', () => {
    describe('... METADATA <fields>', () => {
      const metadataFieldsAndIndex = metadataFields.filter((field) => field !== '_index');

-      test('on <kbd>SPACE</kbd> without comma ",", suggests adding metadata', async () => {
+      test('on <// FROM something METADATA field1, /kbd>SPACE</kbd> without comma ",", suggests adding metadata', async () => {
        const recommendedQueries = getRecommendedQueries({
          fromCommand: '',
          timeField: 'dateField',
@ -88,12 +86,31 @@ describe('autocomplete.suggest', () => {
        await assertSuggestions('from a, b /', expected);
      });

+      test('partially-typed METADATA keyword', async () => {
+        const { assertSuggestions } = await setup();
+
+        assertSuggestions('FROM index1 MET/', ['METADATA $0']);
+      });
+
+      test('not before first index', async () => {
+        const { assertSuggestions } = await setup();
+
+        assertSuggestions('FROM MET/', visibleIndices);
+      });
+
      test('on <kbd>SPACE</kbd> after "METADATA" keyword suggests all metadata fields', async () => {
        const { assertSuggestions } = await setup();

        await assertSuggestions('from a, b METADATA /', metadataFields);
      });

+      test('metadata field prefixes', async () => {
+        const { assertSuggestions } = await setup();
+
+        await assertSuggestions('from a, b METADATA _/', metadataFields);
+        await assertSuggestions('from a, b METADATA _sour/', metadataFields);
+      });
+
      test('on <kbd>SPACE</kbd> after "METADATA" column suggests command and pipe operators', async () => {
        const { assertSuggestions } = await setup();

--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
@ -368,11 +368,10 @@ describe('autocomplete', () => {

  // @TODO: get updated eval block from main
  describe('values suggestions', () => {
-    testSuggestions('FROM "i/"', ['index'], undefined, [, [{ name: 'index', hidden: false }]]);
-    testSuggestions('FROM "index/"', ['index'], undefined, [, [{ name: 'index', hidden: false }]]);
-    // TODO — re-enable these tests when we can support this case
-    testSuggestions.skip('FROM "  a/"', []);
-    testSuggestions.skip('FROM "foo b/"', []);
+    testSuggestions('FROM "i/"', []);
+    testSuggestions('FROM "index/"', []);
+    testSuggestions('FROM "  a/"', []);
+    testSuggestions('FROM "foo b/"', []);
    testSuggestions('FROM a | WHERE tags == " /"', [], ' ');
    testSuggestions('FROM a | WHERE tags == """ /"""', [], ' ');
    testSuggestions('FROM a | WHERE tags == "a/"', []);
@ -497,12 +496,7 @@ describe('autocomplete', () => {

    // FROM source METADATA
    recommendedQuerySuggestions = getRecommendedQueriesSuggestions('', 'dateField');
-    testSuggestions('FROM index1 M/', [
-      ',',
-      'METADATA $0',
-      '| ',
-      ...recommendedQuerySuggestions.map((q) => q.queryString),
-    ]);
+    testSuggestions('FROM index1 M/', ['METADATA $0']);

    // FROM source METADATA field
    testSuggestions('FROM index1 METADATA _/', METADATA_FIELDS);
@ -890,12 +884,7 @@ describe('autocomplete', () => {

    recommendedQuerySuggestions = getRecommendedQueriesSuggestions('', 'dateField');
    // FROM source METADATA
-    testSuggestions('FROM index1 M/', [
-      ',',
-      attachAsSnippet(attachTriggerCommand('METADATA $0')),
-      '| ',
-      ...recommendedQuerySuggestions.map((q) => q.queryString),
-    ]);
+    testSuggestions('FROM index1 M/', [attachAsSnippet(attachTriggerCommand('METADATA $0'))]);

    describe('ENRICH', () => {
      testSuggestions(
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
@ -8,13 +8,13 @@
 */

 import { uniq, uniqBy } from 'lodash';
-import type {
-  AstProviderFn,
-  ESQLAstItem,
-  ESQLCommand,
-  ESQLCommandOption,
-  ESQLFunction,
-  ESQLSingleAstItem,
+import {
+  type AstProviderFn,
+  type ESQLAstItem,
+  type ESQLCommand,
+  type ESQLCommandOption,
+  type ESQLFunction,
+  type ESQLSingleAstItem,
 } from '@kbn/esql-ast';
 import { ESQL_NUMBER_TYPES, isNumericType } from '../shared/esql_types';
 import type { EditorContext, ItemKind, SuggestionRawDefinition, GetColumnsByTypeFn } from './types';
@ -44,7 +44,6 @@ import {
  noCaseCompare,
  correctQuerySyntax,
  getColumnByName,
-  sourceExists,
  findFinalWord,
  getAllCommands,
  getExpressionType,
@ -63,7 +62,6 @@ import {
 import {
  buildFieldsDefinitions,
  buildPoliciesDefinitions,
-  buildSourcesDefinitions,
  getNewVariableSuggestion,
  buildNoPoliciesAvailableDefinition,
  getFunctionSuggestions,
@ -80,7 +78,7 @@ import {
  getOperatorSuggestions,
  getSuggestionsAfterNot,
 } from './factories';
-import { EDITOR_MARKER, FULL_TEXT_SEARCH_FUNCTIONS, METADATA_FIELDS } from '../shared/constants';
+import { EDITOR_MARKER, FULL_TEXT_SEARCH_FUNCTIONS } from '../shared/constants';
 import { getAstContext, removeMarkerArgFromArgsList } from '../shared/context';
 import {
  buildQueryUntilPreviousCommand,
@ -99,7 +97,6 @@ import {
  getQueryForFields,
  getSourcesFromCommands,
  isAggFunctionUsedAlready,
-  removeQuoteForSuggestedSources,
  getValidSignaturesAndTypesToSuggestNext,
  handleFragment,
  getFieldsOrFunctionsSuggestions,
@ -109,7 +106,6 @@ import {
  checkFunctionInvocationComplete,
 } from './helper';
 import { FunctionParameter, isParameterType } from '../definitions/types';
-import { metadataOption } from '../definitions/options';
 import { comparisonFunctions } from '../definitions/builtin';
 import { getRecommendedQueriesSuggestions } from './recommended_queries/suggestions';

@ -213,7 +209,8 @@ export async function suggest(

  if (
    astContext.type === 'expression' ||
-    (astContext.type === 'option' && astContext.command?.name === 'join')
+    (astContext.type === 'option' && astContext.command?.name === 'join') ||
+    (astContext.type === 'option' && astContext.command?.name === 'from')
  ) {
    return getSuggestionsWithinCommandExpression(
      innerText,
@ -313,17 +310,6 @@ function getPolicyRetriever(resourceRetriever?: ESQLCallbacks) {
  };
 }

-function getSourceSuggestions(sources: ESQLSourceResult[]) {
-  // hide indexes that start with .
-  return buildSourcesDefinitions(
-    sources
-      .filter(({ hidden }) => !hidden)
-      .map(({ name, dataStreams, title, type }) => {
-        return { name, isIntegration: Boolean(dataStreams && dataStreams.length), title, type };
-      })
-  );
-}
-
 function findNewVariable(variables: Map<string, ESQLVariable[]>) {
  let autoGeneratedVariableCounter = 0;
  let name = `var${autoGeneratedVariableCounter++}`;
@ -422,19 +408,23 @@ async function getSuggestionsWithinCommandExpression(
  const references = { fields: fieldsMap, variables: anyVariables };
  if (commandDef.suggest) {
    // The new path.
-    return commandDef.suggest(
+    return commandDef.suggest({
      innerText,
      command,
      getColumnsByType,
-      (col: string) => Boolean(getColumnByName(col, references)),
-      () => findNewVariable(anyVariables),
-      (expression: ESQLAstItem | undefined) =>
+      columnExists: (col: string) => Boolean(getColumnByName(col, references)),
+      getSuggestedVariableName: () => findNewVariable(anyVariables),
+      getExpressionType: (expression: ESQLAstItem | undefined) =>
        getExpressionType(expression, references.fields, references.variables),
      getPreferences,
-      commands,
-      commandDef,
-      callbacks
-    );
+      definition: commandDef,
+      getSources,
+      getRecommendedQueriesSuggestions: (prefix) =>
+        getRecommendedQueriesSuggestions(getColumnsByType, prefix),
+      getSourcesFromQuery: (type) => getSourcesFromCommands(commands, type),
+      previousCommands: commands,
+      callbacks,
+    });
  } else {
    // The deprecated path.
    return getExpressionSuggestionsByType(
@ -484,12 +474,6 @@ async function getExpressionSuggestionsByType(
    return [];
  }

-  // TODO - this is a workaround because it was too difficult to handle this case in a generic way :(
-  if (commandDef.name === 'from' && node && isSourceItem(node) && /\s/.test(node.name)) {
-    // FROM " <suggest>"
-    return [];
-  }
-
  // A new expression is considered either
  // * just after a command name => i.e. ... | STATS <here>
  // * or after a comma => i.e. STATS fieldA, <here>
@ -522,7 +506,6 @@ async function getExpressionSuggestionsByType(
    const optArg = optionsAlreadyDeclared.find(({ name: optionName }) => optionName === name);
    return (!optArg && !optionsAlreadyDeclared.length) || (optArg && index > optArg.index);
  });
-  const hasRecommendedQueries = Boolean(commandDef?.hasRecommendedQueries);
  // get the next definition for the given command
  let argDef = commandDef.signature.params[argIndex];
  // tune it for the variadic case
@ -903,82 +886,6 @@ async function getExpressionSuggestionsByType(
          });
        }
        suggestions.push(...(policies.length ? policies : [buildNoPoliciesAvailableDefinition()]));
-      } else {
-        const indexes = getSourcesFromCommands(commands, 'index');
-        const lastIndex = indexes[indexes.length - 1];
-        const canRemoveQuote = isNewExpression && innerText.includes('"');
-        // Function to add suggestions based on canRemoveQuote
-        const addSuggestionsBasedOnQuote = async (definitions: SuggestionRawDefinition[]) => {
-          suggestions.push(
-            ...(canRemoveQuote ? removeQuoteForSuggestedSources(definitions) : definitions)
-          );
-        };
-
-        if (lastIndex && lastIndex.text && lastIndex.text !== EDITOR_MARKER) {
-          const sources = await getSources();
-
-          const recommendedQueriesSuggestions = hasRecommendedQueries
-            ? await getRecommendedQueriesSuggestions(getFieldsByType)
-            : [];
-
-          const suggestionsToAdd = await handleFragment(
-            innerText,
-            (fragment) =>
-              sourceExists(fragment, new Set(sources.map(({ name: sourceName }) => sourceName))),
-            (_fragment, rangeToReplace) => {
-              return getSourceSuggestions(sources).map((suggestion) => ({
-                ...suggestion,
-                rangeToReplace,
-              }));
-            },
-            (fragment, rangeToReplace) => {
-              const exactMatch = sources.find(({ name: _name }) => _name === fragment);
-              if (exactMatch?.dataStreams) {
-                // this is an integration name, suggest the datastreams
-                const definitions = buildSourcesDefinitions(
-                  exactMatch.dataStreams.map(({ name }) => ({ name, isIntegration: false }))
-                );
-
-                return canRemoveQuote ? removeQuoteForSuggestedSources(definitions) : definitions;
-              } else {
-                const _suggestions: SuggestionRawDefinition[] = [
-                  {
-                    ...pipeCompleteItem,
-                    filterText: fragment,
-                    text: fragment + ' | ',
-                    command: TRIGGER_SUGGESTION_COMMAND,
-                    rangeToReplace,
-                  },
-                  {
-                    ...commaCompleteItem,
-                    filterText: fragment,
-                    text: fragment + ', ',
-                    command: TRIGGER_SUGGESTION_COMMAND,
-                    rangeToReplace,
-                  },
-                  {
-                    ...buildOptionDefinition(metadataOption),
-                    filterText: fragment,
-                    text: fragment + ' METADATA ',
-                    asSnippet: false, // turn this off because $ could be contained within the source name
-                    rangeToReplace,
-                  },
-                  ...recommendedQueriesSuggestions.map((suggestion) => ({
-                    ...suggestion,
-                    rangeToReplace,
-                    filterText: fragment,
-                    text: fragment + suggestion.text,
-                  })),
-                ];
-                return _suggestions;
-              }
-            }
-          );
-          addSuggestionsBasedOnQuote(suggestionsToAdd);
-        } else {
-          // FROM <suggest> or no index/text
-          await addSuggestionsBasedOnQuote(getSourceSuggestions(await getSources()));
-        }
      }
    }
  }
@ -1021,11 +928,6 @@ async function getExpressionSuggestionsByType(
      }));
      suggestions.push(...finalSuggestions);
    }
-
-    // handle recommended queries for from
-    if (hasRecommendedQueries) {
-      suggestions.push(...(await getRecommendedQueriesSuggestions(getFieldsByType)));
-    }
  }
  // Due to some logic overlapping functions can be repeated
  // so dedupe here based on text string (it can differ from name)
@ -1508,53 +1410,6 @@ async function getOptionArgsSuggestions(
    }
  }

-  if (option.name === 'metadata') {
-    const existingFields = new Set(option.args.filter(isColumnItem).map(({ name }) => name));
-    const filteredMetaFields = METADATA_FIELDS.filter((name) => !existingFields.has(name));
-    if (isNewExpression) {
-      suggestions.push(
-        ...(await handleFragment(
-          innerText,
-          (fragment) => METADATA_FIELDS.includes(fragment),
-          (_fragment, rangeToReplace) =>
-            buildFieldsDefinitions(filteredMetaFields).map((suggestion) => ({
-              ...suggestion,
-              rangeToReplace,
-            })),
-          (fragment, rangeToReplace) => {
-            const _suggestions = [
-              {
-                ...pipeCompleteItem,
-                text: fragment + ' | ',
-                filterText: fragment,
-                command: TRIGGER_SUGGESTION_COMMAND,
-                rangeToReplace,
-              },
-            ];
-            if (filteredMetaFields.length > 1) {
-              _suggestions.push({
-                ...commaCompleteItem,
-                text: fragment + ', ',
-                filterText: fragment,
-                command: TRIGGER_SUGGESTION_COMMAND,
-                rangeToReplace,
-              });
-            }
-            return _suggestions;
-          }
-        ))
-      );
-    } else {
-      if (existingFields.size > 0) {
-        // METADATA field <suggest>
-        if (filteredMetaFields.length > 0) {
-          suggestions.push(commaCompleteItem);
-        }
-        suggestions.push(pipeCompleteItem);
-      }
-    }
-  }
-
  if (optionDef) {
    if (!suggestions.length) {
      const argDefIndex = optionDef.signature.multipleParams
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/drop/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/drop/index.ts
@ -7,24 +7,24 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import type { ESQLCommand } from '@kbn/esql-ast';
+import { CommandSuggestParams } from '../../../definitions/types';
 import {
  findPreviousWord,
  getLastNonWhitespaceChar,
  isColumnItem,
  noCaseCompare,
 } from '../../../shared/helpers';
-import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
+import type { SuggestionRawDefinition } from '../../types';
 import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
 import { handleFragment } from '../../helper';
 import { TRIGGER_SUGGESTION_COMMAND } from '../../factories';

-export async function suggest(
-  innerText: string,
-  command: ESQLCommand<'drop'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  columnExists: (column: string) => boolean
-): Promise<SuggestionRawDefinition[]> {
+export async function suggest({
+  innerText,
+  getColumnsByType,
+  command,
+  columnExists,
+}: CommandSuggestParams<'drop'>): Promise<SuggestionRawDefinition[]> {
  if (
    /\s/.test(innerText[innerText.length - 1]) &&
    getLastNonWhitespaceChar(innerText) !== ',' &&
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/from/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/from/index.ts
@ -0,0 +1,218 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ESQLCommandOption } from '@kbn/esql-ast';
+import { isMarkerNode } from '../../../shared/context';
+import { metadataOption } from '../../../definitions/options';
+import type { SuggestionRawDefinition } from '../../types';
+import { getOverlapRange, handleFragment, removeQuoteForSuggestedSources } from '../../helper';
+import { CommandSuggestParams } from '../../../definitions/types';
+import {
+  isColumnItem,
+  isOptionItem,
+  isRestartingExpression,
+  isSingleItem,
+  sourceExists,
+} from '../../../shared/helpers';
+import {
+  TRIGGER_SUGGESTION_COMMAND,
+  buildFieldsDefinitions,
+  buildOptionDefinition,
+  buildSourcesDefinitions,
+} from '../../factories';
+import { ESQLSourceResult } from '../../../shared/types';
+import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
+import { METADATA_FIELDS } from '../../../shared/constants';
+
+export async function suggest({
+  innerText,
+  command,
+  getSources,
+  getRecommendedQueriesSuggestions,
+  getSourcesFromQuery,
+}: CommandSuggestParams<'from'>): Promise<SuggestionRawDefinition[]> {
+  if (/\".*$/.test(innerText)) {
+    // FROM "<suggest>"
+    return [];
+  }
+
+  const suggestions: SuggestionRawDefinition[] = [];
+
+  const indexes = getSourcesFromQuery('index');
+  const canRemoveQuote = innerText.includes('"');
+  // Function to add suggestions based on canRemoveQuote
+  const addSuggestionsBasedOnQuote = (definitions: SuggestionRawDefinition[]) => {
+    suggestions.push(
+      ...(canRemoveQuote ? removeQuoteForSuggestedSources(definitions) : definitions)
+    );
+  };
+
+  const metadataNode = command.args.find((arg) => isOptionItem(arg) && arg.name === 'metadata') as
+    | ESQLCommandOption
+    | undefined;
+
+  // FROM index METADATA ... /
+  if (metadataNode) {
+    return suggestForMetadata(metadataNode, innerText);
+  }
+
+  const metadataOverlap = getOverlapRange(innerText, 'METADATA');
+
+  // FROM /
+  if (indexes.length === 0) {
+    addSuggestionsBasedOnQuote(getSourceSuggestions(await getSources()));
+  }
+  // FROM something /
+  else if (indexes.length > 0 && /\s$/.test(innerText) && !isRestartingExpression(innerText)) {
+    suggestions.push(buildOptionDefinition(metadataOption));
+    suggestions.push(commaCompleteItem);
+    suggestions.push(pipeCompleteItem);
+    suggestions.push(...(await getRecommendedQueriesSuggestions()));
+  }
+  // FROM something MET/
+  else if (
+    indexes.length > 0 &&
+    /^FROM\s+\S+\s+/i.test(innerText) &&
+    metadataOverlap.start !== metadataOverlap.end
+  ) {
+    suggestions.push(buildOptionDefinition(metadataOption));
+  }
+  // FROM someth/
+  // FROM something/
+  // FROM something, /
+  else if (indexes.length) {
+    const sources = await getSources();
+
+    const recommendedQuerySuggestions = await getRecommendedQueriesSuggestions();
+
+    const suggestionsToAdd = await handleFragment(
+      innerText,
+      (fragment) =>
+        sourceExists(fragment, new Set(sources.map(({ name: sourceName }) => sourceName))),
+      (_fragment, rangeToReplace) => {
+        return getSourceSuggestions(sources).map((suggestion) => ({
+          ...suggestion,
+          rangeToReplace,
+        }));
+      },
+      (fragment, rangeToReplace) => {
+        const exactMatch = sources.find(({ name: _name }) => _name === fragment);
+        if (exactMatch?.dataStreams) {
+          // this is an integration name, suggest the datastreams
+          const definitions = buildSourcesDefinitions(
+            exactMatch.dataStreams.map(({ name }) => ({ name, isIntegration: false }))
+          );
+
+          return canRemoveQuote ? removeQuoteForSuggestedSources(definitions) : definitions;
+        } else {
+          const _suggestions: SuggestionRawDefinition[] = [
+            {
+              ...pipeCompleteItem,
+              filterText: fragment,
+              text: fragment + ' | ',
+              command: TRIGGER_SUGGESTION_COMMAND,
+              rangeToReplace,
+            },
+            {
+              ...commaCompleteItem,
+              filterText: fragment,
+              text: fragment + ', ',
+              command: TRIGGER_SUGGESTION_COMMAND,
+              rangeToReplace,
+            },
+            {
+              ...buildOptionDefinition(metadataOption),
+              filterText: fragment,
+              text: fragment + ' METADATA ',
+              asSnippet: false, // turn this off because $ could be contained within the source name
+              rangeToReplace,
+            },
+            ...recommendedQuerySuggestions.map((suggestion) => ({
+              ...suggestion,
+              rangeToReplace,
+              filterText: fragment,
+              text: fragment + suggestion.text,
+            })),
+          ];
+          return _suggestions;
+        }
+      }
+    );
+    addSuggestionsBasedOnQuote(suggestionsToAdd);
+  }
+
+  return suggestions;
+}
+
+function getSourceSuggestions(sources: ESQLSourceResult[]) {
+  // hide indexes that start with .
+  return buildSourcesDefinitions(
+    sources
+      .filter(({ hidden }) => !hidden)
+      .map(({ name, dataStreams, title, type }) => {
+        return { name, isIntegration: Boolean(dataStreams && dataStreams.length), title, type };
+      })
+  );
+}
+
+async function suggestForMetadata(metadata: ESQLCommandOption, innerText: string) {
+  const existingFields = new Set(metadata.args.filter(isColumnItem).map(({ name }) => name));
+  const filteredMetaFields = METADATA_FIELDS.filter((name) => !existingFields.has(name));
+  const suggestions: SuggestionRawDefinition[] = [];
+  // FROM something METADATA /
+  // FROM something METADATA field/
+  // FROM something METADATA field, /
+  if (
+    metadata.args.filter((arg) => isSingleItem(arg) && !isMarkerNode(arg)).length === 0 ||
+    isRestartingExpression(innerText)
+  ) {
+    suggestions.push(
+      ...(await handleFragment(
+        innerText,
+        (fragment) => METADATA_FIELDS.includes(fragment),
+        (_fragment, rangeToReplace) =>
+          buildFieldsDefinitions(filteredMetaFields).map((suggestion) => ({
+            ...suggestion,
+            rangeToReplace,
+          })),
+        (fragment, rangeToReplace) => {
+          const _suggestions = [
+            {
+              ...pipeCompleteItem,
+              text: fragment + ' | ',
+              filterText: fragment,
+              command: TRIGGER_SUGGESTION_COMMAND,
+              rangeToReplace,
+            },
+          ];
+          if (filteredMetaFields.length > 1) {
+            _suggestions.push({
+              ...commaCompleteItem,
+              text: fragment + ', ',
+              filterText: fragment,
+              command: TRIGGER_SUGGESTION_COMMAND,
+              rangeToReplace,
+            });
+          }
+          return _suggestions;
+        }
+      ))
+    );
+  } else {
+    // METADATA field /
+    if (existingFields.size > 0) {
+      if (filteredMetaFields.length > 0) {
+        suggestions.push(commaCompleteItem);
+      }
+      suggestions.push(pipeCompleteItem);
+    }
+  }
+
+  return suggestions;
+}
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/join/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/join/index.ts
@ -8,14 +8,14 @@
 */

 import { i18n } from '@kbn/i18n';
-import { type ESQLAstItem, ESQLCommand, mutate, LeafPrinter } from '@kbn/esql-ast';
+import { ESQLCommand, mutate, LeafPrinter } from '@kbn/esql-ast';
 import type { ESQLAstJoinCommand } from '@kbn/esql-ast';
 import type { ESQLCallbacks } from '../../../shared/types';
 import {
  CommandBaseDefinition,
  CommandDefinition,
+  CommandSuggestParams,
  CommandTypeDefinition,
-  type SupportedDataType,
 } from '../../../definitions/types';
 import {
  getPosition,
@ -96,18 +96,14 @@ const suggestFields = async (
  return [...intersection, ...union];
 };

-export const suggest: CommandBaseDefinition<'join'>['suggest'] = async (
-  innerText: string,
-  command: ESQLCommand<'join'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  columnExists: (column: string) => boolean,
-  getSuggestedVariableName: () => string,
-  getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-  getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
-  previousCommands?: ESQLCommand[],
-  definition?: CommandDefinition<'join'>,
-  callbacks?: ESQLCallbacks
-): Promise<SuggestionRawDefinition[]> => {
+export const suggest: CommandBaseDefinition<'join'>['suggest'] = async ({
+  innerText,
+  command,
+  getColumnsByType,
+  definition,
+  callbacks,
+  previousCommands,
+}: CommandSuggestParams<'join'>): Promise<SuggestionRawDefinition[]> => {
  let commandText: string = innerText;

  if (command.location) {
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/keep/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/keep/index.ts
@ -7,24 +7,24 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import type { ESQLCommand } from '@kbn/esql-ast';
+import { CommandSuggestParams } from '../../../definitions/types';
 import {
  findPreviousWord,
  getLastNonWhitespaceChar,
  isColumnItem,
  noCaseCompare,
 } from '../../../shared/helpers';
-import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
+import type { SuggestionRawDefinition } from '../../types';
 import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
 import { handleFragment } from '../../helper';
 import { TRIGGER_SUGGESTION_COMMAND } from '../../factories';

-export async function suggest(
-  innerText: string,
-  command: ESQLCommand<'keep'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  columnExists: (column: string) => boolean
-): Promise<SuggestionRawDefinition[]> {
+export async function suggest({
+  innerText,
+  getColumnsByType,
+  command,
+  columnExists,
+}: CommandSuggestParams<'keep'>): Promise<SuggestionRawDefinition[]> {
  if (
    /\s/.test(innerText[innerText.length - 1]) &&
    getLastNonWhitespaceChar(innerText) !== ',' &&
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/sort/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/sort/index.ts
@ -7,20 +7,19 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import type { ESQLCommand } from '@kbn/esql-ast';
+import { CommandSuggestParams } from '../../../definitions/types';
 import { noCaseCompare } from '../../../shared/helpers';
 import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
 import { TRIGGER_SUGGESTION_COMMAND } from '../../factories';
 import { getFieldsOrFunctionsSuggestions, handleFragment, pushItUpInTheList } from '../../helper';
-import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
+import type { SuggestionRawDefinition } from '../../types';
 import { getSortPos, sortModifierSuggestions } from './helper';

-export async function suggest(
-  innerText: string,
-  _command: ESQLCommand<'sort'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  columnExists: (column: string) => boolean
-): Promise<SuggestionRawDefinition[]> {
+export async function suggest({
+  innerText,
+  getColumnsByType,
+  columnExists,
+}: CommandSuggestParams<'sort'>): Promise<SuggestionRawDefinition[]> {
  const prependSpace = (s: SuggestionRawDefinition) => ({ ...s, text: ' ' + s.text });

  const { pos, nulls } = getSortPos(innerText);
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/stats/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/stats/index.ts
@ -7,9 +7,8 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import type { ESQLAstItem, ESQLCommand } from '@kbn/esql-ast';
-import { SupportedDataType } from '../../../definitions/types';
-import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
+import { CommandSuggestParams } from '../../../definitions/types';
+import type { SuggestionRawDefinition } from '../../types';
 import {
  TRIGGER_SUGGESTION_COMMAND,
  getNewVariableSuggestion,
@ -19,15 +18,13 @@ import { commaCompleteItem, pipeCompleteItem } from '../../complete_items';
 import { pushItUpInTheList } from '../../helper';
 import { byCompleteItem, getDateHistogramCompletionItem, getPosition } from './util';

-export async function suggest(
-  innerText: string,
-  command: ESQLCommand<'stats'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  _columnExists: (column: string) => boolean,
-  getSuggestedVariableName: () => string,
-  _getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-  getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>
-): Promise<SuggestionRawDefinition[]> {
+export async function suggest({
+  innerText,
+  command,
+  getColumnsByType,
+  getSuggestedVariableName,
+  getPreferences,
+}: CommandSuggestParams<'stats'>): Promise<SuggestionRawDefinition[]> {
  const pos = getPosition(innerText, command);

  const columnSuggestions = pushItUpInTheList(
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts
@ -7,17 +7,11 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import {
-  Walker,
-  type ESQLAstItem,
-  type ESQLCommand,
-  type ESQLSingleAstItem,
-  type ESQLFunction,
-} from '@kbn/esql-ast';
+import { Walker, type ESQLSingleAstItem, type ESQLFunction } from '@kbn/esql-ast';
 import { logicalOperators } from '../../../definitions/builtin';
-import { isParameterType, type SupportedDataType } from '../../../definitions/types';
+import { CommandSuggestParams, isParameterType } from '../../../definitions/types';
 import { isFunctionItem } from '../../../shared/helpers';
-import type { GetColumnsByTypeFn, SuggestionRawDefinition } from '../../types';
+import type { SuggestionRawDefinition } from '../../types';
 import {
  getFunctionSuggestions,
  getOperatorSuggestion,
@ -33,16 +27,13 @@ import {
  UNSUPPORTED_COMMANDS_BEFORE_QSTR,
 } from '../../../shared/constants';

-export async function suggest(
-  innerText: string,
-  command: ESQLCommand<'where'>,
-  getColumnsByType: GetColumnsByTypeFn,
-  _columnExists: (column: string) => boolean,
-  _getSuggestedVariableName: () => string,
-  getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-  _getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
-  previousCommands?: ESQLCommand[]
-): Promise<SuggestionRawDefinition[]> {
+export async function suggest({
+  innerText,
+  command,
+  getColumnsByType,
+  getExpressionType,
+  previousCommands,
+}: CommandSuggestParams<'where'>): Promise<SuggestionRawDefinition[]> {
  const suggestions: SuggestionRawDefinition[] = [];

  /**
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts
@ -102,7 +102,10 @@ export function getQueryForFields(queryString: string, commands: ESQLCommand[])
 export function getSourcesFromCommands(commands: ESQLCommand[], sourceType: 'index' | 'policy') {
  const fromCommand = commands.find(({ name }) => name === 'from');
  const args = (fromCommand?.args ?? []) as ESQLSource[];
-  return args.filter((arg) => arg.sourceType === sourceType);
+  // the marker gets added in queries like "FROM "
+  return args.filter(
+    (arg) => arg.sourceType === sourceType && arg.name !== '' && arg.name !== EDITOR_MARKER
+  );
 }

 export function removeQuoteForSuggestedSources(suggestions: SuggestionRawDefinition[]) {
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/commands.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/commands.ts
@ -43,6 +43,7 @@ import { suggest as suggestForDrop } from '../autocomplete/commands/drop';
 import { suggest as suggestForStats } from '../autocomplete/commands/stats';
 import { suggest as suggestForWhere } from '../autocomplete/commands/where';
 import { suggest as suggestForJoin } from '../autocomplete/commands/join';
+import { suggest as suggestForFrom } from '../autocomplete/commands/from';

 const statsValidator = (command: ESQLCommand) => {
  const messages: ESQLMessage[] = [];
@ -208,11 +209,11 @@ export const commandDefinitions: Array<CommandDefinition<any>> = [
    examples: ['from logs', 'from logs-*', 'from logs_*, events-*'],
    options: [metadataOption],
    modes: [],
-    hasRecommendedQueries: true,
    signature: {
      multipleParams: true,
      params: [{ name: 'index', type: 'source', wildcards: true }],
    },
+    suggest: suggestForFrom,
  },
  {
    name: 'show',
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/types.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/types.ts
@ -13,9 +13,10 @@ import type {
  ESQLCommandOption,
  ESQLFunction,
  ESQLMessage,
+  ESQLSource,
 } from '@kbn/esql-ast';
 import { GetColumnsByTypeFn, SuggestionRawDefinition } from '../autocomplete/types';
-import type { ESQLCallbacks } from '../shared/types';
+import type { ESQLCallbacks, ESQLSourceResult } from '../shared/types';

 /**
 * All supported field types in ES|QL. This is all the types
@ -183,6 +184,73 @@ export interface FunctionDefinition {
  operator?: string;
 }

+export interface CommandSuggestParams<CommandName extends string> {
+  /**
+   * The text of the query to the left of the cursor.
+   */
+  innerText: string;
+  /**
+   * The AST node of this command.
+   */
+  command: ESQLCommand<CommandName>;
+  /**
+   * Get a list of columns by type. This includes fields from any sources as well as
+   * variables defined in the query.
+   */
+  getColumnsByType: GetColumnsByTypeFn;
+  /**
+   * Check for the existence of a column by name.
+   * @param column
+   * @returns
+   */
+  columnExists: (column: string) => boolean;
+  /**
+   * Gets the name that should be used for the next variable.
+   * @returns
+   */
+  getSuggestedVariableName: () => string;
+  /**
+   * Examine the AST to determine the type of an expression.
+   * @param expression
+   * @returns
+   */
+  getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown';
+  /**
+   * Get a list of system preferences (currently the target value for the histogram bar)
+   * @returns
+   */
+  getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>;
+  /**
+   * The definition for the current command.
+   */
+  definition: CommandDefinition<CommandName>;
+  /**
+   * Fetch a list of all available sources
+   * @returns
+   */
+  getSources: () => Promise<ESQLSourceResult[]>;
+  /**
+   * Inspect the AST and returns the sources that are used in the query.
+   * @param type
+   * @returns
+   */
+  getSourcesFromQuery: (type: 'index' | 'policy') => ESQLSource[];
+  /**
+   * Generate a list of recommended queries
+   * @returns
+   */
+  getRecommendedQueriesSuggestions: (prefix?: string) => Promise<SuggestionRawDefinition[]>;
+  /**
+   * The AST for the query behind the cursor.
+   */
+  previousCommands?: ESQLCommand[];
+  callbacks?: ESQLCallbacks;
+}
+
+export type CommandSuggestFunction<CommandName extends string> = (
+  params: CommandSuggestParams<CommandName>
+) => Promise<SuggestionRawDefinition[]>;
+
 export interface CommandBaseDefinition<CommandName extends string> {
  name: CommandName;

@ -201,18 +269,7 @@ export interface CommandBaseDefinition<CommandName extends string> {
   * Whether to show or hide in autocomplete suggestion list
   */
  hidden?: boolean;
-  suggest?: (
-    innerText: string,
-    command: ESQLCommand<CommandName>,
-    getColumnsByType: GetColumnsByTypeFn,
-    columnExists: (column: string) => boolean,
-    getSuggestedVariableName: () => string,
-    getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown',
-    getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>,
-    previousCommands?: ESQLCommand[],
-    definition?: CommandDefinition<CommandName>,
-    callbacks?: ESQLCallbacks
-  ) => Promise<SuggestionRawDefinition[]>;
+  suggest?: CommandSuggestFunction<CommandName>;
  /** @deprecated this property will disappear in the future */
  signature: {
    multipleParams: boolean;
@ -259,7 +316,6 @@ export interface CommandDefinition<CommandName extends string>
  extends CommandBaseDefinition<CommandName> {
  examples: string[];
  validate?: (option: ESQLCommand) => ESQLMessage[];
-  hasRecommendedQueries?: boolean;
  /** @deprecated this property will disappear in the future */
  modes: CommandModeDefinition[];
  /** @deprecated this property will disappear in the future */
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/shared/context.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/shared/context.ts
@ -86,7 +86,7 @@ function findCommandSubType<T extends ESQLCommandMode | ESQLCommandOption>(
  }
 }

-function isMarkerNode(node: ESQLSingleAstItem | undefined): boolean {
+export function isMarkerNode(node: ESQLSingleAstItem | undefined): boolean {
  return Boolean(
    node &&
      (isColumnItem(node) || isIdentifier(node) || isSourceItem(node)) &&
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json
@ -84,6 +84,10 @@
      "name": "dateNanosField",
      "type": "date_nanos"
    },
+    {
+      "name": "functionNamedParametersField",
+      "type": "function_named_parameters"
+    },
    {
      "name": "any#Char$Field",
      "type": "double"
@ -4447,6 +4451,46 @@
      "error": [],
      "warning": []
    },
+    {
+      "query": "from a_index | where functionNamedParametersField IS NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField IS null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField is null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField is NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField IS NOT NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField IS NOT null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField IS not NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | where functionNamedParametersField Is nOt NuLL",
+      "error": [],
+      "warning": []
+    },
    {
      "query": "from a_index | where textField == \"a\" or null",
      "error": [],
@ -5208,6 +5252,41 @@
      "error": [],
      "warning": []
    },
+    {
+      "query": "from a_index | eval functionNamedParametersField IS NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField IS null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField is null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField is NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField IS NOT NULL",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField IS NOT null",
+      "error": [],
+      "warning": []
+    },
+    {
+      "query": "from a_index | eval functionNamedParametersField IS not NULL",
+      "error": [],
+      "warning": []
+    },
    {
      "query": "from a_index | eval - doubleField",
      "error": [],
--- a/test/api_integration/apis/esql/errors.ts
+++ b/test/api_integration/apis/esql/errors.ts
@ -62,7 +62,7 @@ function createIndexRequest(
          if (type === 'cartesian_shape') {
            esType = 'shape';
          }
-          if (type === 'unsupported') {
+          if (type === 'unsupported' || type === 'function_named_parameters') {
            esType = 'integer_range';
          }
          memo[name] = { type: esType } as MappingProperty;