[ES|QL] Improve support for Invoke completion trigger kind (#188877)

## Summary Resolve https://github.com/elastic/kibana/issues/188677 This PR extends our support for Monaco's "Invoke" completion trigger kind ([ref](https://microsoft.github.io/monaco-editor/typedoc/enums/languages.CompletionTriggerKind.html)). What this means for the user is that suggestions are now shown when the user types a character in a word even if the suggestion menu isn't already open. This situation often occurs when the user is editing a query. For example, they may follow a flow like this: 1. `FROM my-index | LIMIT 10<cursor-here>` 2. `FROM <cursor-here> | LIMIT 10` (deleted the index name) 3. `FROM k<cursor-here> | LIMIT 10` (types the first character of the index name) Previously, they wouldn't get any help. Now, we show the list of suggestions. The following table shows the cases that I considered as well as the ones that are now fixed. I also added test coverage, even for cases that worked before to prevent subtle regressions. | | Before | After | |------------------------------|------------------------------------------|-----------------------| | Source command | ✅ | ✅ | | Pipe command | ❌ | ✅ | | Function argument | ✅ | ✅ | | FROM source | ❌ | ✅ | | FROM source METADATA | ✅ | ✅ | | FROM source METADATA field | ❌ (for `_`) | ✅ | | EVAL argument | ✅ | ✅ | | DISSECT field | ✅ | ✅ | | DISSECT field pattern | ❌ | ❌ | | DROP field | ✅ | ✅ | | DROP field1, field2 | ❌ | ✅ | | ENRICH policy | ✅ | ✅ | | ENRICH policy ON | ✅ | ✅ | | ENRICH policy ON field | ✅ | ✅ | | ENRICH policy WITH | ✅ | ✅ | | ENRICH policy WITH field | ✅ | ✅ | | GROK field | ✅ | ✅ | | GROK field pattern | ❌ | ❌ | | KEEP field | ✅ | ✅ | | KEEP field1, field2 | ❌ | ✅ | | LIMIT number | ❌ | ❌ | | MV_EXPAND field | ✅ | ✅ | | RENAME field | ✅ | ✅ | | RENAME field AS | ❌ | ✅ | | RENAME field AS var0 | ✅ | ✅ | | SORT field | ✅ | ✅ | | SORT field order | ✅ | ✅ | | SORT field order nulls-order | ✅ | ✅ | | STATS argument | ✅ | ✅ | | STATS argument BY | ✅ | ✅ | | STATS argument BY expression | ✅ | ✅ | | WHERE argument | ✅ | ✅ | | WHERE argument comparison | ✅ | ✅ | As I worked, I encountered a couple small bugs which I also fixed. ### Escaping field values in `ENRICH` **Before** https://github.com/user-attachments/assets/88f2bf35-a703-4cf4-8d45-a24452a1b59d **After** https://github.com/user-attachments/assets/69ad0770-f158-44fb-be8d-66b497c7f7f7 ### Suggesting variable assignment in `ENRICH ... WITH` **Before** https://github.com/user-attachments/assets/fec47b6d-eae5-44e8-b739-9b2eecc7458b **After** https://github.com/user-attachments/assets/dff5afe1-37e7-470b-bb9c-edb5ac87e76d ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-24 17:59:23 -04:00 · 2024-07-24 09:34:57 -06:00 · 2024-07-24 09:34:57 -06:00 · c77cd80070
commit c77cd80070
parent 6cc7444f7c
8 changed files with 361 additions and 96 deletions
--- a/packages/kbn-esql-ast/src/ast_walker.ts
+++ b/packages/kbn-esql-ast/src/ast_walker.ts
@ -416,7 +416,7 @@ export function visitRenameClauses(clausesCtx: RenameClauseContext[]): ESQLAstIt
  return clausesCtx
    .map((clause) => {
      const asToken = clause.getToken(esql_parser.AS, 0);
-      if (asToken) {
+      if (asToken && textExistsAndIsValid(asToken.getText())) {
        const fn = createOption(asToken.getText().toLowerCase(), clause);
        for (const arg of [clause._oldName, clause._newName]) {
          if (textExistsAndIsValid(arg.getText())) {
--- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.stats.test.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.stats.test.ts
@ -16,7 +16,7 @@ const allEvaFunctions = getFunctionSignaturesByReturnType(
  'stats',
  'any',
  {
-    evalMath: true,
+    scalar: true,
    grouping: false,
  },
  undefined,
@ -77,37 +77,37 @@ describe('autocomplete.suggest', () => {
          'from a | stats by bucket(/',
          [
            ...getFieldNamesByType(['number', 'date']),
-            ...getFunctionSignaturesByReturnType('eval', ['date', 'number'], { evalMath: true }),
+            ...getFunctionSignaturesByReturnType('eval', ['date', 'number'], { scalar: true }),
          ].map((field) => `${field},`)
        );

        await assertSuggestions('from a | stats round(/', [
          ...getFunctionSignaturesByReturnType('stats', 'number', { agg: true, grouping: true }),
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
            'round',
          ]),
        ]);
        await assertSuggestions('from a | stats round(round(/', [
          ...getFunctionSignaturesByReturnType('stats', 'number', { agg: true }),
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
            'round',
          ]),
        ]);
        await assertSuggestions('from a | stats avg(round(/', [
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
            'round',
          ]),
        ]);
        await assertSuggestions('from a | stats avg(/', [
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
        ]);
        await assertSuggestions('from a | stats round(avg(/', [
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
            'round',
          ]),
        ]);
@ -118,7 +118,7 @@ describe('autocomplete.suggest', () => {
        const expected = [
          ...getFieldNamesByType(['number', 'date', 'boolean', 'ip']),
          ...getFunctionSignaturesByReturnType('stats', ['number', 'date', 'boolean', 'ip'], {
-            evalMath: true,
+            scalar: true,
          }),
        ];

@ -132,7 +132,7 @@ describe('autocomplete.suggest', () => {

        await assertSuggestions('from a | stats avg(b/) by stringField', [
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
        ]);
      });

@ -198,7 +198,7 @@ describe('autocomplete.suggest', () => {
        await assertSuggestions('from a | stats avg(b) by c, /', [
          'var0 =',
          ...getFieldNamesByType('any'),
-          ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+          ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
          ...allGroupingFunctions,
        ]);
      });
@ -209,7 +209,7 @@ describe('autocomplete.suggest', () => {
        await assertSuggestions('from a | stats avg(b) by numberField % /', [
          ...getFieldNamesByType('number'),
          '`avg(b)`',
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
          ...allGroupingFunctions,
        ]);
        await assertSuggestions('from a | stats avg(b) by var0 = /', [
--- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/tests/helpers.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/tests/helpers.ts
@ -127,7 +127,7 @@ export function getFunctionSignaturesByReturnType(
  {
    agg,
    grouping,
-    evalMath,
+    scalar,
    builtin,
    // skipAssign here is used to communicate to not propose an assignment if it's not possible
    // within the current context (the actual logic has it, but here we want a shortcut)
@ -135,7 +135,7 @@ export function getFunctionSignaturesByReturnType(
  }: {
    agg?: boolean;
    grouping?: boolean;
-    evalMath?: boolean;
+    scalar?: boolean;
    builtin?: boolean;
    skipAssign?: boolean;
  } = {},
@ -157,7 +157,7 @@ export function getFunctionSignaturesByReturnType(
    list.push(...groupingFunctionDefinitions);
  }
  // eval functions (eval is a special keyword in JS)
-  if (evalMath) {
+  if (scalar) {
    list.push(...evalFunctionDefinitions);
  }
  if (builtin) {
--- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
@ -10,7 +10,12 @@ import { suggest } from './autocomplete';
 import { evalFunctionDefinitions } from '../definitions/functions';
 import { timeUnitsToSuggest } from '../definitions/literals';
 import { commandDefinitions } from '../definitions/commands';
-import { getUnitDuration, TRIGGER_SUGGESTION_COMMAND, TIME_SYSTEM_PARAMS } from './factories';
+import {
+  getSafeInsertText,
+  getUnitDuration,
+  TRIGGER_SUGGESTION_COMMAND,
+  TIME_SYSTEM_PARAMS,
+} from './factories';
 import { camelCase, partition } from 'lodash';
 import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
 import { FunctionParameter } from '../definitions/types';
@ -28,6 +33,7 @@ import {
  PartialSuggestionWithText,
  TIME_PICKER_SUGGESTION,
 } from './__tests__/helpers';
+import { METADATA_FIELDS } from '../shared/constants';

 describe('autocomplete', () => {
  type TestArgs = [
@ -148,7 +154,7 @@ describe('autocomplete', () => {

  describe('where', () => {
    const allEvalFns = getFunctionSignaturesByReturnType('where', 'any', {
-      evalMath: true,
+      scalar: true,
    });
    testSuggestions('from a | where ', [...getFieldNamesByType('any'), ...allEvalFns]);
    testSuggestions('from a | eval var0 = 1 | where ', [
@ -169,12 +175,12 @@ describe('autocomplete', () => {
    ]);
    testSuggestions('from a | where stringField >= ', [
      ...getFieldNamesByType('string'),
-      ...getFunctionSignaturesByReturnType('where', 'string', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('where', 'string', { scalar: true }),
    ]);
    // Skip these tests until the insensitive case equality gets restored back
    testSuggestions.skip('from a | where stringField =~ ', [
      ...getFieldNamesByType('string'),
-      ...getFunctionSignaturesByReturnType('where', 'string', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('where', 'string', { scalar: true }),
    ]);
    testSuggestions('from a | where stringField >= stringField ', [
      '|',
@ -201,14 +207,14 @@ describe('autocomplete', () => {
    for (const op of ['and', 'or']) {
      testSuggestions(`from a | where stringField >= stringField ${op} `, [
        ...getFieldNamesByType('any'),
-        ...getFunctionSignaturesByReturnType('where', 'any', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('where', 'any', { scalar: true }),
      ]);
      testSuggestions(`from a | where stringField >= stringField ${op} numberField `, [
        ...getFunctionSignaturesByReturnType('where', 'boolean', { builtin: true }, ['number']),
      ]);
      testSuggestions(`from a | where stringField >= stringField ${op} numberField == `, [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }),
      ]);
    }
    testSuggestions('from a | stats a=avg(numberField) | where a ', [
@ -232,7 +238,7 @@ describe('autocomplete', () => {
      'from a | where log10()',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }, undefined, [
          'log10',
        ]),
      ],
@ -246,7 +252,7 @@ describe('autocomplete', () => {
      'from a | WHERE pow(numberField, )',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }, undefined, [
          'pow',
        ]),
      ],
@ -257,7 +263,7 @@ describe('autocomplete', () => {
    testSuggestions('from index | WHERE stringField NOT ', ['LIKE $0', 'RLIKE $0', 'IN $0']);
    testSuggestions('from index | WHERE not ', [
      ...getFieldNamesByType('boolean'),
-      ...getFunctionSignaturesByReturnType('eval', 'boolean', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'boolean', { scalar: true }),
    ]);
    testSuggestions('from index | WHERE numberField in ', ['( $0 )']);
    testSuggestions('from index | WHERE numberField not in ', ['( $0 )']);
@ -265,7 +271,7 @@ describe('autocomplete', () => {
      'from index | WHERE numberField not in ( )',
      [
        ...getFieldNamesByType('number').filter((name) => name !== 'numberField'),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }),
      ],
      '('
    );
@ -275,7 +281,7 @@ describe('autocomplete', () => {
        ...getFieldNamesByType('number').filter(
          (name) => name !== '`any#Char$Field`' && name !== 'numberField'
        ),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }),
      ],
      undefined,
      54 // after the first suggestions
@ -286,7 +292,7 @@ describe('autocomplete', () => {
        ...getFieldNamesByType('number').filter(
          (name) => name !== '`any#Char$Field`' && name !== 'numberField'
        ),
-        ...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('where', 'number', { scalar: true }),
      ],
      undefined,
      58 // after the first suggestions
@ -339,7 +345,7 @@ describe('autocomplete', () => {
  describe('sort', () => {
    testSuggestions('from a | sort ', [
      ...getFieldNamesByType('any'),
-      ...getFunctionSignaturesByReturnType('sort', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('sort', 'any', { scalar: true }),
    ]);
    testSuggestions('from a | sort stringField ', ['ASC', 'DESC', ',', '|']);
    testSuggestions('from a | sort stringField desc ', ['NULLS FIRST', 'NULLS LAST', ',', '|']);
@ -371,6 +377,12 @@ describe('autocomplete', () => {
        getFieldNamesByType('any').filter((name) => name !== 'stringField')
      );

+      testSuggestions(
+        `from a | ${command} stringField,`,
+        getFieldNamesByType('any').filter((name) => name !== 'stringField'),
+        ','
+      );
+
      testSuggestions(
        `from a_index | eval round(numberField) + 1 | eval \`round(numberField) + 1\` + 1 | eval \`\`\`round(numberField) + 1\`\` + 1\` + 1 | eval \`\`\`\`\`\`\`round(numberField) + 1\`\`\`\` + 1\`\` + 1\` + 1 | eval \`\`\`\`\`\`\`\`\`\`\`\`\`\`\`round(numberField) + 1\`\`\`\`\`\`\`\` + 1\`\`\`\` + 1\`\` + 1\` + 1 | ${command} `,
        [
@ -416,7 +428,7 @@ describe('autocomplete', () => {
        'geoShapeField',
        'cartesianPointField',
        'cartesianShapeField',
-        'any#Char$Field',
+        '`any#Char$Field`',
        'kubernetes.something.something',
      ]);
      testSuggestions(`from a ${prevCommand}| enrich policy on b `, ['WITH $0', ',', '|']);
@ -459,7 +471,7 @@ describe('autocomplete', () => {
    testSuggestions('from a | eval ', [
      'var0 =',
      ...getFieldNamesByType('any'),
-      ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
    ]);
    testSuggestions('from a | eval numberField ', [
      ...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
@ -475,37 +487,37 @@ describe('autocomplete', () => {
      'from index | EVAL numberField in ( )',
      [
        ...getFieldNamesByType('number').filter((name) => name !== 'numberField'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
      ],
      '('
    );
    testSuggestions('from index | EVAL numberField not in ', ['( $0 )']);
    testSuggestions('from index | EVAL not ', [
      ...getFieldNamesByType('boolean'),
-      ...getFunctionSignaturesByReturnType('eval', 'boolean', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'boolean', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=', [
-      ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=abs(numberField), b= ', [
-      ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=numberField, ', [
      'var0 =',
      ...getFieldNamesByType('any'),
      'a',
-      ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
    ]);
    // Skip this test until the insensitive case equality gets restored back
    testSuggestions.skip('from a | eval a=stringField =~ ', [
      ...getFieldNamesByType('string'),
-      ...getFunctionSignaturesByReturnType('eval', 'string', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }),
    ]);
    testSuggestions(
      'from a | eval a=round()',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
          'round',
        ]),
      ],
@ -545,7 +557,7 @@ describe('autocomplete', () => {
      'from a | eval a=round(numberField, ',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
          'round',
        ]),
      ],
@ -555,7 +567,7 @@ describe('autocomplete', () => {
      'from a | eval round(numberField, ',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
          'round',
        ]),
      ],
@ -565,34 +577,34 @@ describe('autocomplete', () => {
      'var0 =',
      ...getFieldNamesByType('any'),
      'a',
-      ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=round(numberField) + ', [
      ...getFieldNamesByType('number'),
      'a', // @TODO remove this
-      ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=round(numberField)+ ', [
      ...getFieldNamesByType('number'),
      'a', // @TODO remove this
-      ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=numberField+ ', [
      ...getFieldNamesByType('number'),
      'a', // @TODO remove this
-      ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
    ]);
    testSuggestions('from a | eval a=`any#Char$Field`+ ', [
      ...getFieldNamesByType('number'),
      'a', // @TODO remove this
-      ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
+      ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }),
    ]);
    testSuggestions(
      'from a | stats avg(numberField) by stringField | eval ',
      [
        'var0 =',
        '`avg(numberField)`',
-        ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
      ],
      ' ',
      undefined,
@ -605,7 +617,7 @@ describe('autocomplete', () => {
        'var0 =',
        ...getFieldNamesByType('any'),
        '`abs(numberField) + 1`',
-        ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
      ],
      ' '
    );
@ -614,7 +626,7 @@ describe('autocomplete', () => {
      [
        'var0 =',
        '`avg(numberField)`',
-        ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
      ],
      ' ',
      undefined,
@ -627,7 +639,7 @@ describe('autocomplete', () => {
        'var0 =',
        '`avg(numberField)`',
        '`avg(kubernetes.something.something)`',
-        ...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
+        ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
      ],
      ' ',
      undefined,
@ -645,7 +657,7 @@ describe('autocomplete', () => {
      'from a | eval a=round(numberField), b=round()',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
          'round',
        ]),
      ],
@ -654,7 +666,7 @@ describe('autocomplete', () => {
    // test that comma is correctly added to the suggestions if minParams is not reached yet
    testSuggestions('from a | eval a=concat( ', [
      ...getFieldNamesByType('string').map((v) => `${v},`),
-      ...getFunctionSignaturesByReturnType('eval', 'string', { evalMath: true }, undefined, [
+      ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }, undefined, [
        'concat',
      ]).map((v) => `${v},`),
    ]);
@ -662,7 +674,7 @@ describe('autocomplete', () => {
      'from a | eval a=concat(stringField, ',
      [
        ...getFieldNamesByType('string'),
-        ...getFunctionSignaturesByReturnType('eval', 'string', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }, undefined, [
          'concat',
        ]),
      ],
@ -673,7 +685,7 @@ describe('autocomplete', () => {
      'from a | eval a=cidr_match(ipField, stringField, ',
      [
        ...getFieldNamesByType('string'),
-        ...getFunctionSignaturesByReturnType('eval', 'string', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }, undefined, [
          'cidr_match',
        ]),
      ],
@ -682,7 +694,7 @@ describe('autocomplete', () => {
    // test that comma is correctly added to the suggestions if minParams is not reached yet
    testSuggestions('from a | eval a=cidr_match( ', [
      ...getFieldNamesByType('ip').map((v) => `${v},`),
-      ...getFunctionSignaturesByReturnType('eval', 'ip', { evalMath: true }, undefined, [
+      ...getFunctionSignaturesByReturnType('eval', 'ip', { scalar: true }, undefined, [
        'cidr_match',
      ]).map((v) => `${v},`),
    ]);
@ -690,7 +702,7 @@ describe('autocomplete', () => {
      'from a | eval a=cidr_match(ipField, ',
      [
        ...getFieldNamesByType('string'),
-        ...getFunctionSignaturesByReturnType('eval', 'string', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }, undefined, [
          'cidr_match',
        ]),
      ],
@ -705,7 +717,7 @@ describe('autocomplete', () => {
        `from a | eval a=${Array(nesting).fill('round(').join('')}`,
        [
          ...getFieldNamesByType('number'),
-          ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
            'round',
          ]),
        ],
@ -730,7 +742,7 @@ describe('autocomplete', () => {
      'from a | eval var0 = abs(b) | eval abs(var0)',
      [
        ...getFieldNamesByType('number'),
-        ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
+        ...getFunctionSignaturesByReturnType('eval', 'number', { scalar: true }, undefined, [
          'abs',
        ]),
      ],
@ -785,7 +797,7 @@ describe('autocomplete', () => {
                      ...getFunctionSignaturesByReturnType(
                        'eval',
                        getTypesFromParamDefs(acceptsFieldParamDefs),
-                        { evalMath: true },
+                        { scalar: true },
                        undefined,
                        [fn.name]
                      ),
@ -805,7 +817,7 @@ describe('autocomplete', () => {
                      ...getFunctionSignaturesByReturnType(
                        'eval',
                        getTypesFromParamDefs(acceptsFieldParamDefs),
-                        { evalMath: true },
+                        { scalar: true },
                        undefined,
                        [fn.name]
                      ),
@ -870,7 +882,7 @@ describe('autocomplete', () => {
        [
          ...[...TIME_SYSTEM_PARAMS].map((t) => `${t},`),
          ...getLiteralsByType('time_literal').map((t) => `${t},`),
-          ...getFunctionSignaturesByReturnType('eval', 'date', { evalMath: true }, undefined, [
+          ...getFunctionSignaturesByReturnType('eval', 'date', { scalar: true }, undefined, [
            'date_trunc',
          ]).map((t) => `${t},`),
          ...getFieldNamesByType('date').map((t) => `${t},`),
@ -962,4 +974,222 @@ describe('autocomplete', () => {
      ).toBeTruthy();
    });
  });
+
+  /**
+   * Monaco asks for suggestions in at least two different scenarios.
+   * 1. When the user types a non-whitespace character (e.g. 'FROM k') - this is the Invoke trigger kind
+   * 2. When the user types a character we've registered as a trigger character (e.g. ',') - this is the Trigger character trigger kind
+   *
+   * Historically we had good support for the trigger character trigger kind, but not for the Invoke trigger kind. That led
+   * to bad experiences like a list of sources not showing up when the user types 'FROM kib'. There they had to delete "kib"
+   * and press <space> to trigger suggestions via a trigger character.
+   *
+   * See https://microsoft.github.io/monaco-editor/typedoc/enums/languages.CompletionTriggerKind.html for more details
+   */
+  describe('Invoke trigger kind (all commands)', () => {
+    // source command
+    testSuggestions(
+      'f',
+      sourceCommands.map((cmd) => `${cmd.toUpperCase()} $0`),
+      undefined,
+      1
+    );
+
+    // pipe command
+    testSuggestions(
+      'FROM k | E',
+      commandDefinitions
+        .filter(({ name }) => !sourceCommands.includes(name))
+        .map(({ name }) => name.toUpperCase() + ' $0'),
+      undefined,
+      10
+    );
+
+    // function argument
+    testSuggestions(
+      'FROM kibana_sample_data_logs | EVAL TRIM(e)',
+      [
+        ...getFieldNamesByType('string'),
+        ...getFunctionSignaturesByReturnType('eval', 'string', { scalar: true }, undefined, [
+          'trim',
+        ]),
+      ],
+      undefined,
+      42
+    );
+
+    // FROM source
+    testSuggestions('FROM k', ['index1', 'index2'], undefined, 6, [
+      ,
+      [
+        { name: 'index1', hidden: false },
+        { name: 'index2', hidden: false },
+      ],
+    ]);
+
+    // FROM source METADATA
+    testSuggestions('FROM index1 M', [',', 'METADATA $0', '|'], undefined, 13);
+
+    // FROM source METADATA field
+    testSuggestions('FROM index1 METADATA _', METADATA_FIELDS, undefined, 22);
+
+    // EVAL argument
+    testSuggestions(
+      'FROM index1 | EVAL b',
+      [
+        'var0 =',
+        ...getFieldNamesByType('any'),
+        ...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true }),
+      ],
+      undefined,
+      20
+    );
+
+    testSuggestions(
+      'FROM index1 | EVAL var0 = f',
+      [...getFunctionSignaturesByReturnType('eval', 'any', { scalar: true })],
+      undefined,
+      27
+    );
+
+    // DISSECT field
+    testSuggestions('FROM index1 | DISSECT b', getFieldNamesByType('string'), undefined, 23);
+
+    // DROP (first field)
+    testSuggestions('FROM index1 | DROP f', getFieldNamesByType('any'), undefined, 20);
+
+    // DROP (subsequent field)
+    testSuggestions('FROM index1 | DROP field1, f', getFieldNamesByType('any'), undefined, 28);
+
+    // ENRICH policy
+    testSuggestions(
+      'FROM index1 | ENRICH p',
+      policies.map(({ name }) => getSafeInsertText(name)),
+      undefined,
+      22
+    );
+
+    // ENRICH policy ON
+    testSuggestions('FROM index1 | ENRICH policy O', ['ON $0', 'WITH $0', '|'], undefined, 29);
+
+    // ENRICH policy ON field
+    testSuggestions('FROM index1 | ENRICH policy ON f', getFieldNamesByType('any'), undefined, 32);
+
+    // ENRICH policy WITH policyfield
+    testSuggestions(
+      'FROM index1 | ENRICH policy WITH v',
+      ['var0 =', ...getPolicyFields('policy')],
+      undefined,
+      34
+    );
+
+    testSuggestions(
+      'FROM index1 | ENRICH policy WITH \tv',
+      ['var0 =', ...getPolicyFields('policy')],
+      undefined,
+      34
+    );
+
+    // GROK field
+    testSuggestions('FROM index1 | GROK f', getFieldNamesByType('string'), undefined, 20);
+
+    // KEEP (first field)
+    testSuggestions('FROM index1 | KEEP f', getFieldNamesByType('any'), undefined, 20);
+
+    // KEEP (subsequent fields)
+    testSuggestions(
+      'FROM index1 | KEEP booleanField, f',
+      getFieldNamesByType('any').filter((name) => name !== 'booleanField'),
+      undefined,
+      34
+    );
+
+    // LIMIT argument
+    // Here we actually test that the invoke trigger kind does not work
+    // because it isn't very useful to see literal suggestions when typing a number
+    testSuggestions('FROM a | LIMIT 1', ['|'], undefined, 16);
+
+    // MV_EXPAND field
+    testSuggestions('FROM index1 | MV_EXPAND f', getFieldNamesByType('any'), undefined, 25);
+
+    // RENAME field
+    testSuggestions('FROM index1 | RENAME f', getFieldNamesByType('any'), undefined, 22);
+
+    // RENAME field AS
+    testSuggestions('FROM index1 | RENAME field A', ['AS $0'], undefined, 28);
+
+    // RENAME field AS var0
+    testSuggestions('FROM index1 | RENAME field AS v', ['var0'], undefined, 31);
+
+    // SORT field
+    testSuggestions(
+      'FROM index1 | SORT f',
+      [
+        ...getFunctionSignaturesByReturnType('sort', 'any', { scalar: true }),
+        ...getFieldNamesByType('any'),
+      ],
+      undefined,
+      20
+    );
+
+    // SORT field order
+    testSuggestions('FROM index1 | SORT stringField a', ['ASC', 'DESC', ',', '|'], undefined, 32);
+
+    // SORT field order nulls
+    testSuggestions(
+      'FROM index1 | SORT stringField ASC n',
+      ['NULLS FIRST', 'NULLS LAST', ',', '|'],
+      undefined,
+      36
+    );
+
+    // STATS argument
+    testSuggestions(
+      'FROM index1 | STATS f',
+      ['var0 =', ...getFunctionSignaturesByReturnType('stats', 'any', { scalar: true, agg: true })],
+      undefined,
+      21
+    );
+
+    // STATS argument BY
+    testSuggestions('FROM index1 | STATS AVG(booleanField) B', ['BY $0', ',', '|'], undefined, 39);
+
+    // STATS argument BY expression
+    testSuggestions(
+      'FROM index1 | STATS field BY f',
+      [
+        'var0 =',
+        ...getFunctionSignaturesByReturnType('stats', 'any', { grouping: true, scalar: true }),
+        ...getFieldNamesByType('any'),
+      ],
+      undefined,
+      30
+    );
+
+    // WHERE argument
+    testSuggestions(
+      'FROM index1 | WHERE f',
+      [
+        ...getFieldNamesByType('any'),
+        ...getFunctionSignaturesByReturnType('where', 'any', { scalar: true }),
+      ],
+      undefined,
+      22
+    );
+
+    // WHERE argument comparison
+    testSuggestions(
+      'FROM index1 | WHERE stringField i',
+      getFunctionSignaturesByReturnType(
+        'where',
+        'boolean',
+        {
+          builtin: true,
+        },
+        ['string']
+      ),
+      undefined,
+      33
+    );
+  });
 });
--- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
@ -42,6 +42,7 @@ import {
  isSingleItem,
  nonNullable,
  getColumnExists,
+  findPreviousWord,
 } from '../shared/helpers';
 import { collectVariables, excludeVariablesFromCurrentCommand } from '../shared/variables';
 import type { ESQLPolicy, ESQLRealField, ESQLVariable, ReferenceMaps } from '../validation/types';
@ -93,13 +94,9 @@ import {
 import { FunctionParameter } from '../definitions/types';

 type GetSourceFn = () => Promise<SuggestionRawDefinition[]>;
-type GetDataSourceFn = (sourceName: string) => Promise<
-  | {
-      name: string;
-      dataStreams?: Array<{ name: string; title?: string }>;
-    }
-  | undefined
->;
+type GetDataStreamsForIntegrationFn = (
+  sourceName: string
+) => Promise<Array<{ name: string; title?: string }> | undefined>;
 type GetFieldsByTypeFn = (
  type: string | string[],
  ignored?: string[]
@ -182,9 +179,8 @@ export async function suggest(
  const charThatNeedMarkers = [',', ':'];
  if (
    (context.triggerCharacter && charThatNeedMarkers.includes(context.triggerCharacter)) ||
-    (context.triggerKind === 0 &&
-      unclosedRoundBrackets === 0 &&
-      getLastCharFromTrimmed(innerText) !== '_') ||
+    // monaco.editor.CompletionTriggerKind['Invoke'] === 0
+    (context.triggerKind === 0 && unclosedRoundBrackets === 0) ||
    (context.triggerCharacter === ' ' &&
      (isMathFunction(innerText, offset) ||
        isComma(innerText.trimEnd()[innerText.trimEnd().length - 1])))
@ -326,11 +322,13 @@ function getSourcesRetriever(resourceRetriever?: ESQLCallbacks) {
  };
 }

-function getDatastreamsForIntegrationRetriever(resourceRetriever?: ESQLCallbacks) {
+function getDatastreamsForIntegrationRetriever(
+  resourceRetriever?: ESQLCallbacks
+): GetDataStreamsForIntegrationFn {
  const helper = getSourcesHelper(resourceRetriever);
  return async (sourceName: string) => {
    const list = (await helper()) || [];
-    return list.find(({ name }) => name === sourceName);
+    return list.find(({ name }) => name === sourceName)?.dataStreams;
  };
 }

@ -504,7 +502,7 @@ async function getExpressionSuggestionsByType(
    node: ESQLSingleAstItem | undefined;
  },
  getSources: GetSourceFn,
-  getDatastreamsForIntegration: GetDataSourceFn,
+  getDatastreamsForIntegration: GetDataStreamsForIntegrationFn,
  getFieldsByType: GetFieldsByTypeFn,
  getFieldsMap: GetFieldsMapFn,
  getPolicies: GetPoliciesFn,
@ -861,27 +859,31 @@ async function getExpressionSuggestionsByType(
      } else {
        const index = getSourcesFromCommands(commands, 'index');
        const canRemoveQuote = isNewExpression && innerText.includes('"');
+        // Function to add suggestions based on canRemoveQuote
+        const addSuggestionsBasedOnQuote = async (definitions: SuggestionRawDefinition[]) => {
+          suggestions.push(
+            ...(canRemoveQuote ? removeQuoteForSuggestedSources(definitions) : definitions)
+          );
+        };

-        // This is going to be empty for simple indices, and not empty for integrations
        if (index && index.text && index.text !== EDITOR_MARKER) {
          const source = index.text.replace(EDITOR_MARKER, '');
-          const dataSource = await getDatastreamsForIntegration(source);
+          const dataStreams = await getDatastreamsForIntegration(source);

-          const newDefinitions = buildSourcesDefinitions(
-            dataSource?.dataStreams?.map(({ name }) => ({ name, isIntegration: false })) || []
-          );
-          suggestions.push(
-            ...(canRemoveQuote ? removeQuoteForSuggestedSources(newDefinitions) : newDefinitions)
-          );
+          if (dataStreams) {
+            // Integration name, suggest the datastreams
+            await addSuggestionsBasedOnQuote(
+              buildSourcesDefinitions(
+                dataStreams.map(({ name }) => ({ name, isIntegration: false }))
+              )
+            );
+          } else {
+            // Not an integration, just a partial source name
+            await addSuggestionsBasedOnQuote(await getSources());
+          }
        } else {
-          // FROM <suggest>
-          // @TODO: filter down the suggestions here based on other existing sources defined
-          const sourcesDefinitions = await getSources();
-          suggestions.push(
-            ...(canRemoveQuote
-              ? removeQuoteForSuggestedSources(sourcesDefinitions)
-              : sourcesDefinitions)
-          );
+          // FROM <suggest> or no index/text
+          await addSuggestionsBasedOnQuote(await getSources());
        }
      }
    }
@ -1450,7 +1452,11 @@ async function getOptionArgsSuggestions(
  if (command.name === 'enrich') {
    if (option.name === 'on') {
      // if it's a new expression, suggest fields to match on
-      if (isNewExpression || (option && isAssignment(option.args[0]) && !option.args[1])) {
+      if (
+        isNewExpression ||
+        findPreviousWord(innerText) === 'ON' ||
+        (option && isAssignment(option.args[0]) && !option.args[1])
+      ) {
        const policyName = isSourceItem(command.args[0]) ? command.args[0].name : undefined;
        if (policyName) {
          const policyMetadata = await getPolicyMetadata(policyName);
@ -1483,7 +1489,7 @@ async function getOptionArgsSuggestions(
          innerText
        );

-        if (isNewExpression) {
+        if (isNewExpression || findPreviousWord(innerText) === 'WITH') {
          suggestions.push(buildNewVarDefinition(findNewVariable(anyEnhancedVariables)));
        }

--- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts
@ -35,7 +35,7 @@ export const TRIGGER_SUGGESTION_COMMAND = {
  id: 'editor.action.triggerSuggest',
 };

-function getSafeInsertText(text: string, options: { dashSupported?: boolean } = {}) {
+export function getSafeInsertText(text: string, options: { dashSupported?: boolean } = {}) {
  return shouldBeQuotedText(text, options)
    ? `\`${text.replace(SINGLE_TICK_REGEX, DOUBLE_BACKTICK)}\``
    : text;
@ -265,7 +265,7 @@ export const buildMatchingFieldsDefinition = (
 ): SuggestionRawDefinition[] =>
  fields.map((label) => ({
    label,
-    text: label,
+    text: getSafeInsertText(label),
    kind: 'Variable',
    detail: i18n.translate(
      'kbn-esql-validation-autocomplete.esql.autocomplete.matchingFieldDefinition',
--- a/packages/kbn-esql-validation-autocomplete/src/shared/context.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/shared/context.ts
@ -20,10 +20,10 @@ import { EDITOR_MARKER } from './constants';
 import {
  isOptionItem,
  isColumnItem,
-  getLastCharFromTrimmed,
  getFunctionDefinition,
  isSourceItem,
  isSettingItem,
+  pipePrecedesCurrentWord,
 } from './helpers';

 function findNode(nodes: ESQLAstItem[], offset: number): ESQLSingleAstItem | undefined {
@ -172,12 +172,12 @@ export function getAstContext(queryString: string, ast: ESQLAst, offset: number)
      return { type: 'setting' as const, command, node, option, setting };
    }
  }
-  if (!command || (queryString.length <= offset && getLastCharFromTrimmed(queryString) === '|')) {
+  if (!command || (queryString.length <= offset && pipePrecedesCurrentWord(queryString))) {
    //   // ... | <here>
    return { type: 'newCommand' as const, command: undefined, node, option, setting };
  }

-  if (command && command.args.length) {
+  if (command && isOptionItem(command.args[command.args.length - 1])) {
    if (option) {
      return { type: 'option' as const, command, node, option, setting };
    }
--- a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts
+++ b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts
@ -555,12 +555,41 @@ export function sourceExists(index: string, sources: Set<string>) {
  return Boolean(fuzzySearch(index, sources.keys()));
 }

+/**
+ * Works backward from the cursor position to determine if
+ * the final character of the previous word matches the given character.
+ */
+function characterPrecedesCurrentWord(text: string, char: string) {
+  let inCurrentWord = true;
+  for (let i = text.length - 1; i >= 0; i--) {
+    if (inCurrentWord && /\s/.test(text[i])) {
+      inCurrentWord = false;
+    }
+
+    if (!inCurrentWord && !/\s/.test(text[i])) {
+      return text[i] === char;
+    }
+  }
+}
+
+export function pipePrecedesCurrentWord(text: string) {
+  return characterPrecedesCurrentWord(text, '|');
+}
+
 export function getLastCharFromTrimmed(text: string) {
  return text[text.trimEnd().length - 1];
 }

+/**
+ * Are we after a comma? i.e. STATS fieldA, <here>
+ */
 export function isRestartingExpression(text: string) {
-  return getLastCharFromTrimmed(text) === ',';
+  return getLastCharFromTrimmed(text) === ',' || characterPrecedesCurrentWord(text, ',');
+}
+
+export function findPreviousWord(text: string) {
+  const words = text.split(/\s+/);
+  return words[words.length - 2];
 }

 export function shouldBeQuotedSource(text: string) {