[ES|QL] New @kbn/esql-services package (#179029)

## Summary

Closes https://github.com/elastic/kibana/issues/172649

This PR extracts the ES|QL service like "validation", "autocomplete",
"code_action" (quick fixes) and some other services into a separate
package `@kbn/esql-validation-autocomplete` which has no tie with Monaco
editor anymore.

All the AST and grammar logic has been encapsulated into a single
`@kbn/esql-ast` package who contains the `getAst` function together with
the `parser` and `lexer`.

## Validation enhancements

The validation API has been enhanced to gracefully degrade whenever
callbacks are not provided, via the `ignoreOnMissingCallback` option.
Tests are included as well for this scenario.

### Example plugin

An example plugin app has been developed to show to to use the
validation API

<img width="500" alt="Screenshot 2024-03-22 at 17 45 38"
src="ebc172d3-ee61-4f3a-9e42-dcb9b15c7e69">

The app starts with a missing callback:

<img width="855" alt="Screenshot 2024-03-22 at 17 36 22"
src="c9b81370-8a10-487d-b22f-2359e1365a54">

Toggling the ignore option will hide the index error due to the lack of
`getSources` callback:

<img width="816" alt="Screenshot 2024-03-22 at 17 28 17"
src="931f4e5f-3ad6-46f7-97ca-63d7bb66646b">

Warnings are shown as well when detected:

<img width="807" alt="Screenshot 2024-03-22 at 17 35 56"
src="5e3e0537-cba2-475f-946b-0302867384ca">

#### Code snippet

The example app produces a copyable code snippet of the current
configuration of the validator:

![Code
snippet](https://private-user-images.githubusercontent.com/924948/316838987-fd16da58-8790-40b2-851d-b92aca1811dc.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3MTE0NTY0MzUsIm5iZiI6MTcxMTQ1NjEzNSwicGF0aCI6Ii85MjQ5NDgvMzE2ODM4OTg3LWZkMTZkYTU4LTg3OTAtNDBiMi04NTFkLWI5MmFjYTE4MTFkYy5wbmc_WC1BbXotQWxnb3JpdGhtPUFXUzQtSE1BQy1TSEEyNTYmWC1BbXotQ3JlZGVudGlhbD1BS0lBVkNPRFlMU0E1M1BRSzRaQSUyRjIwMjQwMzI2JTJGdXMtZWFzdC0xJTJGczMlMkZhd3M0X3JlcXVlc3QmWC1BbXotRGF0ZT0yMDI0MDMyNlQxMjI4NTVaJlgtQW16LUV4cGlyZXM9MzAwJlgtQW16LVNpZ25hdHVyZT00ZDhhOGRlMjZjMzZhN2NjNTdkYWMxYTNjOWVkNTA3NDk5NGViOTg2NWEzMTAyZGM4MGYyOTQ0YjQ2N2Q3ZDI1JlgtQW16LVNpZ25lZEhlYWRlcnM9aG9zdCZhY3Rvcl9pZD0wJmtleV9pZD0wJnJlcG9faWQ9MCJ9.H-3r_B57NK1-rS8UbR_bxlNtAMQ0flFGtW0oTIiOlr0)

### Checklist

- [x]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/scripts/steps/esql_grammar_sync.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 synchronize_lexer_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4"
-  destination_file="./packages/kbn-monaco/src/esql/antlr/esql_lexer.g4"
+  destination_file="./packages/kbn-esql-ast/src/antlr/esql_lexer.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file" 
@@ -27,7 +27,7 @@ synchronize_lexer_grammar () {
 synchronize_parser_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4"
-  destination_file="./packages/kbn-monaco/src/esql/antlr/esql_parser.g4"
+  destination_file="./packages/kbn-esql-ast/src/antlr/esql_parser.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file" 
@@ -105,7 +105,7 @@ main () {
   .buildkite/scripts/bootstrap.sh
 
   # Build ANTLR stuff
-  cd ./packages/kbn-monaco/src  
+  cd ./packages/kbn-esql-ast/src  
   yarn build:antlr4:esql 
 
   # Make a commit

.eslintignore

@@ -39,6 +39,7 @@ snapshots.js
 /packages/kbn-ui-framework/dist
 /packages/kbn-flot-charts/lib
 /packages/kbn-monaco/src/**/antlr
+/packages/kbn-esql-ast/src/**/antlr
 
 # Bazel
 /bazel-*

.github/CODEOWNERS

@@ -393,7 +393,10 @@ packages/kbn-eslint-plugin-imports @elastic/kibana-operations
 packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team
 examples/eso_model_version_example @elastic/kibana-security
 x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security
+packages/kbn-esql-ast @elastic/kibana-visualizations
 packages/kbn-esql-utils @elastic/kibana-esql
+packages/kbn-esql-validation-autocomplete @elastic/kibana-visualizations
+examples/esql_validation_example @elastic/kibana-visualizations
 packages/kbn-event-annotation-common @elastic/kibana-visualizations
 packages/kbn-event-annotation-components @elastic/kibana-visualizations
 src/plugins/event_annotation_listing @elastic/kibana-visualizations

.i18nrc.json

@@ -81,6 +81,7 @@
     "interpreter": "src/legacy/core_plugins/interpreter",
     "imageEmbeddable": "src/plugins/image_embeddable",
     "kbn": "src/legacy/core_plugins/kibana",
+    "kbn-esql-validation-autocomplete": "packages/kbn-esql-validation-autocomplete/src",
     "kbnConfig": "packages/kbn-config/src",
     "kbnDocViews": "src/legacy/core_plugins/kbn_doc_views",
     "kibana_react": "src/legacy/core_plugins/kibana_react",

examples/esql_validation_example/README.md

@@ -0,0 +1,9 @@
+# esql_validation_example
+
+Examples of the ES|QL validation service.
+
+To run this example, start kibana with the `--run-examples` flag.
+
+```bash
+yarn start --run-examples
+```

examples/esql_validation_example/kibana.jsonc

@@ -0,0 +1,17 @@
+{
+    "type": "plugin",
+    "id": "@kbn/esql-validation-example-plugin",
+    "owner": "@elastic/kibana-visualizations",
+    "plugin": {
+      "id": "esqlValidationExample",
+      "server": false,
+      "browser": true,
+      "configPath": [
+        "esql_validation_example"
+      ],
+      "requiredPlugins": [
+        "dataViews",
+        "developerExamples"
+      ]
+    }
+  }

examples/esql_validation_example/public/app.tsx

@@ -0,0 +1,178 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React, { useEffect, useMemo, useState } from 'react';
+import {
+  EuiPage,
+  EuiPageBody,
+  EuiPageSection,
+  EuiPageHeader,
+  EuiSpacer,
+  EuiFieldText,
+  EuiCheckboxGroup,
+  EuiFormRow,
+  EuiSwitch,
+  EuiForm,
+  EuiCallOut,
+} from '@elastic/eui';
+
+import type { CoreStart } from '@kbn/core/public';
+
+import { ESQLCallbacks, validateQuery } from '@kbn/esql-validation-autocomplete';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import type { StartDependencies } from './plugin';
+import { CodeSnippet } from './code_snippet';
+
+export const App = (props: { core: CoreStart; plugins: StartDependencies }) => {
+  const [currentErrors, setErrors] = useState<string[]>([]);
+  const [currentWarnings, setWarnings] = useState<string[]>([]);
+  const [currentQuery, setQuery] = useState(
+    'from index1 | eval var0 = round(numberField, 2) | stats by stringField'
+  );
+
+  const [ignoreErrors, setIgnoreErrors] = useState(false);
+  const [callbacksEnabled, setCallbacksEnabled] = useState<
+    Record<'sources' | 'fields' | 'policies' | 'metaFields', boolean>
+  >({
+    sources: false,
+    fields: true,
+    policies: true,
+    metaFields: true,
+  });
+
+  const callbacks: ESQLCallbacks = useMemo(
+    () => ({
+      getSources: callbacksEnabled.sources
+        ? async () =>
+            ['index1', 'anotherIndex', 'dataStream'].map((name) => ({ name, hidden: false }))
+        : undefined,
+      getFieldsFor: callbacksEnabled.fields
+        ? async () => [
+            { name: 'numberField', type: 'number' },
+            { name: 'stringField', type: 'string' },
+          ]
+        : undefined,
+      getMetaFields: callbacksEnabled.metaFields
+        ? async () => ['_version', '_id', '_index', '_source']
+        : undefined,
+      getPolicies: callbacksEnabled.policies
+        ? async () => [
+            {
+              name: 'my-policy',
+              sourceIndices: ['policyIndex'],
+              matchField: 'otherStringField',
+              enrichFields: ['otherNumberField'],
+            },
+          ]
+        : undefined,
+    }),
+    [callbacksEnabled]
+  );
+
+  useEffect(() => {
+    if (currentQuery === '') {
+      return;
+    }
+    validateQuery(
+      currentQuery,
+      getAstAndSyntaxErrors,
+      { ignoreOnMissingCallbacks: ignoreErrors },
+      callbacks
+    ).then(({ errors: validationErrors, warnings: validationWarnings }) => {
+      // syntax errors come with a slight different format than other validation errors
+      setErrors(validationErrors.map((e) => ('severity' in e ? e.message : e.text)));
+      setWarnings(validationWarnings.map((e) => e.text));
+    });
+  }, [currentQuery, ignoreErrors, callbacks]);
+
+  const checkboxes = [
+    {
+      id: 'sources',
+      label: 'getSources callback => index1, anotherIndex, dataStream',
+    },
+    {
+      id: 'fields',
+      label: 'getFieldsFor callback => numberField, stringField',
+    },
+    {
+      id: 'policies',
+      label: 'getPolicies callback => my-policy',
+    },
+    {
+      id: 'metaFields',
+      label: 'getMetaFields callback => _version, _id, _index, _source',
+    },
+  ];
+
+  return (
+    <EuiPage>
+      <EuiPageBody style={{ maxWidth: 800, margin: '0 auto' }}>
+        <EuiPageHeader paddingSize="s" bottomBorder={true} pageTitle="ES|QL validation example" />
+        <EuiPageSection paddingSize="s">
+          <p>This app shows how to use the ES|QL validation API with all its options</p>
+
+          <EuiSpacer />
+          <EuiForm component="form" fullWidth>
+            <EuiCheckboxGroup
+              options={checkboxes}
+              idToSelectedMap={callbacksEnabled}
+              onChange={(optionId) => {
+                setCallbacksEnabled({
+                  ...callbacksEnabled,
+                  [optionId]: !callbacksEnabled[optionId as keyof typeof callbacksEnabled],
+                });
+              }}
+            />
+
+            <EuiSpacer />
+
+            <EuiFormRow label="Validation options" hasChildLabel={false}>
+              <EuiSwitch
+                name="switch"
+                label="Ignore errors on missing callback"
+                checked={ignoreErrors}
+                onChange={() => setIgnoreErrors(!ignoreErrors)}
+              />
+            </EuiFormRow>
+            <EuiSpacer />
+            <EuiFormRow
+              label="Write the ES|QL query here"
+              helpText={currentErrors.length ? '' : 'No errors found'}
+              isInvalid={currentErrors.length > 0}
+              error={currentErrors}
+            >
+              <EuiFieldText
+                fullWidth
+                placeholder="from index1"
+                value={currentQuery}
+                onChange={(e) => setQuery(e.target.value)}
+                isInvalid={currentErrors.length > 0}
+              />
+            </EuiFormRow>
+            {currentWarnings.length ? (
+              <EuiCallOut title="Validation warnings" color="warning" iconType="warning">
+                <p>Here the list of warnings:</p>
+                <ul>
+                  {currentWarnings.map((message) => (
+                    <li key={message}>{message}</li>
+                  ))}
+                </ul>
+              </EuiCallOut>
+            ) : null}
+          </EuiForm>
+          <EuiSpacer />
+          <CodeSnippet
+            currentQuery={currentQuery}
+            callbacks={callbacksEnabled}
+            ignoreErrors={ignoreErrors}
+          />
+        </EuiPageSection>
+      </EuiPageBody>
+    </EuiPage>
+  );
+};

examples/esql_validation_example/public/code_snippet.tsx

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EuiCodeBlock } from '@elastic/eui';
+import React from 'react';
+
+interface CodeSnippetProps {
+  currentQuery: string;
+  callbacks: Record<'sources' | 'fields' | 'policies' | 'metaFields', boolean>;
+  ignoreErrors: boolean;
+}
+
+function getCallbacksCode(callbacks: CodeSnippetProps['callbacks']) {
+  return `({
+        ${
+          callbacks.sources
+            ? `getSources: async () =>
+            ['index1', 'anotherIndex', 'dataStream'].map((name) => ({ name, hidden: false })),`
+            : ''
+        }
+        ${
+          callbacks.fields
+            ? `
+        // the getFieldsFor callback gets an esql query to get the required fields
+        // note that the query is not optimized yet, so things like "| limit 0"
+        // might be appended to speed up the retrieval. 
+        getFieldsFor: async (esqlFieldsQuery: string) => [
+            { name: 'numberField', type: 'number' },
+            { name: 'stringField', type: 'string' },
+        ],`
+            : ''
+        }
+        ${
+          callbacks.metaFields
+            ? `getMetaFields: async () => ['_version', '_id', '_index', '_source'],`
+            : ''
+        }
+        ${
+          callbacks.policies
+            ? `getPolicies: async () => [
+            {
+                name: 'my-policy',
+                sourceIndices: ['policyIndex'],
+                matchField: 'otherStringField',
+                enrichFields: ['otherNumberField'],
+            },
+        ],`
+            : ''
+        }
+})`.replace(/^\s*\n/gm, '');
+}
+
+export function CodeSnippet({ currentQuery, callbacks, ignoreErrors }: CodeSnippetProps) {
+  return (
+    <EuiCodeBlock language="typescript" isCopyable>
+      {`
+import { ESQLCallbacks, validateQuery } from '@kbn/esql-validation-autocomplete';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+
+const currentQuery = "${currentQuery}";
+
+const callbacks: ESQLCallbacks = () => ${getCallbacksCode(callbacks)};
+
+const {errors, warnings} = validateQuery(
+    currentQuery,
+    getAstAndSyntaxErrors,
+    { ignoreOnMissingCallbacks: ${Boolean(ignoreErrors)} },
+    callbacks
+);
+`}
+    </EuiCodeBlock>
+  );
+}

examples/esql_validation_example/public/index.tsx

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ESQLValidationExamplePlugin } from './plugin';
+
+export const plugin = () => new ESQLValidationExamplePlugin();

examples/esql_validation_example/public/mount.tsx

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as React from 'react';
+import { render, unmountComponentAtNode } from 'react-dom';
+
+import type { CoreSetup, AppMountParameters } from '@kbn/core/public';
+import type { StartDependencies } from './plugin';
+
+export const mount =
+  (coreSetup: CoreSetup<StartDependencies>) =>
+  async ({ element }: AppMountParameters) => {
+    const [core, plugins] = await coreSetup.getStartServices();
+    const { App } = await import('./app');
+
+    const i18nCore = core.i18n;
+
+    const reactElement = (
+      <i18nCore.Context>
+        <App core={core} plugins={plugins} />
+      </i18nCore.Context>
+    );
+
+    render(reactElement, element);
+    return () => {
+      unmountComponentAtNode(element);
+      plugins.data.search.session.clear();
+    };
+  };

examples/esql_validation_example/public/plugin.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Plugin, CoreSetup } from '@kbn/core/public';
+import { DataPublicPluginStart } from '@kbn/data-plugin/public';
+import { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
+import { DeveloperExamplesSetup } from '@kbn/developer-examples-plugin/public';
+import { mount } from './mount';
+import image from './esql_validation_app.png';
+
+export interface SetupDependencies {
+  developerExamples: DeveloperExamplesSetup;
+}
+
+export interface StartDependencies {
+  data: DataPublicPluginStart;
+  dataViews: DataViewsPublicPluginStart;
+}
+
+export class ESQLValidationExamplePlugin
+  implements Plugin<void, void, SetupDependencies, StartDependencies>
+{
+  public setup(core: CoreSetup<StartDependencies>, { developerExamples }: SetupDependencies) {
+    core.application.register({
+      id: 'esql_validation_example',
+      title: 'ES|QL Validation example',
+      visibleIn: [],
+      mount: mount(core),
+    });
+
+    developerExamples.register({
+      appId: 'esql_validation_example',
+      title: 'ES|QL Validation',
+      description: 'Validate ES|QL queries using the @kbn/esql-validation-autocomplete package.',
+      image,
+      links: [
+        {
+          label: 'README',
+          href: 'https://github.com/elastic/kibana/tree/main/packages/kbn-esql-validation-autocomplete/README.md',
+          iconType: 'logoGithub',
+          size: 's',
+          target: '_blank',
+        },
+      ],
+    });
+  }
+
+  public start() {}
+
+  public stop() {}
+}

examples/esql_validation_example/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "index.ts",
+    "public/**/*.ts",
+    "public/**/*.tsx",
+    "server/**/*.ts",
+    "../../../typings/**/*"
+  ],
+  "exclude": [
+    "target/**/*",
+  ],
+  "kbn_references": [
+    "@kbn/core",
+    "@kbn/data-plugin",
+    "@kbn/data-views-plugin",
+    "@kbn/developer-examples-plugin",
+    "@kbn/esql-ast",
+    "@kbn/esql-validation-autocomplete",
+  ]
+}

package.json

@@ -433,7 +433,10 @@
     "@kbn/es-ui-shared-plugin": "link:src/plugins/es_ui_shared",
     "@kbn/eso-model-version-example": "link:examples/eso_model_version_example",
     "@kbn/eso-plugin": "link:x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin",
+    "@kbn/esql-ast": "link:packages/kbn-esql-ast",
     "@kbn/esql-utils": "link:packages/kbn-esql-utils",
+    "@kbn/esql-validation-autocomplete": "link:packages/kbn-esql-validation-autocomplete",
+    "@kbn/esql-validation-example-plugin": "link:examples/esql_validation_example",
     "@kbn/event-annotation-common": "link:packages/kbn-event-annotation-common",
     "@kbn/event-annotation-components": "link:packages/kbn-event-annotation-components",
     "@kbn/event-annotation-listing-plugin": "link:src/plugins/event_annotation_listing",

packages/kbn-esql-ast/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+
+SRCS = glob(
+  [
+    "**/*.ts",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SHARED_DEPS = [
+  "@npm//antlr4",
+]
+
+js_library(
+  name = "kbn-esql-ast",
+  package_name = "@kbn/esql-ast",
+  srcs = ["package.json"] + SRCS,
+  deps = SHARED_DEPS,
+  visibility = ["//visibility:public"],
+)

packages/kbn-esql-ast/README.md

@@ -0,0 +1,89 @@
+# ES|QL utility library
+
+## Folder structure
+
+This library brings all the foundation data structure to enable all advanced features within an editor for ES|QL as validation, autocomplete, hover, etc...
+The package is structure as follow:
+
+```
+src
+  |- antlr                      // => contains the ES|QL grammar files and various compilation assets
+  | ast_factory.ts              // => binding to the Antlr that generates the AST data structure
+  | ast_errors.ts               // => error translation utility from raw Antlr to something understandable (somewhat)
+  | antlr_error_listener.ts     // => The ES|QL syntax error listener
+  | antlr_facade.ts             // => getParser and getLexer utilities
+  | ...                         // => miscellaneas utilities to work with AST
+```
+
+### Basic usage
+
+#### Get AST from a query string
+
+This module contains the entire logic to translate from a query string into the AST data structure.
+The `getAstAndSyntaxErrors` function returns the AST data structure, unless a syntax error happens in which case the `errors` array gets populated with a Syntax error.
+
+##### Usage
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+
+const queryString = "from index | stats 1 + avg(myColumn) ";
+const { ast, errors} = await astProvider(queryString);
+
+if(errors){
+  console.log({ syntaxErrors: errors });
+}
+// do stuff with the ast
+```
+
+## How does it work
+
+The general idea of this package is to provide all ES|QL features on top of a custom compact AST definition (all data structure types defined in `./types.ts`) which is designed to be resilient to many grammar changes.
+The pipeline is the following:
+
+```
+Antlr grammar files
+=> Compiled grammar files (.ts assets in the antlr folder)
+=> AST Factory (Antlr Parser tree => custom AST)
+```
+
+Each feature function works with the combination of the AST and the definition files: the former describe the current statement in a easy to traverse way, while the definitions describe what's the expected behaviour of each node in the AST node (i.e. what arguments should it accept? How many arguments? etc...).
+While AST requires the grammar to be compiled to be updated, definitions are static files which can be dynamically updated without running the ANTLR compile task.
+
+#### AST
+
+The AST is generated by 2 files: `ast_factory.ts` and its buddy `ast_walker.ts`:
+* `ast_factory.ts` is a binding to Antlr and access the Parser tree
+* Parser tree is passed over to `ast_walker` to append new AST nodes
+
+In general Antlr is resilient to grammar errors, in the sense that it can produe a Parser tree up to the point of the error, then stops. This is useful to perform partial tasks even with broken queries and this means that a partial AST can be produced even with an invalid query.
+
+### Keeping ES|QL up to date
+
+In general when operating on changes here use the `yarn kbn watch` in a terminal window to make sure changes are correctly compiled.
+
+### How to add new commands/options
+
+When a new command/option is added to ES|QL it is done via a grammar update.
+Therefore adding them requires a two step phase:
+* Update the grammar with the new one
+    * add/fix all AST generator bindings in case of new/changed TOKENS in the `lexer` grammar file
+* Update the definition files for commands/options
+
+To update the grammar:
+1. Make sure the `lexer` and `parser` files are up to date with their ES counterparts
+  * an existing Kibana CI job is updating them already automatically
+2. Run the script into the `package.json` to compile the ES|QL grammar.
+3. open the `ast_factory.ts` file and add a new `exit<Command/Option>` method
+4. write some code in the `ast_walker/ts` to translate the Antlr Parser tree into the custom AST (there are already few utilites for that, but sometimes it is required to write some more code if the `parser` introduced a new flow)
+  * pro tip: use the `http://lab.antlr.org/` to visualize/debug the parser tree for a given statement (copy and paste the grammar files there)
+5. if something goes wrong with new quoted/unquoted identifier token, open the `ast_helpers.ts` and check the ids of the new tokens in the `getQuotedText` and `getUnquotedText` functions - please make sure to leave a comment on the token name
+
+#### Debug and fix grammar changes (tokens, etc...)
+
+On TOKEN renaming or with subtle `lexer` grammar changes it can happens that test breaks, this can be happen for two main issues:
+* A TOKEN name changed so the `ast_walker.ts` doesn't find it any more. Go there and rename the TOKEN name.
+* TOKEN order changed and tests started failing. This probably generated some TOKEN id reorder and there are two functions in `ast_helpers.ts` who rely on hardcoded ids: `getQuotedText` and `getUnquotedText`.
+  * Note that the `getQuotedText` and `getUnquotedText` are automatically updated on grammar changes detected by the Kibana CI sync job.
+  * to fix this just look at the commented tokens and update the ids. If a new token add it and leave a comment to point to the new token name.
+  * This choice was made to reduce the bundle size, as importing the `esql_parser` adds some hundreds of Kbs to the bundle otherwise.

packages/kbn-esql-ast/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type {
+  ESQLAst,
+  ESQLAstItem,
+  ESQLCommand,
+  ESQLCommandOption,
+  ESQLCommandMode,
+  ESQLFunction,
+  ESQLTimeInterval,
+  ESQLLocation,
+  ESQLMessage,
+  ESQLSingleAstItem,
+  ESQLSource,
+  ESQLColumn,
+  ESQLLiteral,
+  AstProviderFn,
+  EditorError,
+} from './src/types';
+
+// Low level functions to parse grammar
+export { getParser, getLexer, ROOT_STATEMENT } from './src/antlr_facade';
+
+/**
+ * ES|QL Query string -> AST data structure
+ * this is the foundational building block for any advanced feature
+ * a developer wants to build on top of the ESQL language
+ **/
+export { getAstAndSyntaxErrors } from './src/ast_parser';
+
+export { ESQLErrorListener } from './src/antlr_error_listener';

packages/kbn-esql-ast/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+    "type": "shared-common",
+    "id": "@kbn/esql-ast",
+    "owner": "@elastic/kibana-visualizations"
+  }

packages/kbn-esql-ast/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "@kbn/esql-ast",
+    "version": "1.0.0",
+    "private": true,
+    "license": "SSPL-1.0 OR Elastic License 2.0",
+    "scripts": {
+        "build:antlr4:esql": "antlr -Dlanguage=TypeScript src/antlr/esql_lexer.g4 src/antlr/esql_parser.g4 && node ./scripts/fix_generated_antlr.js && node ./scripts/esql_update_ast_script.js",
+        "prebuild:antlr4": "brew bundle --file=./scripts/antlr4_tools/brewfile",
+        "build:antlr4": "npm run build:antlr4:esql"
+    },
+    "sideEffects": false
+  }

packages/kbn-esql-ast/scripts/antlr4_tools/.gitignore

@@ -0,0 +1 @@
+brewfile.lock.json

packages/kbn-esql-ast/scripts/antlr4_tools/README.md

@@ -0,0 +1,3 @@
+## antlr4-tools
+
+defines a fairly quick way to get [antlr](https://github.com/antlr/antlr4) setup on any machine with brew support, simply run `brew bundle` within this directory. On running the aforementioned command, `antlr` should be become available in your path as a valid executable.

packages/kbn-esql-ast/scripts/antlr4_tools/brewfile

@@ -0,0 +1,2 @@
+brew "openjdk"
+brew "antlr"

packages/kbn-esql-ast/scripts/esql_update_ast_script.js

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { join } = require('path');
+const { ESLint } = require('eslint');
+const partition = require('lodash/partition');
+const { readdirSync, readFileSync, writeFileSync } = require('fs');
+const ora = require('ora');
+const log = ora('Updating ES|QL AST walker from antlr grammar').start();
+
+async function execute() {
+  const generatedAntlrFolder = join(__dirname, '..', 'src', 'antlr');
+
+  const generatedAntlrFolderContents = readdirSync(generatedAntlrFolder);
+
+  const tokenRegex = /public static readonly (?<name>[A-Z_]*(UN)*QUOTED_[A-Z_]+) = (?<value>\d+);/;
+  const lexerFile = generatedAntlrFolderContents.find((file) => file === 'esql_parser.ts');
+  const lexerFileRows = readFileSync(join(generatedAntlrFolder, lexerFile), 'utf8')
+    .toString()
+    .split('\n');
+  const tokenList = [];
+  for (const row of lexerFileRows) {
+    const match = row.match(tokenRegex);
+    if (match?.groups) {
+      tokenList.push(match.groups);
+    }
+  }
+  const [unquotedList, quotedList] = partition(tokenList, ({ name }) => /UNQUOTED/.test(name));
+
+  // now all quote/unquoted tokens are registered
+  // dump them into the ast_helper file
+  const astHelperFileFolder = join(__dirname, '..', 'src');
+  const astHelperFilename = 'ast_helpers.ts';
+
+  try {
+    const astHelperContentRows = readFileSync(join(astHelperFileFolder, astHelperFilename), 'utf8')
+      .toString()
+      .split('\n');
+
+    const startAutoGeneratedComment = astHelperContentRows.findIndex(
+      (row) => row === '/* SCRIPT_MARKER_START */'
+    );
+    const endAutoGeneratedComment =
+      astHelperContentRows.findIndex((row) => row === '/* SCRIPT_MARKER_END */') + 1;
+
+    const newFunctionsContent = `
+/* SCRIPT_MARKER_START */
+function getQuotedText(ctx: ParserRuleContext) {
+    return [
+        ${quotedList.map(({ name, value }) => `${value} /* esql_parser.${name} */`).join(', ')}
+    ]
+        .map((keyCode) => ctx.getToken(keyCode, 0))
+        .filter(nonNullable)[0];
+    }
+    
+function getUnquotedText(ctx: ParserRuleContext) {
+    return [
+        ${unquotedList.map(({ name, value }) => `${value} /* esql_parser.${name} */`).join(', ')}
+
+    ]
+        .map((keyCode) => ctx.getToken(keyCode, 0))
+        .filter(nonNullable)[0];
+}
+/* SCRIPT_MARKER_END */
+`;
+
+    const fileContent = astHelperContentRows
+      .slice(0, startAutoGeneratedComment)
+      .concat(newFunctionsContent.split('\n'), astHelperContentRows.slice(endAutoGeneratedComment));
+
+    const fileContentString = fileContent.join('\n');
+
+    const eslint = new ESLint({
+      fix: true,
+      overrideConfig: {
+        parser: '@typescript-eslint/parser',
+        parserOptions: {
+          sourceType: 'module',
+          ecmaVersion: 2018,
+        },
+        rules: {
+          '@kbn/imports/no_unresolvable_imports': 'off',
+          'prettier/prettier': [
+            'error',
+            {
+              parser: 'typescript',
+            },
+          ],
+        },
+      },
+    });
+
+    const results = await eslint.lintText(fileContentString);
+
+    if (results.some(({ messages }) => messages.length > 0)) {
+      const formatter = await eslint.loadFormatter('stylish');
+      const resultText = formatter.format(results);
+      process.exitCode = 1;
+      return log.fail(resultText);
+    }
+
+    const filePath = join(astHelperFileFolder, astHelperFilename);
+    writeFileSync(filePath, results[0].output || '', { encoding: 'utf8' });
+  } catch (err) {
+    return log.fail(err.message);
+  }
+
+  log.succeed('Updated ES|QL helper from antlr grammar successfully');
+}
+
+execute();

packages/kbn-esql-ast/scripts/fix_generated_antlr.js

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { join } = require('path');
+const { readdirSync, readFileSync, writeFileSync, renameSync } = require('fs');
+const ora = require('ora');
+const log = ora('Updating generated antlr grammar').start();
+
+function execute() {
+  const generatedAntlrFolder = join(__dirname, '..', 'src', 'antlr');
+
+  const generatedAntlrFolderContents = readdirSync(generatedAntlrFolder);
+
+  // The generated TS produces some TS linting errors
+  // This script adds a //@ts-nocheck comment at the top of each generated file
+  // so that the errors can be ignored for now
+  generatedAntlrFolderContents
+    .filter((file) => {
+      const fileExtension = file.split('.')[1];
+      return fileExtension.includes('ts');
+    })
+    .forEach((file) => {
+      try {
+        const fileContentRows = readFileSync(join(generatedAntlrFolder, file), 'utf8')
+          .toString()
+          .split('\n');
+
+        if (!/\@ts-nocheck/.test(fileContentRows[0])) {
+          fileContentRows.unshift('// @ts-nocheck');
+        }
+
+        const filePath = join(generatedAntlrFolder, file);
+        const fileContent = fileContentRows.join('\n');
+
+        writeFileSync(filePath, fileContent, { encoding: 'utf8' });
+      } catch (err) {
+        return log.fail(err.message);
+      }
+    });
+
+  // Rename generated parserListener file to snakecase to satisfy file casing check
+  // There doesn't appear to be a way to fix this OOTB with antlr4
+  try {
+    renameSync(
+      join(generatedAntlrFolder, `esql_parserListener.ts`),
+      join(generatedAntlrFolder, `esql_parser_listener.ts`)
+    );
+  } catch (err) {
+    log.warn(`Unable to rename parserListener file to snake-case: ${err.message}`);
+  }
+
+  log.succeed('Updated generated antlr grammar successfully');
+}
+
+execute();

packages/kbn-esql-ast/src/antlr/esql_lexer.ts

@@ -1,5 +1,5 @@
 // @ts-nocheck
-// Generated from src/esql/antlr/esql_lexer.g4 by ANTLR 4.13.1
+// Generated from src/antlr/esql_lexer.g4 by ANTLR 4.13.1
 // noinspection ES6UnusedImports,JSUnusedGlobalSymbols,JSUnusedLocalSymbols
 import {
 	ATN,

packages/kbn-esql-ast/src/antlr/esql_parser.ts

@@ -1,5 +1,5 @@
 // @ts-nocheck
-// Generated from src/esql/antlr/esql_parser.g4 by ANTLR 4.13.1
+// Generated from src/antlr/esql_parser.g4 by ANTLR 4.13.1
 // noinspection ES6UnusedImports,JSUnusedGlobalSymbols,JSUnusedLocalSymbols
 
 import {

packages/kbn-esql-ast/src/antlr/esql_parser_listener.ts

@@ -1,5 +1,5 @@
 // @ts-nocheck
-// Generated from src/esql/antlr/esql_parser.g4 by ANTLR 4.13.1
+// Generated from src/antlr/esql_parser.g4 by ANTLR 4.13.1
 
 import {ParseTreeListener} from "antlr4";
 

packages/kbn-esql-ast/src/antlr_error_listener.ts

@@ -7,10 +7,13 @@
  */
 
 import type { Recognizer, RecognitionException } from 'antlr4';
-import { ANTLRErrorListener } from '../../../common/error_listener';
-import { getPosition } from '../ast/ast_position_utils';
+import { ErrorListener } from 'antlr4';
+import type { EditorError } from './types';
+import { getPosition } from './ast_position_utils';
+
+export class ESQLErrorListener extends ErrorListener<any> {
+  protected errors: EditorError[] = [];
 
-export class ESQLErrorListener extends ANTLRErrorListener {
   syntaxError(
     recognizer: Recognizer<any>,
     offendingSymbol: any,
@@ -31,7 +34,11 @@ export class ESQLErrorListener extends ANTLRErrorListener {
       startColumn,
       endColumn,
       message: textMessage,
-      severity: 8, // monaco.MarkerSeverity.Error,
+      severity: 'error',
     });
   }
+
+  getErrors(): EditorError[] {
+    return this.errors;
+  }
 }

packages/kbn-esql-ast/src/antlr_facade.ts

@@ -8,9 +8,9 @@
 
 import { CommonTokenStream, type CharStream, type ErrorListener } from 'antlr4';
 
-import { default as ESQLLexer } from '../antlr/esql_lexer';
-import { default as ESQLParser } from '../antlr/esql_parser';
-import { default as ESQLParserListener } from '../antlr/esql_parser_listener';
+import { default as ESQLLexer } from './antlr/esql_lexer';
+import { default as ESQLParser } from './antlr/esql_parser';
+import { default as ESQLParserListener } from './antlr/esql_parser_listener';
 
 export const ROOT_STATEMENT = 'singleStatement';
 

packages/kbn-esql-ast/src/ast_factory.ts

@@ -28,8 +28,8 @@ import {
   type WhereCommandContext,
   default as esql_parser,
   type MetaCommandContext,
-} from '../../antlr/esql_parser';
-import { default as ESQLParserListener } from '../../antlr/esql_parser_listener';
+} from './antlr/esql_parser';
+import { default as ESQLParserListener } from './antlr/esql_parser_listener';
 import {
   createCommand,
   createFunction,

packages/kbn-esql-ast/src/ast_helpers.ts

@@ -16,9 +16,9 @@ import type {
   DecimalValueContext,
   IntegerValueContext,
   QualifiedIntegerLiteralContext,
-} from '../../antlr/esql_parser';
+} from './antlr/esql_parser';
 import { getPosition } from './ast_position_utils';
-import { DOUBLE_TICKS_REGEX, SINGLE_BACKTICK, TICKS_REGEX } from './shared/constants';
+import { DOUBLE_TICKS_REGEX, SINGLE_BACKTICK, TICKS_REGEX } from './constants';
 import type {
   ESQLCommand,
   ESQLLiteral,

packages/kbn-esql-ast/src/ast_parser.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { CharStreams } from 'antlr4';
+import { ESQLErrorListener } from './antlr_error_listener';
+import { getParser, ROOT_STATEMENT } from './antlr_facade';
+import { AstListener } from './ast_factory';
+import type { ESQLAst, EditorError } from './types';
+
+export async function getAstAndSyntaxErrors(text: string | undefined): Promise<{
+  errors: EditorError[];
+  ast: ESQLAst;
+}> {
+  if (text == null) {
+    return { ast: [], errors: [] };
+  }
+  const errorListener = new ESQLErrorListener();
+  const parseListener = new AstListener();
+  const parser = getParser(CharStreams.fromString(text), errorListener, parseListener);
+
+  parser[ROOT_STATEMENT]();
+
+  return { ...parseListener.getAst(), errors: errorListener.getErrors() };
+}

packages/kbn-esql-ast/src/ast_walker.ts

@@ -57,7 +57,7 @@ import {
   type ValueExpressionContext,
   ValueExpressionDefaultContext,
   FromIdentifierContext,
-} from '../../antlr/esql_parser';
+} from './antlr/esql_parser';
 import {
   createSource,
   createColumn,
@@ -499,7 +499,7 @@ export function visitByOption(ctx: StatsCommandContext, expr: FieldsContext | un
 }
 
 export function visitOrderExpression(ctx: OrderExpressionContext[]) {
-  const ast = [];
+  const ast: ESQLAstItem[] = [];
   for (const orderCtx of ctx) {
     const expression = collectBooleanExpression(orderCtx.booleanExpression());
     if (orderCtx._ordering) {

packages/kbn-esql-ast/src/types.ts

@@ -6,8 +6,6 @@
  * Side Public License, v 1.
  */
 
-import { EditorError } from '../../../types';
-
 export type ESQLAst = ESQLCommand[];
 
 export type ESQLSingleAstItem =
@@ -87,6 +85,17 @@ export interface ESQLMessage {
   code: string;
 }
 
-export type AstProviderFn = (
-  text: string | undefined
-) => Promise<{ ast: ESQLAst; errors: EditorError[] }>;
+export type AstProviderFn = (text: string | undefined) => Promise<{
+  ast: ESQLAst;
+  errors: EditorError[];
+}>;
+
+export interface EditorError {
+  startLineNumber: number;
+  endLineNumber: number;
+  startColumn: number;
+  endColumn: number;
+  message: string;
+  code?: string;
+  severity: 'error' | 'warning' | number;
+}

packages/kbn-esql-ast/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*",
+    "**/*.ts",
+  ],
+  "kbn_references": [],
+  "exclude": [
+    "target/**/*",
+  ]
+}

packages/kbn-esql-validation-autocomplete/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+
+SRCS = glob(
+  [
+    "**/*.ts",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SHARED_DEPS = [
+  "//packages/kbn-i18n",
+  "@npm//js-levenshtein",
+]
+
+js_library(
+  name = "kbn-esql-validation-autocomplete",
+  package_name = "@kbn/esql-validation-autocomplete",
+  srcs = ["package.json"] + SRCS,
+  deps = SHARED_DEPS,
+  visibility = ["//visibility:public"],
+)

packages/kbn-esql-validation-autocomplete/README.md

@@ -0,0 +1,191 @@
+# ES|QL utility library
+
+## Folder structure
+
+This library enables all the advanced features for ES|QL, as validation, autocomplete, hover, etc...
+The package is structure as follow:
+
+```
+src
+ | autocomplete         // => the autocomplete/suggest service logic
+ | code_actions         // => the quick fixes service logic
+ | definitions          // => static assets to define all components behaviour of a ES|QL query: commands, functions, etc...
+ | validation           // => the validation logic
+```
+
+### Basic usage
+
+#### Validation
+
+This module contains the validation logic useful to perform a full check of an ES|QL query string.
+The validation service can be gracefully degraded leveraging the `ignoreOnMissingCallbacks` option when it is not possible to pass all callbacks: this is useful in environments where it is not possible to connect to a ES instance to retrieve more metadata, while preserving most of the validation value.
+For instance, not passing the `getSources` callback will report all index mentioned in the ES|QL with the `Unknown index [...]` error, but with the `ignoreOnMissingCallbacks` option enabled this type of errors will be muted.
+
+##### Usage
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { validateQuery } from '@kbn/esql-validation-autocomplete';
+
+// define all callbacks
+const myCallbacks = {
+  getSources: async () => [{name: 'index', hidden: false}],
+  ...
+};
+
+// Full validation performed
+const { errors, warnings } = await validateQuery("from index | stats 1 + avg(myColumn)", getAstAndSyntaxErrors, undefined, myCallbacks);
+```
+
+If not all callbacks are available it is possible to gracefully degradate the validation experience with the `ignoreOnMissingCallbacks` option:
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { validateQuery } from '@kbn/esql-validation-autocomplete';
+
+// define only the getSources callback
+const myCallbacks = {
+  getSources: async () => [{name: 'index', hidden: false}],
+};
+
+// ignore errors that might be triggered by the lack of some callbacks (i.e. "Unknown columns", etc...)
+const { errors, warnings } = await validateQuery(
+  "from index | stats 1 + avg(myColumn)",
+  getAstAndSyntaxErrors,
+  { ignoreOnMissingCallbacks: true },
+  myCallbacks
+);
+```
+
+#### Autocomplete
+
+This is the complete logic for the ES|QL autocomplete language, it is completely indepedent from the actual editor (i.e. Monaco) and the suggestions reported need to be wrapped against the specific editor shape.
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { suggest } from '@kbn/esql-validation-autocomplete';
+
+const queryString = "from index | stats 1 + avg(myColumn) ";
+const myCallbacks = {
+  getSources: async () => [{name: 'index', hidden: false}],
+  ...
+};
+
+const suggestions = await suggest(
+  queryString,
+  queryString.length - 1, // the cursor position in a single line context
+  { triggerCharacter: " "; triggerKind: 1 }, // kind = 0 is a programmatic trigger, while other values are ignored
+  getAstAndSyntaxErrors,
+  myCallbacks
+);
+
+// Log the actual text to be injected as suggestion
+console.log(suggestions.map(({text}) => text));
+
+// for Monaco editor it is required to map each suggestion with the editor specific type
+suggestions.map( s => ({
+  label: s.label,
+  insertText: s.text,
+  insertTextRules: asSnippet
+        ? monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet
+        : undefined,
+  ...
+  }))
+```
+
+Note that the autocomplete service will work as best effort with invalid queries, trying to correct them on the fly before generating the suggestions. In case an invalid query cannot be handled an empty suggestion result set will be returned.
+
+#### Quick fixes
+
+This feature provides a list of suggestions to propose as fixes for a subset of validation errors.
+The feature works in combination with the validation service.
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { validateQuery, getActions } from '@kbn/esql-validation-autocomplete';
+
+const queryString = "from index2 | stats 1 + avg(myColumn)"
+
+const myCallbacks = {
+  getSources: async () => [{name: 'index', hidden: false}],
+  ...
+};
+const { errors, warnings } = await validateQuery(queryString, getAstAndSyntaxErrors, undefined, myCallbacks);
+
+const {title, edits} = await getActions(
+  queryString,
+  errors,
+  getAstAndSyntaxErrors,
+  myCallbacks
+);
+
+// log the title of the fix suggestion and the proposed change
+// in this example it should suggest to change from "index2" to "index"
+console.log({ title, edits });
+```
+
+### getAstContext
+
+This is an important function in order to build more features on top of the existing one.
+For instance to show contextual information on Hover the `getAstContext` function can be leveraged to get the correct context for the cursor position:
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { getAstContext } from '@kbn/esql-validation-autocomplete';
+
+const queryString = "from index2 | stats 1 + avg(myColumn)";
+const offset = queryString.indexOf("avg");
+
+const astContext = getAstContext(queryString, getAstAndSyntaxErrors(queryString), offset);
+
+if(astContext.type === "function"){
+  const fnNode = astContext.node;
+  const fnDefinition = getFunctionDefinition(fnNode.name);
+
+  // show something like "avg( field: number ): number"
+  console.log(getFunctionSignature(fnDefinition));
+}
+```
+
+
+### How does it work
+
+The general idea of this package is to provide all ES|QL features on top of a custom compact AST definition (all data structure types defined in `@kbn/esql-ast`) which is designed to be resilient to many grammar changes.
+The pipeline is the following:
+
+```
+Antlr grammar files
+=> Compiled grammar files (.ts assets in the antlr folder)
+=> AST Factory (Antlr Parser tree => custom AST)
+=> featureFn( AST, Definitions, ESQLCallbacks )
+```
+
+Each feature function works with the combination of the AST and the definition files: the former describe the current statement in a easy to traverse way, while the definitions describe what's the expected behaviour of each node in the AST node (i.e. what arguments should it accept? How many arguments? etc...).
+ESQLCallbacks are a set of utilities to retrieve context metadata like fields/index/policies list and policies metadata.
+
+While AST requires the grammar to be compiled to be updated, definitions are static files which can be dynamically updated without running the ANTLR compile task.
+
+#### Validation
+
+Validation takes an AST as input and generates a list of messages to show to the user.
+The validation function leverages the definition files to check if the current AST is respecting the defined behaviour.
+Most of the logic rely purely on the definitions, but in some specific cases some ad-hoc conditions are defined within the code for specific commands/options.
+The validation test suite generates a set of fixtures at the end of its execution, which are then re-used for other test suites (i.e. some FTR integration tests) as `esql_validation_meta_tests.json`.
+
+#### Autocomplete
+
+The autocomplete/suggest task takes a query as input together with the current cursor position, then produces internally an AST to work with, to generate a list of suggestions for the given query.
+Note that autocomplete works most of the time with incomplete/invalid queries, so some logic to manipulate the query into something valid (see the `EDITOR_MARKER` or the `countBracketsUnclosed` functions for more).
+
+Once the AST is produced there's a `getAstContext` function that finds the cursor position node (and its parent command), together with some hint like the type of current context: `expression`, `function`, `newCommand`, `option`.
+The most complex case is the `expression` as it can cover a moltitude of cases. The function is highly commented in order to identify the specific cases, but there's probably some obscure area still to comment/clarify.
+
+### Adding new commands/options/functions/erc...
+
+To update the definitions:
+1. open either approriate definition file within the `definitions` folder and add a new entry to the relative array
+2. write new tests for validation and autocomplete
+  * if a new function is added tests are automatically generated fro both validation and autocomplete with some standard checks
+  * if a new function requires a new field types, make sure to add the new type to the initial part of the test file
+    * this will be automatically picked up by the test generator to produce new test cases
+  * if a new function requires a new type of test, make sure to write it manually

packages/kbn-esql-validation-autocomplete/index.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type { SuggestionRawDefinition, ItemKind } from './src/autocomplete/types';
+export type { CodeAction } from './src/code_actions/types';
+export type {
+  FunctionDefinition,
+  CommandDefinition,
+  CommandOptionsDefinition,
+  CommandModeDefinition,
+  Literals,
+} from './src/definitions/types';
+export type { ESQLCallbacks } from './src/shared/types';
+
+/**
+ * High level functions
+ */
+
+// Given an the query string, its AST and the cursor position, it returns the node and some context information
+export { getAstContext } from './src/shared/context';
+// Validation function
+export { validateQuery } from './src/validation/validation';
+// Autocomplete function
+export { suggest } from './src/autocomplete/autocomplete';
+// Quick fixes function
+export { getActions } from './src/code_actions/actions';
+
+/**
+ * Some utility functions that can be useful to build more feature
+ * for the ES|QL language
+ */
+export type {
+  ValidationErrors,
+  ESQLVariable,
+  ESQLRealField,
+  ESQLPolicy,
+  ErrorTypes as ESQLValidationErrorTypes,
+} from './src/validation/types';
+export { collectVariables } from './src/shared/variables';
+export {
+  getAllFunctions,
+  isSupportedFunction,
+  getFunctionDefinition,
+  getCommandDefinition,
+  getAllCommands,
+  getCommandOption,
+  getColumnHit,
+  columnExists,
+  shouldBeQuotedText,
+  printFunctionSignature,
+  isEqualType,
+  isSourceItem,
+  isSettingItem,
+  isFunctionItem,
+  isOptionItem,
+  isColumnItem,
+  isLiteralItem,
+  isTimeIntervalItem,
+  isAssignment,
+  isExpression,
+  isAssignmentComplete,
+  isSingleItem,
+} from './src/shared/helpers';
+export { ENRICH_MODES } from './src/definitions/settings';
+export { getFunctionSignatures } from './src/definitions/helpers';
+
+export {
+  getFieldsByTypeHelper,
+  getPolicyHelper,
+  getSourcesHelper,
+} from './src/shared/resources_helpers';

packages/kbn-esql-validation-autocomplete/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-esql-validation-autocomplete'],
+};

packages/kbn-esql-validation-autocomplete/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+    "type": "shared-common",
+    "id": "@kbn/esql-validation-autocomplete",
+    "owner": "@elastic/kibana-visualizations"
+  }

packages/kbn-esql-validation-autocomplete/package.json

@@ -0,0 +1,7 @@
+{
+    "name": "@kbn/esql-validation-autocomplete",
+    "version": "1.0.0",
+    "private": true,
+    "license": "SSPL-1.0 OR Elastic License 2.0",
+    "sideEffects": false
+  }

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -6,12 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { monaco } from '../../../../monaco_imports';
-import { CharStreams } from 'antlr4';
 import { suggest } from './autocomplete';
-import { getParser, ROOT_STATEMENT } from '../../antlr_facade';
-import { ESQLErrorListener } from '../../monaco/esql_error_listener';
-import { AstListener } from '../ast_factory';
 import { evalFunctionsDefinitions } from '../definitions/functions';
 import { builtinFunctions } from '../definitions/builtin';
 import { statsAggregationFunctionDefinitions } from '../definitions/aggs';
@@ -19,6 +14,7 @@ import { chronoLiterals, timeLiterals } from '../definitions/literals';
 import { commandDefinitions } from '../definitions/commands';
 import { TRIGGER_SUGGESTION_COMMAND } from './factories';
 import { camelCase } from 'lodash';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
 
 const triggerCharacters = [',', '(', '=', ' '];
 
@@ -180,13 +176,6 @@ function createCustomCallbackMocks(
   };
 }
 
-function createModelAndPosition(text: string, offset: number) {
-  return {
-    model: { getValue: () => text } as monaco.editor.ITextModel,
-    position: { lineNumber: 1, column: offset } as monaco.Position,
-  };
-}
-
 function createSuggestContext(text: string, triggerCharacter?: string) {
   if (triggerCharacter) {
     return { triggerCharacter, triggerKind: 1 }; // any number is fine here
@@ -209,16 +198,6 @@ function getPolicyFields(policyName: string) {
 }
 
 describe('autocomplete', () => {
-  const getAstAndErrors = async (text: string) => {
-    const errorListener = new ESQLErrorListener();
-    const parseListener = new AstListener();
-    const parser = getParser(CharStreams.fromString(text), errorListener, parseListener);
-
-    parser[ROOT_STATEMENT]();
-
-    return { ...parseListener.getAst(), errors: [] };
-  };
-
   type TestArgs = [
     string,
     string[],
@@ -244,7 +223,7 @@ describe('autocomplete', () => {
     const context = createSuggestContext(statement, triggerCharacterString);
     const offset =
       typeof triggerCharacter === 'string'
-        ? statement.lastIndexOf(context.triggerCharacter) + 2
+        ? statement.lastIndexOf(context.triggerCharacter) + 1
         : triggerCharacter;
     const testFn = only ? test.only : skip ? test.skip : test;
 
@@ -254,20 +233,20 @@ describe('autocomplete', () => {
       )}"]`,
       async () => {
         const callbackMocks = createCustomCallbackMocks(...customCallbacksArgs);
-        const { model, position } = createModelAndPosition(statement, offset);
         const suggestions = await suggest(
-          model,
-          position,
+          statement,
+          offset,
           context,
-          async (text) => (text ? await getAstAndErrors(text) : { ast: [], errors: [] }),
+          async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
           callbackMocks
         );
-        expect(
-          suggestions
-            // simulate the editor behaviour for sorting suggestions
-            .sort((a, b) => (a.sortText || '').localeCompare(b.sortText || ''))
-            .map((i) => i.insertText)
-        ).toEqual(expected);
+        const suggestionInertTextSorted = suggestions
+          // simulate the editor behaviour for sorting suggestions
+          .sort((a, b) => (a.sortText || '').localeCompare(b.sortText || ''))
+          .map((i) => i.text);
+        for (const [index, receivedSuggestion] of suggestionInertTextSorted.entries()) {
+          expect(receivedSuggestion).toEqual(expected[index]);
+        }
       }
     );
   };
@@ -340,11 +319,11 @@ describe('autocomplete', () => {
     testSuggestions('from a,', suggestedIndexes);
     testSuggestions('from a, b ', ['metadata $0', ',', '|']);
     testSuggestions('from *,', suggestedIndexes);
-    testSuggestions('from index', suggestedIndexes, 6 /* index index in from */);
-    testSuggestions('from a, b [metadata ]', ['_index', '_score'], 20);
-    testSuggestions('from a, b metadata ', ['_index', '_score'], 19);
-    testSuggestions('from a, b [metadata _index, ]', ['_score'], 27);
-    testSuggestions('from a, b metadata _index, ', ['_score'], 26);
+    testSuggestions('from index', suggestedIndexes, 5 /* space before index */);
+    testSuggestions('from a, b [metadata ]', ['_index', '_score'], ' ]');
+    testSuggestions('from a, b metadata ', ['_index', '_score'], ' ');
+    testSuggestions('from a, b [metadata _index, ]', ['_score'], ' ]');
+    testSuggestions('from a, b metadata _index, ', ['_score'], ' ');
   });
 
   describe('show', () => {
@@ -645,7 +624,7 @@ describe('autocomplete', () => {
           evalMath: true,
         }),
       ],
-      32
+      '('
     );
 
     testSuggestions('from a | eval var0=round(b), var1=round(c) | stats ', [
@@ -657,18 +636,14 @@ describe('autocomplete', () => {
     ]);
 
     // smoke testing with suggestions not at the end of the string
-    testSuggestions(
-      'from a | stats a = min(b) | sort b',
-      ['by $0', ',', '|'],
-      27 /* " " after min(b) */
-    );
+    testSuggestions('from a | stats a = min(b) | sort b', ['by $0', ',', '|'], ') ');
     testSuggestions(
       'from a | stats avg(b) by stringField',
       [
         ...getFieldNamesByType('number'),
         ...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
       ],
-      21 /* b column in avg */
+      '(b'
     );
 
     // while nested functions are not suggested, complete them should be possible via suggestions
@@ -712,7 +687,7 @@ describe('autocomplete', () => {
           'round',
         ]),
       ],
-      27
+      '('
     );
     testSuggestions(
       'from a | stats avg(round(',
@@ -722,7 +697,7 @@ describe('autocomplete', () => {
           'round',
         ]),
       ],
-      26
+      '('
     );
     testSuggestions(
       'from a | stats avg(',
@@ -740,7 +715,7 @@ describe('autocomplete', () => {
           'round',
         ]),
       ],
-      26
+      '('
     );
   });
 
@@ -1159,12 +1134,11 @@ describe('autocomplete', () => {
       const statement = 'from a | drop stringField | eval var0 = abs(numberField) ';
       const triggerOffset = statement.lastIndexOf(' ');
       const context = createSuggestContext(statement, statement[triggerOffset]);
-      const { model, position } = createModelAndPosition(statement, triggerOffset + 2);
       await suggest(
-        model,
-        position,
+        statement,
+        triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
       expect(callbackMocks.getFieldsFor).toHaveBeenCalledWith({
@@ -1176,12 +1150,11 @@ describe('autocomplete', () => {
       const statement = 'from a | drop | eval var0 = abs(numberField) ';
       const triggerOffset = statement.lastIndexOf('p') + 1; // drop <here>
       const context = createSuggestContext(statement, statement[triggerOffset]);
-      const { model, position } = createModelAndPosition(statement, triggerOffset + 2);
       await suggest(
-        model,
-        position,
+        statement,
+        triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
       expect(callbackMocks.getFieldsFor).toHaveBeenCalledWith({ query: 'from a' });
@@ -1193,12 +1166,11 @@ describe('autocomplete', () => {
       const callbackMocks = createCustomCallbackMocks(undefined, undefined, undefined);
       const triggerOffset = statement.lastIndexOf(' ') + 1; // drop <here>
       const context = createSuggestContext(statement, statement[triggerOffset]);
-      const { model, position } = createModelAndPosition(statement, triggerOffset + 2);
       return suggest(
-        model,
-        position,
+        statement,
+        triggerOffset + 1,
         context,
-        async (text) => (text ? await getAstAndErrors(text) : { ast: [], errors: [] }),
+        async (text) => (text ? await getAstAndSyntaxErrors(text) : { ast: [], errors: [] }),
         callbackMocks
       );
     }
@@ -1207,12 +1179,14 @@ describe('autocomplete', () => {
       // test that all functions will retrigger suggestions
       expect(
         suggestions
-          .filter(({ kind }) => kind === 1)
+          .filter(({ kind }) => kind === 'Function')
           .every(({ command }) => command === TRIGGER_SUGGESTION_COMMAND)
       ).toBeTruthy();
       // now test that non-function won't retrigger
       expect(
-        suggestions.filter(({ kind }) => kind !== 1).every(({ command }) => command == null)
+        suggestions
+          .filter(({ kind }) => kind !== 'Function')
+          .every(({ command }) => command == null)
       ).toBeTruthy();
     });
     it('should trigger further suggestions for commands', async () => {

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts

@@ -7,9 +7,15 @@
  */
 
 import uniqBy from 'lodash/uniqBy';
-import type { monaco } from '../../../../monaco_imports';
-import type { AutocompleteCommandDefinition } from './types';
-import { nonNullable } from '../ast_helpers';
+import type {
+  AstProviderFn,
+  ESQLAstItem,
+  ESQLCommand,
+  ESQLCommandOption,
+  ESQLFunction,
+  ESQLSingleAstItem,
+} from '@kbn/esql-ast';
+import type { EditorContext, SuggestionRawDefinition } from './types';
 import {
   columnExists,
   getColumnHit,
@@ -32,19 +38,11 @@ import {
   isSettingItem,
   isSourceItem,
   isTimeIntervalItem,
-  monacoPositionToOffset,
   getAllFunctions,
   isSingleItem,
+  nonNullable,
 } from '../shared/helpers';
 import { collectVariables, excludeVariablesFromCurrentCommand } from '../shared/variables';
-import type {
-  AstProviderFn,
-  ESQLAstItem,
-  ESQLCommand,
-  ESQLCommandOption,
-  ESQLFunction,
-  ESQLSingleAstItem,
-} from '../types';
 import type { ESQLPolicy, ESQLRealField, ESQLVariable, ReferenceMaps } from '../validation/types';
 import {
   colonCompleteItem,
@@ -82,13 +80,13 @@ import {
 import { ESQLCallbacks } from '../shared/types';
 import { getFunctionsToIgnoreForStats, isAggFunctionUsedAlready } from './helper';
 
-type GetSourceFn = () => Promise<AutocompleteCommandDefinition[]>;
+type GetSourceFn = () => Promise<SuggestionRawDefinition[]>;
 type GetFieldsByTypeFn = (
   type: string | string[],
   ignored?: string[]
-) => Promise<AutocompleteCommandDefinition[]>;
+) => Promise<SuggestionRawDefinition[]>;
 type GetFieldsMapFn = () => Promise<Map<string, ESQLRealField>>;
-type GetPoliciesFn = () => Promise<AutocompleteCommandDefinition[]>;
+type GetPoliciesFn = () => Promise<SuggestionRawDefinition[]>;
 type GetPolicyMetadataFn = (name: string) => Promise<ESQLPolicy | undefined>;
 type GetMetaFieldsFn = () => Promise<string[]>;
 
@@ -147,15 +145,12 @@ function countBracketsUnclosed(bracketType: '(' | '[', text: string) {
 }
 
 export async function suggest(
-  model: monaco.editor.ITextModel,
-  position: monaco.Position,
-  context: monaco.languages.CompletionContext,
+  fullText: string,
+  offset: number,
+  context: EditorContext,
   astProvider: AstProviderFn,
   resourceRetriever?: ESQLCallbacks
-): Promise<AutocompleteCommandDefinition[]> {
-  // take the full text but then slice it to the current position
-  const fullText = model.getValue();
-  const offset = monacoPositionToOffset(fullText, position);
+): Promise<SuggestionRawDefinition[]> {
   const innerText = fullText.substring(0, offset);
 
   let finalText = innerText;
@@ -564,7 +559,7 @@ async function getExpressionSuggestionsByType(
 
   const references = { fields: fieldsMap, variables: anyVariables };
 
-  const suggestions: AutocompleteCommandDefinition[] = [];
+  const suggestions: SuggestionRawDefinition[] = [];
 
   // in this flow there's a clear plan here from argument definitions so try to follow it
   if (argDef) {
@@ -877,8 +872,8 @@ async function getExpressionSuggestionsByType(
     }
   }
   // Due to some logic overlapping functions can be repeated
-  // so dedupe here based on insertText string (it can differ from name)
-  return uniqBy(suggestions, (suggestion) => suggestion.insertText);
+  // so dedupe here based on text string (it can differ from name)
+  return uniqBy(suggestions, (suggestion) => suggestion.text);
 }
 
 async function getBuiltinFunctionNextArgument(
@@ -965,7 +960,7 @@ async function getBuiltinFunctionNextArgument(
   return suggestions;
 }
 
-function pushItUpInTheList(suggestions: AutocompleteCommandDefinition[], shouldPromote: boolean) {
+function pushItUpInTheList(suggestions: SuggestionRawDefinition[], shouldPromote: boolean) {
   if (!shouldPromote) {
     return suggestions;
   }
@@ -996,9 +991,9 @@ async function getFieldsOrFunctionsSuggestions(
     ignoreFn?: string[];
     ignoreFields?: string[];
   } = {}
-): Promise<AutocompleteCommandDefinition[]> {
+): Promise<SuggestionRawDefinition[]> {
   const filteredFieldsByType = pushItUpInTheList(
-    (await (fields ? getFieldsByType(types, ignoreFields) : [])) as AutocompleteCommandDefinition[],
+    (await (fields ? getFieldsByType(types, ignoreFields) : [])) as SuggestionRawDefinition[],
     functions
   );
 
@@ -1058,7 +1053,7 @@ async function getFunctionArgsSuggestions(
   getFieldsByType: GetFieldsByTypeFn,
   getFieldsMap: GetFieldsMapFn,
   getPolicyMetadata: GetPolicyMetadataFn
-): Promise<AutocompleteCommandDefinition[]> {
+): Promise<SuggestionRawDefinition[]> {
   const fnDefinition = getFunctionDefinition(node.name);
   // early exit on no hit
   if (!fnDefinition) {
@@ -1183,19 +1178,18 @@ async function getFunctionArgsSuggestions(
       suggestion !== commaCompleteItem
         ? {
             ...suggestion,
-            insertText:
+            text:
               hasMoreMandatoryArgs && fnDefinition.type !== 'builtin'
-                ? `${suggestion.insertText},`
-                : suggestion.insertText,
+                ? `${suggestion.text},`
+                : suggestion.text,
           }
         : suggestion
     );
   }
 
-  return suggestions.map(({ insertText, ...rest }) => ({
+  return suggestions.map(({ text, ...rest }) => ({
     ...rest,
-    insertText:
-      hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' ? `${insertText},` : insertText,
+    text: hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' ? `${text},` : text,
   }));
 }
 

packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts

@@ -7,36 +7,36 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import type { AutocompleteCommandDefinition } from './types';
+import type { SuggestionRawDefinition } from './types';
 import { statsAggregationFunctionDefinitions } from '../definitions/aggs';
 import { builtinFunctions } from '../definitions/builtin';
 import { evalFunctionsDefinitions } from '../definitions/functions';
 import { getAllCommands } from '../shared/helpers';
 import {
-  getAutocompleteFunctionDefinition,
-  getAutocompleteBuiltinDefinition,
-  getAutocompleteCommandDefinition,
+  getSuggestionFunctionDefinition,
+  getSuggestionBuiltinDefinition,
+  getSuggestionCommandDefinition,
   TRIGGER_SUGGESTION_COMMAND,
   buildConstantsDefinitions,
 } from './factories';
 
-export const mathCommandDefinition: AutocompleteCommandDefinition[] = evalFunctionsDefinitions.map(
-  getAutocompleteFunctionDefinition
+export const mathCommandDefinition: SuggestionRawDefinition[] = evalFunctionsDefinitions.map(
+  getSuggestionFunctionDefinition
 );
 
-export const aggregationFunctionsDefinitions: AutocompleteCommandDefinition[] =
-  statsAggregationFunctionDefinitions.map(getAutocompleteFunctionDefinition);
+export const aggregationFunctionsDefinitions: SuggestionRawDefinition[] =
+  statsAggregationFunctionDefinitions.map(getSuggestionFunctionDefinition);
 
 export function getAssignmentDefinitionCompletitionItem() {
   const assignFn = builtinFunctions.find(({ name }) => name === '=')!;
-  return getAutocompleteBuiltinDefinition(assignFn);
+  return getSuggestionBuiltinDefinition(assignFn);
 }
 
 export const getNextTokenForNot = (
   command: string,
   option: string | undefined,
   argType: string
-): AutocompleteCommandDefinition[] => {
+): SuggestionRawDefinition[] => {
   const compatibleFunctions = builtinFunctions.filter(
     ({ name, supportedCommands, supportedOptions, ignoreAsSuggestion }) =>
       !ignoreAsSuggestion &&
@@ -47,14 +47,14 @@ export const getNextTokenForNot = (
     // suggest IS, LIKE, RLIKE and TRUE/FALSE
     return compatibleFunctions
       .filter(({ name }) => name === 'like' || name === 'rlike' || name === 'in')
-      .map(getAutocompleteBuiltinDefinition);
+      .map(getSuggestionBuiltinDefinition);
   }
   if (argType === 'boolean') {
     // suggest IS, NOT and TRUE/FALSE
     return [
       ...compatibleFunctions
         .filter(({ name }) => name === 'in')
-        .map(getAutocompleteBuiltinDefinition),
+        .map(getSuggestionBuiltinDefinition),
       ...buildConstantsDefinitions(['true', 'false']),
     ];
   }
@@ -67,7 +67,7 @@ export const getBuiltinCompatibleFunctionDefinition = (
   argType: string,
   returnTypes?: string[],
   { skipAssign }: { skipAssign?: boolean } = {}
-): AutocompleteCommandDefinition[] => {
+): SuggestionRawDefinition[] => {
   const compatibleFunctions = builtinFunctions.filter(
     ({ name, supportedCommands, supportedOptions, signatures, ignoreAsSuggestion }) =>
       !ignoreAsSuggestion &&
@@ -80,7 +80,7 @@ export const getBuiltinCompatibleFunctionDefinition = (
       )
   );
   if (!returnTypes) {
-    return compatibleFunctions.map(getAutocompleteBuiltinDefinition);
+    return compatibleFunctions.map(getSuggestionBuiltinDefinition);
   }
   return compatibleFunctions
     .filter((mathDefinition) =>
@@ -88,29 +88,29 @@ export const getBuiltinCompatibleFunctionDefinition = (
         (signature) => returnTypes[0] === 'any' || returnTypes.includes(signature.returnType)
       )
     )
-    .map(getAutocompleteBuiltinDefinition);
+    .map(getSuggestionBuiltinDefinition);
 };
 
-export const commandAutocompleteDefinitions: AutocompleteCommandDefinition[] = getAllCommands().map(
-  getAutocompleteCommandDefinition
+export const commandAutocompleteDefinitions: SuggestionRawDefinition[] = getAllCommands().map(
+  getSuggestionCommandDefinition
 );
 
 function buildCharCompleteItem(
   label: string,
   detail: string,
   { sortText, quoted }: { sortText?: string; quoted: boolean } = { quoted: false }
-): AutocompleteCommandDefinition {
+): SuggestionRawDefinition {
   return {
     label,
-    insertText: quoted ? `"${label}"` : label,
-    kind: 11,
+    text: quoted ? `"${label}"` : label,
+    kind: 'Operator',
     detail,
     sortText,
   };
 }
 export const pipeCompleteItem = buildCharCompleteItem(
   '|',
-  i18n.translate('monaco.esql.autocomplete.pipeDoc', {
+  i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.pipeDoc', {
     defaultMessage: 'Pipe (|)',
   }),
   { sortText: 'C', quoted: false }
@@ -118,7 +118,7 @@ export const pipeCompleteItem = buildCharCompleteItem(
 
 export const commaCompleteItem = buildCharCompleteItem(
   ',',
-  i18n.translate('monaco.esql.autocomplete.commaDoc', {
+  i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.commaDoc', {
     defaultMessage: 'Comma (,)',
   }),
   { sortText: 'B', quoted: false }
@@ -126,25 +126,25 @@ export const commaCompleteItem = buildCharCompleteItem(
 
 export const colonCompleteItem = buildCharCompleteItem(
   ':',
-  i18n.translate('monaco.esql.autocomplete.colonDoc', {
+  i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.colonDoc', {
     defaultMessage: 'Colon (:)',
   }),
   { sortText: 'A', quoted: true }
 );
 export const semiColonCompleteItem = buildCharCompleteItem(
   ';',
-  i18n.translate('monaco.esql.autocomplete.semiColonDoc', {
+  i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.semiColonDoc', {
     defaultMessage: 'Semi colon (;)',
   }),
   { sortText: 'A', quoted: true }
 );
 
-export const listCompleteItem: AutocompleteCommandDefinition = {
+export const listCompleteItem: SuggestionRawDefinition = {
   label: '( ... )',
-  insertText: '( $0 )',
-  insertTextRules: 4,
-  kind: 11,
-  detail: i18n.translate('monaco.esql.autocomplete.listDoc', {
+  text: '( $0 )',
+  asSnippet: true,
+  kind: 'Operator',
+  detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.listDoc', {
     defaultMessage: 'List of items ( ...)',
   }),
   sortText: 'A',

packages/kbn-esql-validation-autocomplete/src/autocomplete/documentation_util.ts

@@ -8,13 +8,19 @@
 
 import { i18n } from '@kbn/i18n';
 
-const declarationLabel = i18n.translate('monaco.esql.autocomplete.declarationLabel', {
-  defaultMessage: 'Declaration:',
-});
+const declarationLabel = i18n.translate(
+  'kbn-esql-validation-autocomplete.esql.autocomplete.declarationLabel',
+  {
+    defaultMessage: 'Declaration:',
+  }
+);
 
-const examplesLabel = i18n.translate('monaco.esql.autocomplete.examplesLabel', {
-  defaultMessage: 'Examples:',
-});
+const examplesLabel = i18n.translate(
+  'kbn-esql-validation-autocomplete.esql.autocomplete.examplesLabel',
+  {
+    defaultMessage: 'Examples:',
+  }
+);
 
 /** @internal */
 export const buildFunctionDocumentation = (

packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts

@@ -7,7 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { AutocompleteCommandDefinition } from './types';
+import { SuggestionRawDefinition } from './types';
 import { statsAggregationFunctionDefinitions } from '../definitions/aggs';
 import { evalFunctionsDefinitions } from '../definitions/functions';
 import { getFunctionSignatures, getCommandSignature } from '../definitions/helpers';
@@ -35,13 +35,13 @@ function getSafeInsertText(text: string, options: { dashSupported?: boolean } =
     : text;
 }
 
-export function getAutocompleteFunctionDefinition(fn: FunctionDefinition) {
+export function getSuggestionFunctionDefinition(fn: FunctionDefinition): SuggestionRawDefinition {
   const fullSignatures = getFunctionSignatures(fn);
   return {
     label: fullSignatures[0].declaration,
-    insertText: `${fn.name}($0)`,
-    insertTextRules: 4, // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
-    kind: 1,
+    text: `${fn.name}($0)`,
+    asSnippet: true,
+    kind: 'Function',
     detail: fn.description,
     documentation: {
       value: buildFunctionDocumentation(fullSignatures),
@@ -53,15 +53,13 @@ export function getAutocompleteFunctionDefinition(fn: FunctionDefinition) {
   };
 }
 
-export function getAutocompleteBuiltinDefinition(
-  fn: FunctionDefinition
-): AutocompleteCommandDefinition {
+export function getSuggestionBuiltinDefinition(fn: FunctionDefinition): SuggestionRawDefinition {
   const hasArgs = fn.signatures.some(({ params }) => params.length > 1);
   return {
     label: fn.name,
-    insertText: hasArgs ? `${fn.name} $0` : fn.name,
-    ...(hasArgs ? { insertTextRules: 4 } : {}), // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
-    kind: 11,
+    text: hasArgs ? `${fn.name} $0` : fn.name,
+    ...(hasArgs ? { insertTextRules: 4 } : {}), // kbn-esql-validation-autocomplete.languages.CompletionItemInsertTextRule.InsertAsSnippet,
+    kind: 'Operator',
     detail: fn.description,
     documentation: {
       value: '',
@@ -76,14 +74,14 @@ export const getCompatibleFunctionDefinition = (
   option: string | undefined,
   returnTypes?: string[],
   ignored: string[] = []
-): AutocompleteCommandDefinition[] => {
+): SuggestionRawDefinition[] => {
   const fnSupportedByCommand = allFunctions.filter(
     ({ name, supportedCommands, supportedOptions }) =>
       (option ? supportedOptions?.includes(option) : supportedCommands.includes(command)) &&
       !ignored.includes(name)
   );
   if (!returnTypes) {
-    return fnSupportedByCommand.map(getAutocompleteFunctionDefinition);
+    return fnSupportedByCommand.map(getSuggestionFunctionDefinition);
   }
   return fnSupportedByCommand
     .filter((mathDefinition) =>
@@ -91,21 +89,21 @@ export const getCompatibleFunctionDefinition = (
         (signature) => returnTypes[0] === 'any' || returnTypes.includes(signature.returnType)
       )
     )
-    .map(getAutocompleteFunctionDefinition);
+    .map(getSuggestionFunctionDefinition);
 };
 
-export function getAutocompleteCommandDefinition(
+export function getSuggestionCommandDefinition(
   command: CommandDefinition
-): AutocompleteCommandDefinition {
+): SuggestionRawDefinition {
   const commandDefinition = getCommandDefinition(command.name);
   const commandSignature = getCommandSignature(commandDefinition);
   return {
     label: commandDefinition.name,
-    insertText: commandDefinition.signature.params.length
+    text: commandDefinition.signature.params.length
       ? `${commandDefinition.name} $0`
       : commandDefinition.name,
-    insertTextRules: 4, // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
-    kind: 0,
+    asSnippet: true,
+    kind: 'Method',
     detail: commandDefinition.description,
     documentation: {
       value: buildDocumentation(commandSignature.declaration, commandSignature.examples),
@@ -115,34 +113,37 @@ export function getAutocompleteCommandDefinition(
   };
 }
 
-export const buildFieldsDefinitions = (fields: string[]): AutocompleteCommandDefinition[] =>
+export const buildFieldsDefinitions = (fields: string[]): SuggestionRawDefinition[] =>
   fields.map((label) => ({
     label,
-    insertText: getSafeInsertText(label),
-    kind: 4,
-    detail: i18n.translate('monaco.esql.autocomplete.fieldDefinition', {
+    text: getSafeInsertText(label),
+    kind: 'Variable',
+    detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.fieldDefinition', {
       defaultMessage: `Field specified by the input table`,
     }),
     sortText: 'D',
   }));
 
-export const buildVariablesDefinitions = (variables: string[]): AutocompleteCommandDefinition[] =>
+export const buildVariablesDefinitions = (variables: string[]): SuggestionRawDefinition[] =>
   variables.map((label) => ({
     label,
-    insertText: label,
-    kind: 4,
-    detail: i18n.translate('monaco.esql.autocomplete.variableDefinition', {
-      defaultMessage: `Variable specified by the user within the ES|QL query`,
-    }),
+    text: label,
+    kind: 'Variable',
+    detail: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.autocomplete.variableDefinition',
+      {
+        defaultMessage: `Variable specified by the user within the ES|QL query`,
+      }
+    ),
     sortText: 'D',
   }));
 
-export const buildSourcesDefinitions = (sources: string[]): AutocompleteCommandDefinition[] =>
+export const buildSourcesDefinitions = (sources: string[]): SuggestionRawDefinition[] =>
   sources.map((label) => ({
     label,
-    insertText: getSafeInsertText(label, { dashSupported: true }),
-    kind: 21,
-    detail: i18n.translate('monaco.esql.autocomplete.sourceDefinition', {
+    text: getSafeInsertText(label, { dashSupported: true }),
+    kind: 'Reference',
+    detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.sourceDefinition', {
       defaultMessage: `Index`,
     }),
     sortText: 'A',
@@ -151,25 +152,25 @@ export const buildSourcesDefinitions = (sources: string[]): AutocompleteCommandD
 export const buildConstantsDefinitions = (
   userConstants: string[],
   detail?: string
-): AutocompleteCommandDefinition[] =>
+): SuggestionRawDefinition[] =>
   userConstants.map((label) => ({
     label,
-    insertText: label,
-    kind: 14,
+    text: label,
+    kind: 'Constant',
     detail:
       detail ??
-      i18n.translate('monaco.esql.autocomplete.constantDefinition', {
+      i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.constantDefinition', {
         defaultMessage: `Constant`,
       }),
     sortText: 'A',
   }));
 
-export const buildNewVarDefinition = (label: string): AutocompleteCommandDefinition => {
+export const buildNewVarDefinition = (label: string): SuggestionRawDefinition => {
   return {
     label,
-    insertText: `${label} =`,
-    kind: 21,
-    detail: i18n.translate('monaco.esql.autocomplete.newVarDoc', {
+    text: `${label} =`,
+    kind: 'Variable',
+    detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.newVarDoc', {
       defaultMessage: 'Define a new variable',
     }),
     sortText: '1',
@@ -178,12 +179,12 @@ export const buildNewVarDefinition = (label: string): AutocompleteCommandDefinit
 
 export const buildPoliciesDefinitions = (
   policies: Array<{ name: string; sourceIndices: string[] }>
-): AutocompleteCommandDefinition[] =>
+): SuggestionRawDefinition[] =>
   policies.map(({ name: label, sourceIndices }) => ({
     label,
-    insertText: getSafeInsertText(label, { dashSupported: true }),
-    kind: 5,
-    detail: i18n.translate('monaco.esql.autocomplete.policyDefinition', {
+    text: getSafeInsertText(label, { dashSupported: true }),
+    kind: 'Class',
+    detail: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.policyDefinition', {
       defaultMessage: `Policy defined on {count, plural, one {index} other {indices}}: {indices}`,
       values: {
         count: sourceIndices.length,
@@ -196,17 +197,20 @@ export const buildPoliciesDefinitions = (
 export const buildMatchingFieldsDefinition = (
   matchingField: string,
   fields: string[]
-): AutocompleteCommandDefinition[] =>
+): SuggestionRawDefinition[] =>
   fields.map((label) => ({
     label,
-    insertText: label,
-    kind: 4,
-    detail: i18n.translate('monaco.esql.autocomplete.matchingFieldDefinition', {
-      defaultMessage: `Use to match on {matchingField} on the policy`,
-      values: {
-        matchingField,
-      },
-    }),
+    text: label,
+    kind: 'Variable',
+    detail: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.autocomplete.matchingFieldDefinition',
+      {
+        defaultMessage: `Use to match on {matchingField} on the policy`,
+        values: {
+          matchingField,
+        },
+      }
+    ),
     sortText: 'D',
   }));
 
@@ -214,16 +218,16 @@ export const buildOptionDefinition = (
   option: CommandOptionsDefinition,
   isAssignType: boolean = false
 ) => {
-  const completeItem: AutocompleteCommandDefinition = {
+  const completeItem: SuggestionRawDefinition = {
     label: option.name,
-    insertText: option.name,
-    kind: 21,
+    text: option.name,
+    kind: 'Reference',
     detail: option.description,
     sortText: '1',
   };
   if (isAssignType || option.signature.params.length) {
-    completeItem.insertText = isAssignType ? `${option.name} = $0` : `${option.name} $0`;
-    completeItem.insertTextRules = 4; // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet;
+    completeItem.text = isAssignType ? `${option.name} = $0` : `${option.name} $0`;
+    completeItem.asSnippet = true;
     completeItem.command = TRIGGER_SUGGESTION_COMMAND;
   }
   return completeItem;
@@ -231,32 +235,35 @@ export const buildOptionDefinition = (
 
 export const buildSettingDefinitions = (
   setting: CommandModeDefinition
-): AutocompleteCommandDefinition[] => {
+): SuggestionRawDefinition[] => {
   // for now there's just a single setting with one argument
   return setting.values.map(({ name, description }) => ({
     label: `${setting.prefix || ''}${name}`,
-    insertText: `${setting.prefix || ''}${name}:$0`,
-    insertTextRules: 4, // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
-    kind: 21,
+    text: `${setting.prefix || ''}${name}:$0`,
+    asSnippet: true,
+    kind: 'Reference',
     detail: description ? `${setting.description} - ${description}` : setting.description,
     sortText: 'D',
     command: TRIGGER_SUGGESTION_COMMAND,
   }));
 };
 
-export const buildNoPoliciesAvailableDefinition = (): AutocompleteCommandDefinition => ({
-  label: i18n.translate('monaco.esql.autocomplete.noPoliciesLabel', {
+export const buildNoPoliciesAvailableDefinition = (): SuggestionRawDefinition => ({
+  label: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.noPoliciesLabel', {
     defaultMessage: 'No available policy',
   }),
-  insertText: '',
-  kind: 26,
-  detail: i18n.translate('monaco.esql.autocomplete.noPoliciesLabelsFound', {
-    defaultMessage: 'Click to create',
-  }),
+  text: '',
+  kind: 'Issue',
+  detail: i18n.translate(
+    'kbn-esql-validation-autocomplete.esql.autocomplete.noPoliciesLabelsFound',
+    {
+      defaultMessage: 'Click to create',
+    }
+  ),
   sortText: 'D',
   command: {
     id: 'esql.policies.create',
-    title: i18n.translate('monaco.esql.autocomplete.createNewPolicy', {
+    title: i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.createNewPolicy', {
       defaultMessage: 'Click to create',
     }),
   },
@@ -271,7 +278,7 @@ function getUnitDuration(unit: number = 1) {
 }
 
 export function getCompatibleLiterals(commandName: string, types: string[], names?: string[]) {
-  const suggestions: AutocompleteCommandDefinition[] = [];
+  const suggestions: SuggestionRawDefinition[] = [];
   if (types.includes('number') && commandName === 'limit') {
     // suggest 10/50/100
     suggestions.push(...buildConstantsDefinitions(['10', '100', '1000'], ''));
@@ -294,7 +301,7 @@ export function getCompatibleLiterals(commandName: string, types: string[], name
         suggestions.push(
           ...buildConstantsDefinitions(
             [commandName === 'grok' ? '"%{WORD:firstWord}"' : '"%{firstWord}"'],
-            i18n.translate('monaco.esql.autocomplete.aPatternString', {
+            i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.aPatternString', {
               defaultMessage: 'A pattern string',
             })
           )

packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts

@@ -6,8 +6,8 @@
  * Side Public License, v 1.
  */
 
+import type { ESQLAstItem, ESQLCommand, ESQLFunction } from '@kbn/esql-ast';
 import { getFunctionDefinition, isAssignment, isFunctionItem } from '../shared/helpers';
-import type { ESQLAstItem, ESQLCommand, ESQLFunction } from '../types';
 
 function extractFunctionArgs(args: ESQLAstItem[]): ESQLFunction[] {
   return args.flatMap((arg) => (isAssignment(arg) ? arg.args[1] : arg)).filter(isFunctionItem);

packages/kbn-esql-validation-autocomplete/src/autocomplete/types.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// This is a subset of the Monaco's editor CompletitionItemKind type
+export type ItemKind =
+  | 'Method'
+  | 'Function'
+  | 'Field'
+  | 'Variable'
+  | 'Class'
+  | 'Operator'
+  | 'Value'
+  | 'Constant'
+  | 'Keyword'
+  | 'Text'
+  | 'Reference'
+  | 'Snippet'
+  | 'Issue';
+
+export interface SuggestionRawDefinition {
+  /* The label to show on the suggestion UI for the entry */
+  label: string;
+  /* The actual text to insert into the editor */
+  text: string;
+  /**
+   * Should the text be inserted as a snippet?
+   * That is usually used for special behaviour like moving the cursor in a specific position
+   * after inserting the text.
+   * i.e. 'fnName( $0 )' will insert fnName( ) and move the cursor where $0 is.
+   * */
+  asSnippet?: boolean;
+  /**
+   * This is useful to identify the suggestion type and apply different styles to it.
+   */
+  kind: ItemKind;
+  /**
+   * A very short description for the suggestion entry that can be shown on the UI next to the label
+   */
+  detail: string;
+  /**
+   * A longer description for the suggestion entry that can be shown on demand on the UI.
+   */
+  documentation?: { value: string };
+  /**
+   * A string to use for sorting the suggestion within the suggestions list
+   */
+  sortText?: string;
+  /**
+   * Suggestions can trigger a command by id. This is useful to trigger specific actions in some contexts
+   */
+  command?: {
+    title: string;
+    id: string;
+  };
+}
+
+export interface EditorContext {
+  /** The actual char that triggered the suggestion (1 single char) */
+  triggerCharacter?: string;
+  /** The type of trigger id. triggerKind = 0 is a programmatic trigger, while any other non-zero value is currently ignored. */
+  triggerKind: number;
+}

packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts

@@ -6,17 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { EditorError } from '../../../../types';
-import { CharStreams } from 'antlr4';
 import { getActions } from './actions';
-import { getParser, ROOT_STATEMENT } from '../../antlr_facade';
-import { ESQLErrorListener } from '../../monaco/esql_error_listener';
-import { AstListener } from '../ast_factory';
-import { wrapAsMonacoMessage } from '../shared/monaco_utils';
-import { ESQLAst } from '../types';
-import { validateAst } from '../validation/validation';
-import { monaco } from '../../../../monaco_imports';
+import { validateQuery } from '../validation/validation';
 import { getAllFunctions } from '../shared/helpers';
+import { wrapAsEditorMessage } from './testing_utils';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
 
 function getCallbackMocks() {
   return {
@@ -67,38 +61,6 @@ function getCallbackMocks() {
   };
 }
 
-const getAstAndErrors = async (
-  text: string | undefined
-): Promise<{
-  errors: EditorError[];
-  ast: ESQLAst;
-}> => {
-  if (text == null) {
-    return { ast: [], errors: [] };
-  }
-  const errorListener = new ESQLErrorListener();
-  const parseListener = new AstListener();
-  const parser = getParser(CharStreams.fromString(text), errorListener, parseListener);
-
-  parser[ROOT_STATEMENT]();
-
-  return { ...parseListener.getAst(), errors: errorListener.getErrors() };
-};
-
-function createModelAndRange(text: string) {
-  return {
-    model: { getValue: () => text } as monaco.editor.ITextModel,
-    range: {} as monaco.Range,
-  };
-}
-
-function createMonacoContext(errors: EditorError[]): monaco.languages.CodeActionContext {
-  return {
-    markers: errors,
-    trigger: 1,
-  };
-}
-
 /**
  * There are different wats to test the code here: one is a direct unit test of the feature, another is
  * an integration test passing from the query statement validation. The latter is more realistic, but
@@ -111,17 +73,18 @@ function testQuickFixesFn(
   { only, skip }: { only?: boolean; skip?: boolean } = {}
 ) {
   const testFn = only ? it.only : skip ? it.skip : it;
-  const { model, range } = createModelAndRange(statement);
   testFn(`${statement} => ["${expectedFixes.join('","')}"]`, async () => {
     const callbackMocks = getCallbackMocks();
-    const { errors } = await validateAst(statement, getAstAndErrors, callbackMocks);
-
-    const monacoErrors = wrapAsMonacoMessage('error', statement, errors);
-    const context = createMonacoContext(monacoErrors);
-    const actions = await getActions(model, range, context, getAstAndErrors, callbackMocks);
-    const edits = actions.map(
-      ({ edit }) => (edit?.edits[0] as monaco.languages.IWorkspaceTextEdit).textEdit.text
+    const { errors } = await validateQuery(
+      statement,
+      getAstAndSyntaxErrors,
+      undefined,
+      callbackMocks
     );
+
+    const monacoErrors = wrapAsEditorMessage('error', errors);
+    const actions = await getActions(statement, monacoErrors, getAstAndSyntaxErrors, callbackMocks);
+    const edits = actions.map(({ edits: actionEdits }) => actionEdits[0].text);
     expect(edits).toEqual(
       !options || !options.equalityCheck || options.equalityCheck === 'equal'
         ? expectedFixes
@@ -300,12 +263,15 @@ describe('quick fixes logic', () => {
     it('should not crash if callback functions are not passed', async () => {
       const callbackMocks = getCallbackMocks();
       const statement = `from a | eval b  = a | enrich policy | dissect stringField "%{firstWord}"`;
-      const { model, range } = createModelAndRange(statement);
-      const { errors } = await validateAst(statement, getAstAndErrors, callbackMocks);
-      const monacoErrors = wrapAsMonacoMessage('error', statement, errors);
-      const context = createMonacoContext(monacoErrors);
+      const { errors } = await validateQuery(
+        statement,
+        getAstAndSyntaxErrors,
+        undefined,
+        callbackMocks
+      );
+      const monacoErrors = wrapAsEditorMessage('error', errors);
       try {
-        await getActions(model, range, context, getAstAndErrors, {
+        await getActions(statement, monacoErrors, getAstAndSyntaxErrors, {
           getFieldsFor: undefined,
           getSources: undefined,
           getPolicies: undefined,
@@ -319,12 +285,15 @@ describe('quick fixes logic', () => {
     it('should not crash no callbacks are passed', async () => {
       const callbackMocks = getCallbackMocks();
       const statement = `from a | eval b  = a | enrich policy | dissect stringField "%{firstWord}"`;
-      const { model, range } = createModelAndRange(statement);
-      const { errors } = await validateAst(statement, getAstAndErrors, callbackMocks);
-      const monacoErrors = wrapAsMonacoMessage('error', statement, errors);
-      const context = createMonacoContext(monacoErrors);
+      const { errors } = await validateQuery(
+        statement,
+        getAstAndSyntaxErrors,
+        undefined,
+        callbackMocks
+      );
+      const monacoErrors = wrapAsEditorMessage('error', errors);
       try {
-        await getActions(model, range, context, getAstAndErrors, undefined);
+        await getActions(statement, monacoErrors, getAstAndSyntaxErrors, undefined);
       } catch {
         fail('Should not throw');
       }

packages/kbn-esql-validation-autocomplete/src/code_actions/actions.ts

@@ -7,7 +7,7 @@
  */
 import { i18n } from '@kbn/i18n';
 import levenshtein from 'js-levenshtein';
-import type { monaco } from '../../../../monaco_imports';
+import type { AstProviderFn, ESQLAst, ESQLCommand, EditorError } from '@kbn/esql-ast';
 import {
   getFieldsByTypeHelper,
   getPolicyHelper,
@@ -20,23 +20,9 @@ import {
   shouldBeQuotedText,
 } from '../shared/helpers';
 import { ESQLCallbacks } from '../shared/types';
-import { AstProviderFn, ESQLAst, ESQLCommand } from '../types';
 import { buildQueryForFieldsFromSource } from '../validation/helpers';
 import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants';
-
-type GetSourceFn = () => Promise<string[]>;
-type GetFieldsByTypeFn = (type: string | string[], ignored?: string[]) => Promise<string[]>;
-type GetPoliciesFn = () => Promise<string[]>;
-type GetPolicyFieldsFn = (name: string) => Promise<string[]>;
-type GetMetaFieldsFn = () => Promise<string[]>;
-
-interface Callbacks {
-  getSources: GetSourceFn;
-  getFieldsByType: GetFieldsByTypeFn;
-  getPolicies: GetPoliciesFn;
-  getPolicyFields: GetPolicyFieldsFn;
-  getMetaFields: GetMetaFieldsFn;
-}
+import type { CodeAction, Callbacks } from './types';
 
 function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) {
   const helpers = getFieldsByTypeHelper(queryString, resourceRetriever);
@@ -96,29 +82,17 @@ export const getCompatibleFunctionDefinitions = (
   return fnSupportedByCommand.map(({ name }) => name);
 };
 
-function createAction(
-  title: string,
-  solution: string,
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri
-) {
+function createAction(title: string, solution: string, error: EditorError): CodeAction {
   return {
     title,
     diagnostics: [error],
     kind: 'quickfix',
-    edit: {
-      edits: [
-        {
-          resource: uri,
-          textEdit: {
-            range: error,
-            text: solution,
-          },
-          versionId: undefined,
-        },
-      ],
-    },
-    isPreferred: true,
+    edits: [
+      {
+        range: error,
+        text: solution,
+      },
+    ],
   };
 }
 
@@ -136,8 +110,7 @@ async function getSpellingPossibilities(fn: () => Promise<string[]>, errorText:
 }
 
 async function getSpellingActionForColumns(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   { getFieldsByType, getPolicies, getPolicyFields }: Callbacks
@@ -156,16 +129,15 @@ async function getSpellingActionForColumns(
     }
     return availableFields;
   }, errorText);
-  return wrapIntoSpellingChangeAction(error, uri, possibleFields);
+  return wrapIntoSpellingChangeAction(error, possibleFields);
 }
 
 async function getQuotableActionForColumns(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   { getFieldsByType }: Callbacks
-) {
+): Promise<CodeAction[]> {
   const commandEndIndex = ast.find((command) => command.location.max > error.endColumn)?.location
     .max;
   // the error received is unknwonColumn here, but look around the column to see if there's more
@@ -189,22 +161,21 @@ async function getQuotableActionForColumns(
   const errorText = queryString
     .substring(error.startColumn - 1, error.endColumn + possibleUnquotedText.length)
     .trimEnd();
-  const actions = [];
+  const actions: CodeAction[] = [];
   if (shouldBeQuotedText(errorText)) {
     const availableFields = new Set(await getFieldsByType('any'));
     const solution = `\`${errorText.replace(SINGLE_TICK_REGEX, DOUBLE_BACKTICK)}\``;
     if (availableFields.has(errorText) || availableFields.has(solution)) {
       actions.push(
         createAction(
-          i18n.translate('monaco.esql.quickfix.replaceWithSolution', {
+          i18n.translate('kbn-esql-validation-autocomplete.esql.quickfix.replaceWithSolution', {
             defaultMessage: 'Did you mean {solution} ?',
             values: {
               solution,
             },
           }),
           solution,
-          { ...error, endColumn: error.startColumn + errorText.length }, // override the location
-          uri
+          { ...error, endColumn: error.startColumn + errorText.length } // override the location
         )
       );
     }
@@ -213,8 +184,7 @@ async function getQuotableActionForColumns(
 }
 
 async function getSpellingActionForIndex(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   { getSources }: Callbacks
@@ -230,24 +200,22 @@ async function getSpellingActionForIndex(
     }
     return sources;
   }, errorText);
-  return wrapIntoSpellingChangeAction(error, uri, possibleSources);
+  return wrapIntoSpellingChangeAction(error, possibleSources);
 }
 
 async function getSpellingActionForPolicies(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   { getPolicies }: Callbacks
 ) {
   const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1);
   const possiblePolicies = await getSpellingPossibilities(getPolicies, errorText);
-  return wrapIntoSpellingChangeAction(error, uri, possiblePolicies);
+  return wrapIntoSpellingChangeAction(error, possiblePolicies);
 }
 
 async function getSpellingActionForFunctions(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst
 ) {
@@ -268,26 +236,23 @@ async function getSpellingActionForFunctions(
   );
   return wrapIntoSpellingChangeAction(
     error,
-    uri,
     possibleSolutions.map((fn) => `${fn}${errorText.substring(errorText.lastIndexOf('('))}`)
   );
 }
 
 async function getSpellingActionForMetadata(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   { getMetaFields }: Callbacks
 ) {
   const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1);
   const possibleMetafields = await getSpellingPossibilities(getMetaFields, errorText);
-  return wrapIntoSpellingChangeAction(error, uri, possibleMetafields);
+  return wrapIntoSpellingChangeAction(error, possibleMetafields);
 }
 
 async function getSpellingActionForEnrichMode(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   queryString: string,
   ast: ESQLAst,
   _callbacks: Callbacks
@@ -308,38 +273,33 @@ async function getSpellingActionForEnrichMode(
   if (!possibleEnrichModes.length) {
     possibleEnrichModes.push(...allModes);
   }
-  return wrapIntoSpellingChangeAction(error, uri, possibleEnrichModes);
+  return wrapIntoSpellingChangeAction(error, possibleEnrichModes);
 }
 
 function wrapIntoSpellingChangeAction(
-  error: monaco.editor.IMarkerData,
-  uri: monaco.Uri,
+  error: EditorError,
   possibleSolution: string[]
-): monaco.languages.CodeAction[] {
+): CodeAction[] {
   return possibleSolution.map((solution) =>
     createAction(
       // @TODO: workout why the tooltip is truncating the title here
-      i18n.translate('monaco.esql.quickfix.replaceWithSolution', {
+      i18n.translate('kbn-esql-validation-autocomplete.esql.quickfix.replaceWithSolution', {
         defaultMessage: 'Did you mean {solution} ?',
         values: {
           solution,
         },
       }),
       solution,
-      error,
-      uri
+      error
     )
   );
 }
 
-function extractQuotedText(rawText: string, error: monaco.editor.IMarkerData) {
+function extractQuotedText(rawText: string, error: EditorError) {
   return rawText.substring(error.startColumn - 2, error.endColumn);
 }
 
-function inferCodeFromError(
-  error: monaco.editor.IMarkerData & { owner?: string },
-  rawText: string
-) {
+function inferCodeFromError(error: EditorError & { owner?: string }, rawText: string) {
   if (error.message.endsWith('expecting STRING')) {
     const value = extractQuotedText(rawText, error);
     return /^'(.)*'$/.test(value) ? 'wrongQuotes' : undefined;
@@ -347,17 +307,15 @@ function inferCodeFromError(
 }
 
 export async function getActions(
-  model: monaco.editor.ITextModel,
-  range: monaco.Range,
-  context: monaco.languages.CodeActionContext,
+  innerText: string,
+  markers: EditorError[],
   astProvider: AstProviderFn,
   resourceRetriever?: ESQLCallbacks
-): Promise<monaco.languages.CodeAction[]> {
-  const actions: monaco.languages.CodeAction[] = [];
-  if (context.markers.length === 0) {
+): Promise<CodeAction[]> {
+  const actions: CodeAction[] = [];
+  if (markers.length === 0) {
     return actions;
   }
-  const innerText = model.getValue();
   const { ast } = await astProvider(innerText);
 
   const queryForFields = buildQueryForFieldsFromSource(innerText, ast);
@@ -377,30 +335,23 @@ export async function getActions(
   // Markers are sent only on hover and are limited to the hovered area
   // so unless there are multiple error/markers for the same area, there's just one
   // in some cases, like syntax + semantic errors (i.e. unquoted fields eval field-1 ), there might be more than one
-  for (const error of context.markers) {
+  for (const error of markers) {
     const code = error.code ?? inferCodeFromError(error, innerText);
     switch (code) {
       case 'unknownColumn':
         const [columnsSpellChanges, columnsQuotedChanges] = await Promise.all([
-          getSpellingActionForColumns(error, model.uri, innerText, ast, callbacks),
-          getQuotableActionForColumns(error, model.uri, innerText, ast, callbacks),
+          getSpellingActionForColumns(error, innerText, ast, callbacks),
+          getQuotableActionForColumns(error, innerText, ast, callbacks),
         ]);
         actions.push(...(columnsQuotedChanges.length ? columnsQuotedChanges : columnsSpellChanges));
         break;
       case 'unknownIndex':
-        const indexSpellChanges = await getSpellingActionForIndex(
-          error,
-          model.uri,
-          innerText,
-          ast,
-          callbacks
-        );
+        const indexSpellChanges = await getSpellingActionForIndex(error, innerText, ast, callbacks);
         actions.push(...indexSpellChanges);
         break;
       case 'unknownPolicy':
         const policySpellChanges = await getSpellingActionForPolicies(
           error,
-          model.uri,
           innerText,
           ast,
           callbacks
@@ -408,18 +359,12 @@ export async function getActions(
         actions.push(...policySpellChanges);
         break;
       case 'unknownFunction':
-        const fnsSpellChanges = await getSpellingActionForFunctions(
-          error,
-          model.uri,
-          innerText,
-          ast
-        );
+        const fnsSpellChanges = await getSpellingActionForFunctions(error, innerText, ast);
         actions.push(...fnsSpellChanges);
         break;
       case 'unknownMetadataField':
         const metadataSpellChanges = await getSpellingActionForMetadata(
           error,
-          model.uri,
           innerText,
           ast,
           callbacks
@@ -431,7 +376,7 @@ export async function getActions(
         const errorText = extractQuotedText(innerText, error);
         actions.push(
           createAction(
-            i18n.translate('monaco.esql.quickfix.replaceWithQuote', {
+            i18n.translate('kbn-esql-validation-autocomplete.esql.quickfix.replaceWithQuote', {
               defaultMessage: 'Change quote to " (double)',
             }),
             errorText.replaceAll("'", '"'),
@@ -440,15 +385,13 @@ export async function getActions(
               ...error,
               startColumn: error.startColumn - 1,
               endColumn: error.startColumn + errorText.length,
-            },
-            model.uri
+            }
           )
         );
         break;
       case 'unsupportedSettingCommandValue':
         const enrichModeSpellChanges = await getSpellingActionForEnrichMode(
           error,
-          model.uri,
           innerText,
           ast,
           callbacks

packages/kbn-esql-validation-autocomplete/src/code_actions/testing_utils.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { EditorError, ESQLMessage } from '@kbn/esql-ast';
+
+export function wrapAsEditorMessage(
+  type: 'error' | 'warning',
+  messages: Array<ESQLMessage | EditorError>
+): EditorError[] {
+  return messages.map((e) => {
+    if ('severity' in e) {
+      return e;
+    }
+    const startPosition = e.location ? e.location.min + 1 : 0;
+    const endPosition = e.location ? e.location.max + 1 : 0;
+    return {
+      message: e.text,
+      startColumn: startPosition,
+      startLineNumber: 1,
+      endColumn: endPosition + 1,
+      endLineNumber: 1,
+      severity: type,
+      _source: 'client' as const,
+      code: e.code,
+    };
+  });
+}

packages/kbn-esql-validation-autocomplete/src/code_actions/types.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { EditorError } from '../types';
+
+type GetSourceFn = () => Promise<string[]>;
+type GetFieldsByTypeFn = (type: string | string[], ignored?: string[]) => Promise<string[]>;
+type GetPoliciesFn = () => Promise<string[]>;
+type GetPolicyFieldsFn = (name: string) => Promise<string[]>;
+type GetMetaFieldsFn = () => Promise<string[]>;
+
+export interface Callbacks {
+  getSources: GetSourceFn;
+  getFieldsByType: GetFieldsByTypeFn;
+  getPolicies: GetPoliciesFn;
+  getPolicyFields: GetPolicyFieldsFn;
+  getMetaFields: GetMetaFieldsFn;
+}
+
+export interface CodeAction {
+  title: string;
+  diagnostics: EditorError[];
+  kind: 'quickfix';
+  edits: Array<{
+    range: EditorError;
+    text: string;
+  }>;
+}

packages/kbn-esql-validation-autocomplete/src/definitions/aggs.ts

@@ -7,7 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { FunctionDefinition } from './types';
+import type { FunctionDefinition } from './types';
 
 function createNumericAggDefinition({
   name,
@@ -48,46 +48,52 @@ function createNumericAggDefinition({
 export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
   {
     name: 'avg',
-    description: i18n.translate('monaco.esql.definitions.avgDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.avgDoc', {
       defaultMessage: 'Returns the average of the values in a field',
     }),
   },
   {
     name: 'max',
-    description: i18n.translate('monaco.esql.definitions.maxDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.maxDoc', {
       defaultMessage: 'Returns the maximum value in a field.',
     }),
   },
   {
     name: 'min',
-    description: i18n.translate('monaco.esql.definitions.minDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.minDoc', {
       defaultMessage: 'Returns the minimum value in a field.',
     }),
   },
   {
     name: 'sum',
-    description: i18n.translate('monaco.esql.definitions.sumDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.sumDoc', {
       defaultMessage: 'Returns the sum of the values in a field.',
     }),
   },
   {
     name: 'median',
-    description: i18n.translate('monaco.esql.definitions.medianDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.medianDoc', {
       defaultMessage: 'Returns the 50% percentile.',
     }),
   },
   {
     name: 'median_absolute_deviation',
-    description: i18n.translate('monaco.esql.definitions.medianDeviationDoc', {
-      defaultMessage:
-        'Returns the median of each data point’s deviation from the median of the entire sample.',
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.medianDeviationDoc',
+      {
+        defaultMessage:
+          'Returns the median of each data point’s deviation from the median of the entire sample.',
+      }
+    ),
   },
   {
     name: 'percentile',
-    description: i18n.translate('monaco.esql.definitions.percentiletDoc', {
-      defaultMessage: 'Returns the n percentile of a field.',
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.percentiletDoc',
+      {
+        defaultMessage: 'Returns the n percentile of a field.',
+      }
+    ),
     args: [{ name: 'percentile', type: 'number', value: '90', literalOnly: true }],
   },
 ]
@@ -96,7 +102,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
     {
       name: 'count',
       type: 'agg',
-      description: i18n.translate('monaco.esql.definitions.countDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.countDoc', {
         defaultMessage: 'Returns the count of the values in a field.',
       }),
       supportedCommands: ['stats'],
@@ -119,9 +125,12 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
     {
       name: 'count_distinct',
       type: 'agg',
-      description: i18n.translate('monaco.esql.definitions.countDistinctDoc', {
-        defaultMessage: 'Returns the count of distinct values in a field.',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definitions.countDistinctDoc',
+        {
+          defaultMessage: 'Returns the count of distinct values in a field.',
+        }
+      ),
       supportedCommands: ['stats'],
       signatures: [
         {
@@ -140,9 +149,12 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
     {
       name: 'st_centroid',
       type: 'agg',
-      description: i18n.translate('monaco.esql.definitions.stCentroidDoc', {
-        defaultMessage: 'Returns the count of distinct values in a field.',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definitions.stCentroidDoc',
+        {
+          defaultMessage: 'Returns the count of distinct values in a field.',
+        }
+      ),
       supportedCommands: ['stats'],
       signatures: [
         {

packages/kbn-esql-validation-autocomplete/src/definitions/builtin.ts

@@ -7,7 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { FunctionDefinition } from './types';
+import type { FunctionDefinition } from './types';
 
 function createMathDefinition(
   name: string,
@@ -90,28 +90,28 @@ export const builtinFunctions: FunctionDefinition[] = [
   createMathDefinition(
     '+',
     ['number', 'date', ['date', 'time_literal'], ['time_literal', 'date']],
-    i18n.translate('monaco.esql.definition.addDoc', {
+    i18n.translate('kbn-esql-validation-autocomplete.esql.definition.addDoc', {
       defaultMessage: 'Add (+)',
     })
   ),
   createMathDefinition(
     '-',
     ['number', 'date', ['date', 'time_literal'], ['time_literal', 'date']],
-    i18n.translate('monaco.esql.definition.subtractDoc', {
+    i18n.translate('kbn-esql-validation-autocomplete.esql.definition.subtractDoc', {
       defaultMessage: 'Subtract (-)',
     })
   ),
   createMathDefinition(
     '*',
     ['number'],
-    i18n.translate('monaco.esql.definition.multiplyDoc', {
+    i18n.translate('kbn-esql-validation-autocomplete.esql.definition.multiplyDoc', {
       defaultMessage: 'Multiply (*)',
     })
   ),
   createMathDefinition(
     '/',
     ['number'],
-    i18n.translate('monaco.esql.definition.divideDoc', {
+    i18n.translate('kbn-esql-validation-autocomplete.esql.definition.divideDoc', {
       defaultMessage: 'Divide (/)',
     }),
     (fnDef) => {
@@ -123,13 +123,16 @@ export const builtinFunctions: FunctionDefinition[] = [
             messages.push({
               type: 'warning' as const,
               code: 'divideByZero',
-              text: i18n.translate('monaco.esql.divide.warning.divideByZero', {
-                defaultMessage: 'Cannot divide by zero: {left}/{right}',
-                values: {
-                  left: left.text,
-                  right: right.value,
-                },
-              }),
+              text: i18n.translate(
+                'kbn-esql-validation-autocomplete.esql.divide.warning.divideByZero',
+                {
+                  defaultMessage: 'Cannot divide by zero: {left}/{right}',
+                  values: {
+                    left: left.text,
+                    right: right.value,
+                  },
+                }
+              ),
               location: fnDef.location,
             });
           }
@@ -141,7 +144,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   createMathDefinition(
     '%',
     ['number'],
-    i18n.translate('monaco.esql.definition.moduleDoc', {
+    i18n.translate('kbn-esql-validation-autocomplete.esql.definition.moduleDoc', {
       defaultMessage: 'Module (%)',
     }),
     (fnDef) => {
@@ -153,13 +156,16 @@ export const builtinFunctions: FunctionDefinition[] = [
             messages.push({
               type: 'warning' as const,
               code: 'moduleByZero',
-              text: i18n.translate('monaco.esql.divide.warning.zeroModule', {
-                defaultMessage: 'Module by zero can return null value: {left}%{right}',
-                values: {
-                  left: left.text,
-                  right: right.value,
-                },
-              }),
+              text: i18n.translate(
+                'kbn-esql-validation-autocomplete.esql.divide.warning.zeroModule',
+                {
+                  defaultMessage: 'Module by zero can return null value: {left}%{right}',
+                  values: {
+                    left: left.text,
+                    right: right.value,
+                  },
+                }
+              ),
               location: fnDef.location,
             });
           }
@@ -171,39 +177,51 @@ export const builtinFunctions: FunctionDefinition[] = [
   ...[
     {
       name: '==',
-      description: i18n.translate('monaco.esql.definition.equalToDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.equalToDoc', {
         defaultMessage: 'Equal to',
       }),
     },
     {
       name: '!=',
-      description: i18n.translate('monaco.esql.definition.notEqualToDoc', {
-        defaultMessage: 'Not equal to',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definition.notEqualToDoc',
+        {
+          defaultMessage: 'Not equal to',
+        }
+      ),
     },
     {
       name: '<',
-      description: i18n.translate('monaco.esql.definition.lessThanDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.lessThanDoc', {
         defaultMessage: 'Less than',
       }),
     },
     {
       name: '>',
-      description: i18n.translate('monaco.esql.definition.greaterThanDoc', {
-        defaultMessage: 'Greater than',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definition.greaterThanDoc',
+        {
+          defaultMessage: 'Greater than',
+        }
+      ),
     },
     {
       name: '<=',
-      description: i18n.translate('monaco.esql.definition.lessThanOrEqualToDoc', {
-        defaultMessage: 'Less than or equal to',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definition.lessThanOrEqualToDoc',
+        {
+          defaultMessage: 'Less than or equal to',
+        }
+      ),
     },
     {
       name: '>=',
-      description: i18n.translate('monaco.esql.definition.greaterThanOrEqualToDoc', {
-        defaultMessage: 'Greater than or equal to',
-      }),
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definition.greaterThanOrEqualToDoc',
+        {
+          defaultMessage: 'Greater than or equal to',
+        }
+      ),
     },
   ].map((op): FunctionDefinition => createComparisonDefinition(op)),
   ...[
@@ -211,20 +229,20 @@ export const builtinFunctions: FunctionDefinition[] = [
     // new special comparison operator for strings only
     // {
     //   name: '=~',
-    //   description: i18n.translate('monaco.esql.definition.equalToCaseInsensitiveDoc', {
+    //   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.equalToCaseInsensitiveDoc', {
     //     defaultMessage: 'Case insensitive equality',
     //   }),
     // },
     {
       name: 'like',
-      description: i18n.translate('monaco.esql.definition.likeDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.likeDoc', {
         defaultMessage: 'Filter data based on string patterns',
       }),
     },
     { name: 'not_like', description: '' },
     {
       name: 'rlike',
-      description: i18n.translate('monaco.esql.definition.rlikeDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.rlikeDoc', {
         defaultMessage: 'Filter data based on string regular expressions',
       }),
     },
@@ -249,7 +267,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   ...[
     {
       name: 'in',
-      description: i18n.translate('monaco.esql.definition.inDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.inDoc', {
         defaultMessage:
           'Tests if the value an expression takes is contained in a list of other expressions',
       }),
@@ -295,13 +313,13 @@ export const builtinFunctions: FunctionDefinition[] = [
   ...[
     {
       name: 'and',
-      description: i18n.translate('monaco.esql.definition.andDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.andDoc', {
         defaultMessage: 'and',
       }),
     },
     {
       name: 'or',
-      description: i18n.translate('monaco.esql.definition.orDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.orDoc', {
         defaultMessage: 'or',
       }),
     },
@@ -324,7 +342,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   {
     type: 'builtin' as const,
     name: 'not',
-    description: i18n.translate('monaco.esql.definition.notDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.notDoc', {
       defaultMessage: 'Not',
     }),
     supportedCommands: ['eval', 'where', 'row'],
@@ -339,13 +357,13 @@ export const builtinFunctions: FunctionDefinition[] = [
   ...[
     {
       name: 'is null',
-      description: i18n.translate('monaco.esql.definition.isNullDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.isNullDoc', {
         defaultMessage: 'Predicate for NULL comparison: returns true if the value is NULL',
       }),
     },
     {
       name: 'is not null',
-      description: i18n.translate('monaco.esql.definition.isNotNullDoc', {
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.isNotNullDoc', {
         defaultMessage: 'Predicate for NULL comparison: returns true if the value is not NULL',
       }),
     },
@@ -364,7 +382,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   {
     type: 'builtin' as const,
     name: '=',
-    description: i18n.translate('monaco.esql.definition.assignDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.assignDoc', {
       defaultMessage: 'Assign (=)',
     }),
     supportedCommands: ['eval', 'stats', 'row', 'dissect', 'where', 'enrich'],
@@ -382,7 +400,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   {
     name: 'functions',
     type: 'builtin',
-    description: i18n.translate('monaco.esql.definition.functionsDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.functionsDoc', {
       defaultMessage: 'Show ES|QL avaialble functions with signatures',
     }),
     supportedCommands: ['meta'],
@@ -396,7 +414,7 @@ export const builtinFunctions: FunctionDefinition[] = [
   {
     name: 'info',
     type: 'builtin',
-    description: i18n.translate('monaco.esql.definition.infoDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definition.infoDoc', {
       defaultMessage: 'Show information about the current ES node',
     }),
     supportedCommands: ['show'],

packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts

@@ -7,6 +7,13 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import type {
+  ESQLColumn,
+  ESQLCommand,
+  ESQLAstItem,
+  ESQLMessage,
+  ESQLFunction,
+} from '@kbn/esql-ast';
 import {
   getFunctionDefinition,
   isAssignment,
@@ -14,8 +21,7 @@ import {
   isFunctionItem,
   isLiteralItem,
 } from '../shared/helpers';
-import type { ESQLColumn, ESQLCommand, ESQLAstItem, ESQLMessage, ESQLFunction } from '../types';
-import { enrichModes } from './settings';
+import { ENRICH_MODES } from './settings';
 import {
   appendSeparatorOption,
   asOption,
@@ -29,7 +35,7 @@ import type { CommandDefinition } from './types';
 export const commandDefinitions: CommandDefinition[] = [
   {
     name: 'row',
-    description: i18n.translate('monaco.esql.definitions.rowDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.rowDoc', {
       defaultMessage:
         'Produces a row with one or more columns with values that you specify. This can be useful for testing.',
     }),
@@ -44,7 +50,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'from',
-    description: i18n.translate('monaco.esql.definitions.fromDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.fromDoc', {
       defaultMessage:
         'Retrieves data from one or more data streams, indices, or aliases. In a query or subquery, you must use the from command first and it does not need a leading pipe. For example, to retrieve data from an index:',
     }),
@@ -58,7 +64,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'meta',
-    description: i18n.translate('monaco.esql.definitions.metaDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.metaDoc', {
       defaultMessage: 'Returns information about the ES|QL environment',
     }),
     examples: ['meta functions'],
@@ -71,7 +77,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'show',
-    description: i18n.translate('monaco.esql.definitions.showDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.showDoc', {
       defaultMessage: 'Returns information about the deployment and its capabilities',
     }),
     examples: ['show info'],
@@ -84,7 +90,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'stats',
-    description: i18n.translate('monaco.esql.definitions.statsDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.statsDoc', {
       defaultMessage:
         'Calculates aggregate statistics, such as average, count, and sum, over the incoming search results set. Similar to SQL aggregation, if the stats command is used without a BY clause, only one row is returned, which is the aggregation over the entire incoming search results set. When you use a BY clause, one row is returned for each distinct value in the field specified in the BY clause. The stats command returns only the fields in the aggregation, and you can use a wide range of statistical functions with the stats command. When you perform more than one aggregation, separate each aggregation with a comma.',
     }),
@@ -100,9 +106,12 @@ export const commandDefinitions: CommandDefinition[] = [
       if (!command.args.length) {
         messages.push({
           location: command.location,
-          text: i18n.translate('monaco.esql.validation.statsNoArguments', {
-            defaultMessage: 'At least one aggregation or grouping expression required in [STATS]',
-          }),
+          text: i18n.translate(
+            'kbn-esql-validation-autocomplete.esql.validation.statsNoArguments',
+            {
+              defaultMessage: 'At least one aggregation or grouping expression required in [STATS]',
+            }
+          ),
           type: 'error',
           code: 'statsNoArguments',
         });
@@ -140,13 +149,16 @@ export const commandDefinitions: CommandDefinition[] = [
           messages.push(
             ...noAggsExpressions.map((fn) => ({
               location: fn.location,
-              text: i18n.translate('monaco.esql.validation.statsNoAggFunction', {
-                defaultMessage:
-                  'At least one aggregation function required in [STATS], found [{expression}]',
-                values: {
-                  expression: fn.text,
-                },
-              }),
+              text: i18n.translate(
+                'kbn-esql-validation-autocomplete.esql.validation.statsNoAggFunction',
+                {
+                  defaultMessage:
+                    'At least one aggregation function required in [STATS], found [{expression}]',
+                  values: {
+                    expression: fn.text,
+                  },
+                }
+              ),
               type: 'error' as const,
               code: 'statsNoAggFunction',
             }))
@@ -179,13 +191,16 @@ export const commandDefinitions: CommandDefinition[] = [
             messages.push(
               ...invalidExpressions.map((fn) => ({
                 location: fn.location,
-                text: i18n.translate('monaco.esql.validation.noCombinationOfAggAndNonAggValues', {
-                  defaultMessage:
-                    'Cannot combine aggregation and non-aggregation values in [STATS], found [{expression}]',
-                  values: {
-                    expression: fn.text,
-                  },
-                }),
+                text: i18n.translate(
+                  'kbn-esql-validation-autocomplete.esql.validation.noCombinationOfAggAndNonAggValues',
+                  {
+                    defaultMessage:
+                      'Cannot combine aggregation and non-aggregation values in [STATS], found [{expression}]',
+                    values: {
+                      expression: fn.text,
+                    },
+                  }
+                ),
                 type: 'error' as const,
                 code: 'statsNoCombinationOfAggAndNonAggValues',
               }))
@@ -198,7 +213,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'eval',
-    description: i18n.translate('monaco.esql.definitions.evalDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.evalDoc', {
       defaultMessage:
         'Calculates an expression and puts the resulting value into a search results field.',
     }),
@@ -217,7 +232,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'rename',
-    description: i18n.translate('monaco.esql.definitions.renameDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.renameDoc', {
       defaultMessage: 'Renames an old column to a new one',
     }),
     examples: ['… | rename old as new', '… | rename old as new, a as b'],
@@ -230,7 +245,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'limit',
-    description: i18n.translate('monaco.esql.definitions.limitDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.limitDoc', {
       defaultMessage:
         'Returns the first search results, in search order, based on the "limit" specified.',
     }),
@@ -244,7 +259,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'keep',
-    description: i18n.translate('monaco.esql.definitions.keepDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.keepDoc', {
       defaultMessage: 'Rearranges fields in the input table by applying the keep clauses in fields',
     }),
     examples: ['… | keep a', '… | keep a,b'],
@@ -257,7 +272,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'drop',
-    description: i18n.translate('monaco.esql.definitions.dropDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.dropDoc', {
       defaultMessage: 'Drops columns',
     }),
     examples: ['… | drop a', '… | drop a,b'],
@@ -274,9 +289,12 @@ export const commandDefinitions: CommandDefinition[] = [
         messages.push(
           ...wildcardItems.map((column) => ({
             location: (column as ESQLColumn).location,
-            text: i18n.translate('monaco.esql.validation.dropAllColumnsError', {
-              defaultMessage: 'Removing all fields is not allowed [*]',
-            }),
+            text: i18n.translate(
+              'kbn-esql-validation-autocomplete.esql.validation.dropAllColumnsError',
+              {
+                defaultMessage: 'Removing all fields is not allowed [*]',
+              }
+            ),
             type: 'error' as const,
             code: 'dropAllColumnsError',
           }))
@@ -288,9 +306,13 @@ export const commandDefinitions: CommandDefinition[] = [
       if (droppingTimestamp) {
         messages.push({
           location: (droppingTimestamp as ESQLColumn).location,
-          text: i18n.translate('monaco.esql.validation.dropTimestampWarning', {
-            defaultMessage: 'Drop [@timestamp] will remove all time filters to the search results',
-          }),
+          text: i18n.translate(
+            'kbn-esql-validation-autocomplete.esql.validation.dropTimestampWarning',
+            {
+              defaultMessage:
+                'Drop [@timestamp] will remove all time filters to the search results',
+            }
+          ),
           type: 'warning',
           code: 'dropTimestampWarning',
         });
@@ -300,7 +322,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'sort',
-    description: i18n.translate('monaco.esql.definitions.sortDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.sortDoc', {
       defaultMessage:
         'Sorts all results by the specified fields. By default, null values are treated as being larger than any other value. With an ascending sort order, null values are sorted last, and with a descending sort order, null values are sorted first. You can change that by providing NULLS FIRST or NULLS LAST',
     }),
@@ -322,7 +344,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'where',
-    description: i18n.translate('monaco.esql.definitions.whereDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.whereDoc', {
       defaultMessage:
         'Uses "predicate-expressions" to filter search results. A predicate expression, when evaluated, returns TRUE or FALSE. The where command only returns the results that evaluate to TRUE. For example, to filter results for a specific field value',
     }),
@@ -336,7 +358,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'dissect',
-    description: i18n.translate('monaco.esql.definitions.dissectDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.dissectDoc', {
       defaultMessage:
         'Extracts multiple string values from a single string input, based on a pattern',
     }),
@@ -353,7 +375,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'grok',
-    description: i18n.translate('monaco.esql.definitions.grokDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.grokDoc', {
       defaultMessage:
         'Extracts multiple string values from a single string input, based on a pattern',
     }),
@@ -370,7 +392,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'mv_expand',
-    description: i18n.translate('monaco.esql.definitions.mvExpandDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvExpandDoc', {
       defaultMessage: 'Expands multivalued fields into one row per value, duplicating other fields',
     }),
     examples: ['row a=[1,2,3] | mv_expand a'],
@@ -383,7 +405,7 @@ export const commandDefinitions: CommandDefinition[] = [
   },
   {
     name: 'enrich',
-    description: i18n.translate('monaco.esql.definitions.enrichDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.enrichDoc', {
       defaultMessage:
         'Enrich table with another table. Before you can use enrich, you need to create and execute an enrich policy.',
     }),
@@ -393,7 +415,7 @@ export const commandDefinitions: CommandDefinition[] = [
       '… | enrich my-policy on pivotField with a = enrichFieldA, b = enrichFieldB',
     ],
     options: [onOption, withOption],
-    modes: [enrichModes],
+    modes: [ENRICH_MODES],
     signature: {
       multipleParams: false,
       params: [{ name: 'policyName', type: 'source', innerType: 'policy' }],

packages/kbn-esql-validation-autocomplete/src/definitions/functions.ts

@@ -7,9 +7,9 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import type { ESQLFunction } from '@kbn/esql-ast';
 import { isLiteralItem } from '../shared/helpers';
-import { ESQLFunction } from '../types';
-import { FunctionDefinition } from './types';
+import type { FunctionDefinition } from './types';
 
 const validateLogFunctions = (fnDef: ESQLFunction) => {
   const messages = [];
@@ -20,12 +20,15 @@ const validateLogFunctions = (fnDef: ESQLFunction) => {
       messages.push({
         type: 'warning' as const,
         code: 'logOfNegativeValue',
-        text: i18n.translate('monaco.esql.divide.warning.logOfNegativeValue', {
-          defaultMessage: 'Log of a negative number results in null: {value}',
-          values: {
-            value: arg.value,
-          },
-        }),
+        text: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.divide.warning.logOfNegativeValue',
+          {
+            defaultMessage: 'Log of a negative number results in null: {value}',
+            values: {
+              value: arg.value,
+            },
+          }
+        ),
         location: arg.location,
       });
     }
@@ -36,7 +39,7 @@ const validateLogFunctions = (fnDef: ESQLFunction) => {
 export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'round',
-    description: i18n.translate('monaco.esql.definitions.roundDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.roundDoc', {
       defaultMessage:
         'Returns a number rounded to the decimal, specified by he closest integer value. The default is to round to an integer.',
     }),
@@ -56,7 +59,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'abs',
-    description: i18n.translate('monaco.esql.definitions.absDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.absDoc', {
       defaultMessage: 'Returns the absolute value.',
     }),
     signatures: [
@@ -69,7 +72,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'ceil',
-    description: i18n.translate('monaco.esql.definitions.ceilDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.ceilDoc', {
       defaultMessage: 'Round a number up to the nearest integer.',
     }),
     signatures: [
@@ -82,7 +85,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'log10',
-    description: i18n.translate('monaco.esql.definitions.log10Doc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.log10Doc', {
       defaultMessage: 'Returns the log base 10.',
     }),
     signatures: [
@@ -97,7 +100,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
 
   {
     name: 'log',
-    description: i18n.translate('monaco.esql.definitions.logDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.logDoc', {
       defaultMessage:
         'A scalar function log(based, value) returns the logarithm of a value for a particular base, as specified in the argument',
     }),
@@ -118,7 +121,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'pow',
-    description: i18n.translate('monaco.esql.definitions.powDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.powDoc', {
       defaultMessage:
         'Returns the the value of a base (first argument) raised to a power (second argument).',
     }),
@@ -135,7 +138,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'concat',
-    description: i18n.translate('monaco.esql.definitions.concatDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.concatDoc', {
       defaultMessage: 'Concatenates two or more strings.',
     }),
     signatures: [
@@ -149,7 +152,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'replace',
-    description: i18n.translate('monaco.esql.definitions.replaceDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.replaceDoc', {
       defaultMessage:
         'The function substitutes in the string (1st argument) any match of the regular expression (2nd argument) with the replacement string (3rd argument). If any of the arguments are NULL, the result is NULL.',
     }),
@@ -167,7 +170,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'substring',
-    description: i18n.translate('monaco.esql.definitions.substringDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.substringDoc', {
       defaultMessage:
         'Returns a substring of a string, specified by a start position and an optional length.',
     }),
@@ -185,7 +188,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_lower',
-    description: i18n.translate('monaco.esql.definitions.toLowerDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toLowerDoc', {
       defaultMessage: 'Returns a new string representing the input string converted to lower case.',
     }),
     signatures: [
@@ -198,7 +201,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_upper',
-    description: i18n.translate('monaco.esql.definitions.toUpperDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toUpperDoc', {
       defaultMessage: 'Returns a new string representing the input string converted to upper case.',
     }),
     signatures: [
@@ -211,7 +214,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'trim',
-    description: i18n.translate('monaco.esql.definitions.trimDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.trimDoc', {
       defaultMessage: 'Removes leading and trailing whitespaces from strings.',
     }),
     signatures: [
@@ -224,7 +227,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'starts_with',
-    description: i18n.translate('monaco.esql.definitions.startsWithDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.startsWithDoc', {
       defaultMessage:
         'Returns a boolean that indicates whether a keyword string starts with another string.',
     }),
@@ -241,7 +244,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'ends_with',
-    description: i18n.translate('monaco.esql.definitions.endsWithDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.endsWithDoc', {
       defaultMessage:
         'Returns a boolean that indicates whether a keyword string ends with another string:',
     }),
@@ -258,7 +261,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'split',
-    description: i18n.translate('monaco.esql.definitions.splitDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.splitDoc', {
       defaultMessage: 'Splits a single valued string into multiple strings.',
     }),
     signatures: [
@@ -275,7 +278,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_string',
     alias: ['to_str'],
-    description: i18n.translate('monaco.esql.definitions.toStringDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toStringDoc', {
       defaultMessage: 'Converts to string.',
     }),
     signatures: [
@@ -289,7 +292,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_boolean',
     alias: ['to_bool'],
-    description: i18n.translate('monaco.esql.definitions.toBooleanDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toBooleanDoc', {
       defaultMessage: 'Converts to boolean.',
     }),
     signatures: [
@@ -302,9 +305,12 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_cartesianpoint',
-    description: i18n.translate('monaco.esql.definitions.toCartesianPointDoc', {
-      defaultMessage: 'Converts an input value to a `point` value.',
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.toCartesianPointDoc',
+      {
+        defaultMessage: 'Converts an input value to a `point` value.',
+      }
+    ),
     signatures: [
       {
         params: [{ name: 'field', type: 'any' }],
@@ -315,9 +321,12 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_cartesianshape',
-    description: i18n.translate('monaco.esql.definitions.toCartesianshapeDoc', {
-      defaultMessage: 'Converts an input value to a cartesian_shape value.',
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.toCartesianshapeDoc',
+      {
+        defaultMessage: 'Converts an input value to a cartesian_shape value.',
+      }
+    ),
     signatures: [
       {
         params: [{ name: 'field', type: 'any' }],
@@ -329,7 +338,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_datetime',
     alias: ['to_dt'],
-    description: i18n.translate('monaco.esql.definitions.toDateTimeDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toDateTimeDoc', {
       defaultMessage: 'Converts to date.',
     }),
     signatures: [
@@ -342,7 +351,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_degrees',
-    description: i18n.translate('monaco.esql.definitions.toDegreesDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toDegreesDoc', {
       defaultMessage: 'Coverts to degrees',
     }),
     signatures: [
@@ -356,7 +365,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_double',
     alias: ['to_dbl'],
-    description: i18n.translate('monaco.esql.definitions.toDoubleDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toDoubleDoc', {
       defaultMessage: 'Converts to double.',
     }),
     signatures: [
@@ -369,7 +378,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_geopoint',
-    description: i18n.translate('monaco.esql.definitions.toGeopointDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toGeopointDoc', {
       defaultMessage: 'Converts to geo_point.',
     }),
     signatures: [
@@ -382,7 +391,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_geoshape',
-    description: i18n.translate('monaco.esql.definitions.toGeoshapeDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toGeoshapeDoc', {
       defaultMessage: 'Converts an input value to a geo_shape value.',
     }),
     signatures: [
@@ -396,7 +405,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_integer',
     alias: ['to_int'],
-    description: i18n.translate('monaco.esql.definitions.toIntegerDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toIntegerDoc', {
       defaultMessage: 'Converts to integer.',
     }),
     signatures: [
@@ -409,7 +418,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_long',
-    description: i18n.translate('monaco.esql.definitions.toLongDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toLongDoc', {
       defaultMessage: 'Converts to long.',
     }),
     signatures: [
@@ -422,7 +431,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_radians',
-    description: i18n.translate('monaco.esql.definitions.toRadiansDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toRadiansDoc', {
       defaultMessage: 'Converts to radians',
     }),
     signatures: [
@@ -436,9 +445,12 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_unsigned_long',
     alias: ['to_ul', 'to_ulong'],
-    description: i18n.translate('monaco.esql.definitions.toUnsignedLongDoc', {
-      defaultMessage: 'Converts to unsigned long.',
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.toUnsignedLongDoc',
+      {
+        defaultMessage: 'Converts to unsigned long.',
+      }
+    ),
     signatures: [
       {
         params: [{ name: 'field', type: 'any' }],
@@ -449,7 +461,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'to_ip',
-    description: i18n.translate('monaco.esql.definitions.toIpDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toIpDoc', {
       defaultMessage: 'Converts to ip.',
     }),
     signatures: [
@@ -463,7 +475,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   {
     name: 'to_version',
     alias: ['to_ver'],
-    description: i18n.translate('monaco.esql.definitions.toVersionDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.toVersionDoc', {
       defaultMessage: 'Converts to version.',
     }),
     signatures: [
@@ -481,9 +493,12 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'date_extract',
-    description: i18n.translate('monaco.esql.definitions.dateExtractDoc', {
-      defaultMessage: `Extracts parts of a date, like year, month, day, hour. The supported field types are those provided by java.time.temporal.ChronoField`,
-    }),
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateExtractDoc',
+      {
+        defaultMessage: `Extracts parts of a date, like year, month, day, hour. The supported field types are those provided by java.time.temporal.ChronoField`,
+      }
+    ),
     signatures: [
       {
         params: [
@@ -502,7 +517,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'date_format',
-    description: i18n.translate('monaco.esql.definitions.dateFormatDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.dateFormatDoc', {
       defaultMessage: `Returns a string representation of a date in the provided format. If no format is specified, the "yyyy-MM-dd'T'HH:mm:ss.SSSZ" format is used.`,
     }),
     signatures: [
@@ -518,7 +533,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'date_trunc',
-    description: i18n.translate('monaco.esql.definitions.dateTruncDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.dateTruncDoc', {
       defaultMessage: `Rounds down a date to the closest interval. Intervals can be expressed using the timespan literal syntax.`,
     }),
     signatures: [
@@ -534,7 +549,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'date_parse',
-    description: i18n.translate('monaco.esql.definitions.dateParseDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.dateParseDoc', {
       defaultMessage: `Parse dates from strings.`,
     }),
     signatures: [
@@ -552,7 +567,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'auto_bucket',
-    description: i18n.translate('monaco.esql.definitions.autoBucketDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.autoBucketDoc', {
       defaultMessage: `Automatically bucket dates based on a given range and bucket target.`,
     }),
     signatures: [
@@ -582,7 +597,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'case',
-    description: i18n.translate('monaco.esql.definitions.caseDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.caseDoc', {
       defaultMessage:
         'Accepts pairs of conditions and values. The function returns the value that belongs to the first condition that evaluates to `true`. If the number of arguments is odd, the last argument is the default value which is returned when no condition matches.',
     }),
@@ -602,7 +617,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'length',
-    description: i18n.translate('monaco.esql.definitions.lengthDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.lengthDoc', {
       defaultMessage: 'Returns the character length of a string.',
     }),
     signatures: [
@@ -615,7 +630,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'acos',
-    description: i18n.translate('monaco.esql.definitions.acosDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.acosDoc', {
       defaultMessage: 'Inverse cosine trigonometric function',
     }),
     signatures: [
@@ -628,7 +643,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'asin',
-    description: i18n.translate('monaco.esql.definitions.asinDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.asinDoc', {
       defaultMessage: 'Inverse sine trigonometric function',
     }),
     signatures: [
@@ -641,7 +656,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'atan',
-    description: i18n.translate('monaco.esql.definitions.atanDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.atanDoc', {
       defaultMessage: 'Inverse tangent trigonometric function',
     }),
     signatures: [
@@ -654,7 +669,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'atan2',
-    description: i18n.translate('monaco.esql.definitions.atan2Doc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.atan2Doc', {
       defaultMessage:
         'The angle between the positive x-axis and the ray from the origin to the point (x , y) in the Cartesian plane',
     }),
@@ -671,7 +686,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'coalesce',
-    description: i18n.translate('monaco.esql.definitions.coalesceDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.coalesceDoc', {
       defaultMessage: 'Returns the first non-null value.',
     }),
     signatures: [
@@ -685,7 +700,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'cos',
-    description: i18n.translate('monaco.esql.definitions.cosDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.cosDoc', {
       defaultMessage: 'Cosine trigonometric function',
     }),
     signatures: [
@@ -698,7 +713,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'cosh',
-    description: i18n.translate('monaco.esql.definitions.coshDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.coshDoc', {
       defaultMessage: 'Cosine hyperbolic function',
     }),
     signatures: [
@@ -711,7 +726,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'floor',
-    description: i18n.translate('monaco.esql.definitions.floorDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.floorDoc', {
       defaultMessage: 'Round a number down to the nearest integer.',
     }),
     signatures: [
@@ -724,7 +739,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'greatest',
-    description: i18n.translate('monaco.esql.definitions.greatestDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.greatestDoc', {
       defaultMessage: 'Returns the maximum value from many columns.',
     }),
     signatures: [
@@ -738,7 +753,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'least',
-    description: i18n.translate('monaco.esql.definitions.leastDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.leastDoc', {
       defaultMessage: 'Returns the minimum value from many columns.',
     }),
     signatures: [
@@ -752,7 +767,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'left',
-    description: i18n.translate('monaco.esql.definitions.leftDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.leftDoc', {
       defaultMessage:
         'Return the substring that extracts length chars from the string starting from the left.',
     }),
@@ -769,7 +784,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'ltrim',
-    description: i18n.translate('monaco.esql.definitions.ltrimDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.ltrimDoc', {
       defaultMessage: 'Removes leading whitespaces from strings.',
     }),
     signatures: [
@@ -782,7 +797,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'now',
-    description: i18n.translate('monaco.esql.definitions.nowDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.nowDoc', {
       defaultMessage: 'Returns current date and time.',
     }),
     signatures: [
@@ -795,7 +810,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'right',
-    description: i18n.translate('monaco.esql.definitions.rightDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.rightDoc', {
       defaultMessage:
         'Return the substring that extracts length chars from the string starting from the right.',
     }),
@@ -812,7 +827,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'rtrim',
-    description: i18n.translate('monaco.esql.definitions.rtrimDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.rtrimDoc', {
       defaultMessage: 'Removes trailing whitespaces from strings.',
     }),
     signatures: [
@@ -825,7 +840,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'sin',
-    description: i18n.translate('monaco.esql.definitions.sinDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.sinDoc', {
       defaultMessage: 'Sine trigonometric function.',
     }),
     signatures: [
@@ -838,7 +853,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'sinh',
-    description: i18n.translate('monaco.esql.definitions.sinhDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.sinhDoc', {
       defaultMessage: 'Sine hyperbolic function.',
     }),
     signatures: [
@@ -851,7 +866,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'sqrt',
-    description: i18n.translate('monaco.esql.definitions.sqrtDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.sqrtDoc', {
       defaultMessage: 'Returns the square root of a number. ',
     }),
     signatures: [
@@ -864,7 +879,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'tan',
-    description: i18n.translate('monaco.esql.definitions.tanDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.tanDoc', {
       defaultMessage: 'Tangent trigonometric function.',
     }),
     signatures: [
@@ -877,7 +892,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'tanh',
-    description: i18n.translate('monaco.esql.definitions.tanhDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.tanhDoc', {
       defaultMessage: 'Tangent hyperbolic function.',
     }),
     signatures: [
@@ -890,7 +905,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'cidr_match',
-    description: i18n.translate('monaco.esql.definitions.cidrMatchDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.cidrMatchDoc', {
       defaultMessage:
         'The function takes a first parameter of type IP, followed by one or more parameters evaluated to a CIDR specificatione.',
     }),
@@ -911,7 +926,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_avg',
-    description: i18n.translate('monaco.esql.definitions.mvAvgDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvAvgDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing the average of all of the values.',
     }),
@@ -925,7 +940,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_concat',
-    description: i18n.translate('monaco.esql.definitions.mvConcatDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvConcatDoc', {
       defaultMessage:
         'Converts a multivalued string field into a single valued field containing the concatenation of all values separated by a delimiter',
     }),
@@ -942,7 +957,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_count',
-    description: i18n.translate('monaco.esql.definitions.mvCountDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvCountDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing a count of the number of values',
     }),
@@ -956,7 +971,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_dedupe',
-    description: i18n.translate('monaco.esql.definitions.mvDedupeDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvDedupeDoc', {
       defaultMessage: 'Removes duplicates from a multivalued field',
     }),
     signatures: [
@@ -969,7 +984,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_first',
-    description: i18n.translate('monaco.esql.definitions.mvFirstDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvFirstDoc', {
       defaultMessage:
         'Reduce a multivalued field to a single valued field containing the first value.',
     }),
@@ -983,7 +998,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_last',
-    description: i18n.translate('monaco.esql.definitions.mvLastDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvLastDoc', {
       defaultMessage:
         'Reduce a multivalued field to a single valued field containing the last value.',
     }),
@@ -997,7 +1012,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_max',
-    description: i18n.translate('monaco.esql.definitions.mvMaxDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvMaxDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing the maximum value.',
     }),
@@ -1011,7 +1026,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_min',
-    description: i18n.translate('monaco.esql.definitions.mvMinDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvMinDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing the minimum value.',
     }),
@@ -1025,7 +1040,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_median',
-    description: i18n.translate('monaco.esql.definitions.mvMedianDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvMedianDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing the median value.',
     }),
@@ -1039,7 +1054,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'mv_sum',
-    description: i18n.translate('monaco.esql.definitions.mvSumDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvSumDoc', {
       defaultMessage:
         'Converts a multivalued field into a single valued field containing the sum of all of the values.',
     }),
@@ -1053,7 +1068,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'pi',
-    description: i18n.translate('monaco.esql.definitions.piDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.piDoc', {
       defaultMessage: 'The ratio of a circle’s circumference to its diameter.',
     }),
     signatures: [
@@ -1066,7 +1081,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'e',
-    description: i18n.translate('monaco.esql.definitions.eDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.eDoc', {
       defaultMessage: 'Euler’s number.',
     }),
     signatures: [
@@ -1079,7 +1094,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
   },
   {
     name: 'tau',
-    description: i18n.translate('monaco.esql.definitions.tauDoc', {
+    description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.tauDoc', {
       defaultMessage: 'The ratio of a circle’s circumference to its radius.',
     }),
     signatures: [

packages/kbn-esql-validation-autocomplete/src/definitions/helpers.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { CommandDefinition, FunctionDefinition } from './types';
+import type { CommandDefinition, FunctionDefinition } from './types';
 
 /**
  * Given a function definition, this function will return a list of function signatures

packages/kbn-esql-validation-autocomplete/src/definitions/literals.ts

@@ -0,0 +1,190 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import type { Literals } from './types';
+
+export const timeLiterals: Literals[] = [
+  {
+    name: 'year',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.year',
+      {
+        defaultMessage: 'Year',
+      }
+    ),
+  },
+  {
+    name: 'years',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.years',
+      {
+        defaultMessage: 'Years (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'month',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.month',
+      {
+        defaultMessage: 'Month',
+      }
+    ),
+  },
+  {
+    name: 'months',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.months',
+      {
+        defaultMessage: 'Months (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'week',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.week',
+      {
+        defaultMessage: 'Week',
+      }
+    ),
+  },
+  {
+    name: 'weeks',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.weeks',
+      {
+        defaultMessage: 'Weeks (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'day',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.day',
+      {
+        defaultMessage: 'Day',
+      }
+    ),
+  },
+  {
+    name: 'days',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.days',
+      {
+        defaultMessage: 'Days (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'hour',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.hour',
+      {
+        defaultMessage: 'Hour',
+      }
+    ),
+  },
+  {
+    name: 'hours',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.hours',
+      {
+        defaultMessage: 'Hours (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'minute',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.minute',
+      {
+        defaultMessage: 'Minute',
+      }
+    ),
+  },
+  {
+    name: 'minutes',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.minutes',
+      {
+        defaultMessage: 'Minutes (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'second',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.second',
+      {
+        defaultMessage: 'Second',
+      }
+    ),
+  },
+  {
+    name: 'seconds',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.seconds',
+      {
+        defaultMessage: 'Seconds (Plural)',
+      }
+    ),
+  },
+  {
+    name: 'millisecond',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.millisecond',
+      {
+        defaultMessage: 'Millisecond',
+      }
+    ),
+  },
+  {
+    name: 'milliseconds',
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.dateDurationDefinition.milliseconds',
+      {
+        defaultMessage: 'Milliseconds (Plural)',
+      }
+    ),
+  },
+];
+
+export const chronoLiterals: Literals[] = [
+  'ALIGNED_DAY_OF_WEEK_IN_MONTH',
+  'ALIGNED_DAY_OF_WEEK_IN_YEAR',
+  'ALIGNED_WEEK_OF_MONTH',
+  'ALIGNED_WEEK_OF_YEAR',
+  'AMPM_OF_DAY',
+  'CLOCK_HOUR_OF_AMPM',
+  'CLOCK_HOUR_OF_DAY',
+  'DAY_OF_MONTH',
+  'DAY_OF_WEEK',
+  'DAY_OF_YEAR',
+  'EPOCH_DAY',
+  'ERA',
+  'HOUR_OF_AMPM',
+  'HOUR_OF_DAY',
+  'INSTANT_SECONDS',
+  'MICRO_OF_DAY',
+  'MICRO_OF_SECOND',
+  'MILLI_OF_DAY',
+  'MILLI_OF_SECOND',
+  'MINUTE_OF_DAY',
+  'MINUTE_OF_HOUR',
+  'MONTH_OF_YEAR',
+  'NANO_OF_DAY',
+  'NANO_OF_SECOND',
+  'OFFSET_SECONDS',
+  'PROLEPTIC_MONTH',
+  'SECOND_OF_DAY',
+  'SECOND_OF_MINUTE',
+  'YEAR',
+  'YEAR_OF_ERA',
+].map((name) => ({ name: `"${name}"`, description: '' }));

packages/kbn-esql-validation-autocomplete/src/definitions/options.ts

@@ -7,13 +7,14 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import type { ESQLCommandOption, ESQLMessage } from '@kbn/esql-ast';
 import { isColumnItem, isLiteralItem } from '../shared/helpers';
-import { ESQLCommandOption, ESQLMessage } from '../types';
-import { CommandOptionsDefinition } from './types';
+import { getMessageFromId } from '../validation/errors';
+import type { CommandOptionsDefinition } from './types';
 
 export const byOption: CommandOptionsDefinition = {
   name: 'by',
-  description: i18n.translate('monaco.esql.definitions.byDoc', {
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.byDoc', {
     defaultMessage: 'By',
   }),
   signature: {
@@ -25,7 +26,7 @@ export const byOption: CommandOptionsDefinition = {
 
 export const metadataOption: CommandOptionsDefinition = {
   name: 'metadata',
-  description: i18n.translate('monaco.esql.definitions.metadataDoc', {
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.metadataDoc', {
     defaultMessage: 'Metadata',
   }),
   signature: {
@@ -38,33 +39,29 @@ export const metadataOption: CommandOptionsDefinition = {
     const messages: ESQLMessage[] = [];
     // need to test the parent command here
     if (/\[metadata/i.test(command.text)) {
-      messages.push({
-        location: option.location,
-        text: i18n.translate('monaco.esql.validation.metadataBracketsDeprecation', {
-          defaultMessage: "Square brackets '[]' need to be removed from FROM METADATA declaration",
-        }),
-        type: 'warning',
-        code: 'metadataBracketsDeprecation',
-      });
+      messages.push(
+        getMessageFromId({
+          messageId: 'metadataBracketsDeprecation',
+          values: {},
+          locations: option.location,
+        })
+      );
     }
     const fields = option.args.filter(isColumnItem);
     const metadataFieldsAvailable = references as unknown as Set<string>;
     if (metadataFieldsAvailable.size > 0) {
       for (const field of fields) {
         if (!metadataFieldsAvailable.has(field.name)) {
-          messages.push({
-            location: field.location,
-            text: i18n.translate('monaco.esql.validation.wrongMetadataArgumentType', {
-              defaultMessage:
-                'Metadata field [{value}] is not available. Available metadata fields are: [{availableFields}]',
+          messages.push(
+            getMessageFromId({
+              messageId: 'unknownMetadataField',
               values: {
                 value: field.name,
                 availableFields: Array.from(metadataFieldsAvailable).join(', '),
               },
-            }),
-            type: 'error',
-            code: 'unknownMetadataField',
-          });
+              locations: field.location,
+            })
+          );
         }
       }
     }
@@ -74,7 +71,9 @@ export const metadataOption: CommandOptionsDefinition = {
 
 export const asOption: CommandOptionsDefinition = {
   name: 'as',
-  description: i18n.translate('monaco.esql.definitions.asDoc', { defaultMessage: 'As' }),
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.asDoc', {
+    defaultMessage: 'As',
+  }),
   signature: {
     multipleParams: false,
     params: [
@@ -87,7 +86,9 @@ export const asOption: CommandOptionsDefinition = {
 
 export const onOption: CommandOptionsDefinition = {
   name: 'on',
-  description: i18n.translate('monaco.esql.definitions.onDoc', { defaultMessage: 'On' }),
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.onDoc', {
+    defaultMessage: 'On',
+  }),
   signature: {
     multipleParams: false,
     params: [{ name: 'matchingColumn', type: 'column' }],
@@ -97,7 +98,9 @@ export const onOption: CommandOptionsDefinition = {
 
 export const withOption: CommandOptionsDefinition = {
   name: 'with',
-  description: i18n.translate('monaco.esql.definitions.withDoc', { defaultMessage: 'With' }),
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.withDoc', {
+    defaultMessage: 'With',
+  }),
   signature: {
     multipleParams: true,
     params: [{ name: 'assignment', type: 'any' }],
@@ -107,10 +110,13 @@ export const withOption: CommandOptionsDefinition = {
 
 export const appendSeparatorOption: CommandOptionsDefinition = {
   name: 'append_separator',
-  description: i18n.translate('monaco.esql.definitions.appendSeparatorDoc', {
-    defaultMessage:
-      'The character(s) that separate the appended fields. Default to empty string ("").',
-  }),
+  description: i18n.translate(
+    'kbn-esql-validation-autocomplete.esql.definitions.appendSeparatorDoc',
+    {
+      defaultMessage:
+        'The character(s) that separate the appended fields. Default to empty string ("").',
+    }
+  ),
   signature: {
     multipleParams: false,
     params: [{ name: 'separator', type: 'string' }],
@@ -125,18 +131,13 @@ export const appendSeparatorOption: CommandOptionsDefinition = {
       (!isLiteralItem(firstArg) || firstArg.literalType !== 'string')
     ) {
       const value = 'value' in firstArg ? firstArg.value : firstArg.name;
-      messages.push({
-        location: firstArg.location,
-        text: i18n.translate('monaco.esql.validation.wrongDissectOptionArgumentType', {
-          defaultMessage:
-            'Invalid value for DISSECT append_separator: expected a string, but was [{value}]',
-          values: {
-            value,
-          },
-        }),
-        type: 'error',
-        code: 'wrongDissectOptionArgumentType',
-      });
+      messages.push(
+        getMessageFromId({
+          messageId: 'wrongDissectOptionArgumentType',
+          values: { value },
+          locations: firstArg.location,
+        })
+      );
     }
     return messages;
   },

packages/kbn-esql-validation-autocomplete/src/definitions/settings.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import type { CommandModeDefinition } from './types';
+
+export const ENRICH_MODES: CommandModeDefinition = {
+  name: 'ccq.mode',
+  description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.ccqModeDoc', {
+    defaultMessage: 'Cross-clusters query mode',
+  }),
+  prefix: '_',
+  values: [
+    {
+      name: 'any',
+      description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.ccqAnyDoc', {
+        defaultMessage: 'Enrich takes place on any cluster',
+      }),
+    },
+    {
+      name: 'coordinator',
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definitions.ccqCoordinatorDoc',
+        {
+          defaultMessage: 'Enrich takes place on the coordinating cluster receiving an ES|QL',
+        }
+      ),
+    },
+    {
+      name: 'remote',
+      description: i18n.translate(
+        'kbn-esql-validation-autocomplete.esql.definitions.ccqRemoteDoc',
+        {
+          defaultMessage: 'Enrich takes place on the cluster hosting the target index.',
+        }
+      ),
+    },
+  ],
+};

packages/kbn-esql-validation-autocomplete/src/definitions/types.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { ESQLCommand, ESQLCommandOption, ESQLFunction, ESQLMessage } from '../types';
+import type { ESQLCommand, ESQLCommandOption, ESQLFunction, ESQLMessage } from '@kbn/esql-ast';
 
 export interface FunctionDefinition {
   type: 'builtin' | 'agg' | 'eval';

packages/kbn-esql-validation-autocomplete/src/shared/constants.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const EDITOR_MARKER = 'marker_esql_editor';
+
+export const TICKS_REGEX = /^`{1}|`{1}$/g;
+export const DOUBLE_TICKS_REGEX = /``/g;
+export const SINGLE_TICK_REGEX = /`/g;
+export const SINGLE_BACKTICK = '`';
+export const DOUBLE_BACKTICK = '``';

packages/kbn-esql-validation-autocomplete/src/shared/context.ts

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import { enrichModes } from '../definitions/settings';
 import type {
   ESQLAstItem,
   ESQLSingleAstItem,
@@ -15,7 +14,8 @@ import type {
   ESQLCommand,
   ESQLCommandOption,
   ESQLCommandMode,
-} from '../types';
+} from '@kbn/esql-ast';
+import { ENRICH_MODES } from '../definitions/settings';
 import { EDITOR_MARKER } from './constants';
 import {
   isOptionItem,
@@ -136,7 +136,20 @@ function isBuiltinFunction(node: ESQLFunction) {
   return getFunctionDefinition(node.name)?.type === 'builtin';
 }
 
-export function getAstContext(innerText: string, ast: ESQLAst, offset: number) {
+/**
+ * Given a ES|QL query string, its AST and the cursor position,
+ * it returns the type of context for the position ("list", "function", "option", "setting", "expression", "newCommand")
+ * plus the whole hierarchy of nodes (command, option, setting and actual position node) context.
+ *
+ * Type details:
+ * * "list": the cursor is inside a "in" list of values (i.e. `a in (1, 2, <here>)`)
+ * * "function": the cursor is inside a function call (i.e. `fn(<here>)`)
+ * * "option": the cursor is inside a command option (i.e. `command ... by <here>`)
+ * * "setting": the cursor is inside a setting (i.e. `command _<here>`)
+ * * "expression": the cursor is inside a command expression (i.e. `command ... <here>` or `command a = ... <here>`)
+ * * "newCommand": the cursor is at the beginning of a new command (i.e. `command1 | command2 | <here>`)
+ */
+export function getAstContext(queryString: string, ast: ESQLAst, offset: number) {
   const { command, option, setting, node } = findAstPosition(ast, offset);
   if (node) {
     if (node.type === 'function') {
@@ -154,13 +167,12 @@ export function getAstContext(innerText: string, ast: ESQLAst, offset: number) {
       return { type: 'option' as const, command, node, option, setting };
     }
     // for now it's only an enrich thing
-    if (node.type === 'source' && node.text === enrichModes.prefix) {
+    if (node.type === 'source' && node.text === ENRICH_MODES.prefix) {
       // command _<here>
       return { type: 'setting' as const, command, node, option, setting };
     }
   }
-
-  if (!command || (innerText.length <= offset && getLastCharFromTrimmed(innerText) === '|')) {
+  if (!command || (queryString.length <= offset && getLastCharFromTrimmed(queryString) === '|')) {
     //   // ... | <here>
     return { type: 'newCommand' as const, command: undefined, node, option, setting };
   }

packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts

@@ -6,8 +6,17 @@
  * Side Public License, v 1.
  */
 
-import type { monaco } from '../../../../monaco_imports';
-import { AutocompleteCommandDefinition } from '../autocomplete/types';
+import type {
+  ESQLAstItem,
+  ESQLColumn,
+  ESQLCommandMode,
+  ESQLCommandOption,
+  ESQLFunction,
+  ESQLLiteral,
+  ESQLSingleAstItem,
+  ESQLSource,
+  ESQLTimeInterval,
+} from '@kbn/esql-ast';
 import { statsAggregationFunctionDefinitions } from '../definitions/aggs';
 import { builtinFunctions } from '../definitions/builtin';
 import { commandDefinitions } from '../definitions/commands';
@@ -22,25 +31,19 @@ import {
   withOption,
   appendSeparatorOption,
 } from '../definitions/options';
-import {
+import type {
   CommandDefinition,
   CommandOptionsDefinition,
   FunctionDefinition,
   SignatureArgType,
 } from '../definitions/types';
-import {
-  ESQLAstItem,
-  ESQLColumn,
-  ESQLCommandMode,
-  ESQLCommandOption,
-  ESQLFunction,
-  ESQLLiteral,
-  ESQLSingleAstItem,
-  ESQLSource,
-  ESQLTimeInterval,
-} from '../types';
-import { ESQLRealField, ESQLVariable, ReferenceMaps } from '../validation/types';
+import type { ESQLRealField, ESQLVariable, ReferenceMaps } from '../validation/types';
 import { removeMarkerArgFromArgsList } from './context';
+import type { ReasonTypes } from './types';
+
+export function nonNullable<T>(v: T): v is NonNullable<T> {
+  return v != null;
+}
 
 export function isSingleItem(arg: ESQLAstItem): arg is ESQLSingleAstItem {
   return arg && !Array.isArray(arg);
@@ -114,22 +117,10 @@ export function isComma(char: string) {
   return char === ',';
 }
 
-export function isSourceCommand({ label }: AutocompleteCommandDefinition) {
+export function isSourceCommand({ label }: { label: string }) {
   return ['from', 'row', 'show'].includes(String(label));
 }
 
-// From Monaco position to linear offset
-export function monacoPositionToOffset(expression: string, position: monaco.Position): number {
-  const lines = expression.split(/\n/);
-  return lines
-    .slice(0, position.lineNumber)
-    .reduce(
-      (prev, current, index) =>
-        prev + (index === position.lineNumber - 1 ? position.column - 1 : current.length + 1),
-      0
-    );
-}
-
 let fnLookups: Map<string, FunctionDefinition> | undefined;
 let commandLookups: Map<string, CommandDefinition> | undefined;
 
@@ -150,8 +141,6 @@ function buildFunctionLookup() {
   return fnLookups;
 }
 
-type ReasonTypes = 'missingCommand' | 'unsupportedFunction' | 'unknownFunction';
-
 export function isSupportedFunction(
   name: string,
   parentCommand?: string,

packages/kbn-esql-validation-autocomplete/src/shared/resources_helpers.ts

@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
+import type { ESQLAst } from '@kbn/esql-ast';
 import type { ESQLCallbacks } from './types';
 import type { ESQLRealField } from '../validation/types';
-import type { ESQLAst } from '../types';
 
 export function buildQueryUntilPreviousCommand(ast: ESQLAst, queryString: string) {
   const prevCommand = ast[Math.max(ast.length - 2, 0)];

packages/kbn-esql-validation-autocomplete/src/shared/types.ts

@@ -18,6 +18,6 @@ export interface ESQLCallbacks {
     {},
     { name: string; sourceIndices: string[]; matchField: string; enrichFields: string[] }
   >;
-  getPolicyFields?: CallbackFn;
-  getPolicyMatchingField?: CallbackFn;
 }
+
+export type ReasonTypes = 'missingCommand' | 'unsupportedFunction' | 'unknownFunction';

packages/kbn-esql-validation-autocomplete/src/shared/variables.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { ESQLAstItem, ESQLCommand, ESQLCommandOption, ESQLFunction } from '../types';
+import type { ESQLAstItem, ESQLCommand, ESQLCommandOption, ESQLFunction } from '@kbn/esql-ast';
 import type { ESQLVariable, ESQLRealField } from '../validation/types';
 import { DOUBLE_BACKTICK, EDITOR_MARKER, SINGLE_BACKTICK } from './constants';
 import {

packages/kbn-esql-validation-autocomplete/src/types.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export interface EditorError {
+  startLineNumber: number;
+  endLineNumber: number;
+  startColumn: number;
+  endColumn: number;
+  message: string;
+  code?: string;
+  severity: 'error' | 'warning' | number;
+}

packages/kbn-esql-validation-autocomplete/src/validation/errors.ts

@@ -0,0 +1,391 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import type { ESQLLocation, ESQLMessage } from '@kbn/esql-ast';
+import type { ErrorTypes, ErrorValues } from './types';
+
+function getMessageAndTypeFromId<K extends ErrorTypes>({
+  messageId,
+  values,
+}: {
+  messageId: K;
+  values: ErrorValues<K>;
+}): { message: string; type?: 'error' | 'warning' } {
+  // Use a less strict type instead of doing a typecast on each message type
+  const out = values as unknown as Record<string, string>;
+  // i18n validation wants to the values prop to be declared inline, so need to unpack and redeclare again all props
+  switch (messageId) {
+    case 'wrongArgumentType':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongArgumentType',
+          {
+            defaultMessage:
+              'Argument of [{name}] must be [{argType}], found value [{value}] type [{givenType}]',
+            values: {
+              name: out.name,
+              argType: out.argType,
+              value: out.value,
+              givenType: out.givenType,
+            },
+          }
+        ),
+      };
+    case 'unknownColumn':
+      return {
+        message: i18n.translate('kbn-esql-validation-autocomplete.esql.validation.unknownColumn', {
+          defaultMessage: 'Unknown column [{name}]',
+          values: { name: out.name },
+        }),
+      };
+    case 'unknownIndex':
+      return {
+        message: i18n.translate('kbn-esql-validation-autocomplete.esql.validation.unknownIndex', {
+          defaultMessage: 'Unknown index [{name}]',
+          values: { name: out.name },
+        }),
+      };
+    case 'unknownFunction':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.missingFunction',
+          {
+            defaultMessage: 'Unknown function [{name}]',
+            values: { name: out.name },
+          }
+        ),
+      };
+    case 'wrongArgumentNumber':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongArgumentExactNumber',
+          {
+            defaultMessage:
+              'Error: [{fn}] function expects exactly {numArgs, plural, one {one argument} other {{numArgs} arguments}}, got {passedArgs}.',
+            values: {
+              fn: out.fn,
+              numArgs: out.numArgs,
+              passedArgs: out.passedArgs,
+            },
+          }
+        ),
+      };
+    case 'wrongArgumentNumberTooMany':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongArgumentTooManyNumber',
+          {
+            defaultMessage:
+              'Error: [{fn}] function expects {extraArgs, plural, =0 {} other {no more than }}{numArgs, plural, one {one argument} other {{numArgs} arguments}}, got {passedArgs}.',
+            values: {
+              fn: out.fn,
+              numArgs: out.numArgs,
+              passedArgs: out.passedArgs,
+              extraArgs: out.extraArgs,
+            },
+          }
+        ),
+      };
+    case 'wrongArgumentNumberTooFew':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongArgumentTooFewNumber',
+          {
+            defaultMessage:
+              'Error: [{fn}] function expects {missingArgs, plural, =0 {} other {at least }}{numArgs, plural, one {one argument} other {{numArgs} arguments}}, got {passedArgs}.',
+            values: {
+              fn: out.fn,
+              numArgs: out.numArgs,
+              passedArgs: out.passedArgs,
+              missingArgs: out.missingArgs,
+            },
+          }
+        ),
+      };
+    case 'noNestedArgumentSupport':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.noNestedArgumentSupport',
+          {
+            defaultMessage:
+              "Aggregate function's parameters must be an attribute, literal or a non-aggregation function; found [{name}] of type [{argType}]",
+            values: { name: out.name, argType: out.argType },
+          }
+        ),
+      };
+    case 'shadowFieldType':
+      return {
+        message: i18n.translate('kbn-esql-validation-autocomplete.esql.validation.typeOverwrite', {
+          defaultMessage:
+            'Column [{field}] of type {fieldType} has been overwritten as new type: {newType}',
+          values: { field: out.field, fieldType: out.fieldType, newType: out.newType },
+        }),
+        type: 'warning',
+      };
+    case 'unsupportedColumnTypeForCommand':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedColumnTypeForCommand',
+          {
+            defaultMessage:
+              '{command} only supports {type} {typeCount, plural, one {type} other {types}} values, found [{column}] of type [{givenType}]',
+            values: {
+              command: out.command,
+              type: out.type,
+              typeCount: out.typeCount,
+              column: out.column,
+              givenType: out.givenType,
+            },
+          }
+        ),
+      };
+    case 'unknownOption':
+      return {
+        message: i18n.translate('kbn-esql-validation-autocomplete.esql.validation.unknownOption', {
+          defaultMessage: 'Invalid option for {command}: [{option}]',
+          values: {
+            command: out.command,
+            option: out.option,
+          },
+        }),
+      };
+    case 'unsupportedFunctionForCommand':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedFunctionForCommand',
+          {
+            defaultMessage: '{command} does not support function {name}',
+            values: {
+              command: out.command,
+              name: out.name,
+            },
+          }
+        ),
+      };
+    case 'unsupportedFunctionForCommandOption':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedFunctionforCommandOption',
+          {
+            defaultMessage: '{command} {option} does not support function {name}',
+            values: {
+              command: out.command,
+              option: out.option,
+              name: out.name,
+            },
+          }
+        ),
+      };
+    case 'unknownInterval':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unknownInterval',
+          {
+            defaultMessage: `Unexpected time interval qualifier: '{value}'`,
+            values: {
+              value: out.value,
+            },
+          }
+        ),
+      };
+    case 'unsupportedTypeForCommand':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedTypeForCommand',
+          {
+            defaultMessage: '{command} does not support [{type}] in expression [{value}]',
+            values: {
+              command: out.command,
+              type: out.type,
+              value: out.value,
+            },
+          }
+        ),
+      };
+    case 'unknownPolicy':
+      return {
+        message: i18n.translate('kbn-esql-validation-autocomplete.esql.validation.unknownPolicy', {
+          defaultMessage: 'Unknown policy [{name}]',
+          values: {
+            name: out.name,
+          },
+        }),
+      };
+    case 'unknownAggregateFunction':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unknowAggregateFunction',
+          {
+            defaultMessage:
+              'Expected an aggregate function or group but got [{value}] of type [{type}]',
+            values: {
+              type: out.type,
+              value: out.value,
+            },
+          }
+        ),
+      };
+    case 'wildcardNotSupportedForCommand':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wildcardNotSupportedForCommand',
+          {
+            defaultMessage: 'Using wildcards (*) in {command} is not allowed [{value}]',
+            values: {
+              command: out.command,
+              value: out.value,
+            },
+          }
+        ),
+      };
+    case 'noWildcardSupportAsArg':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wildcardNotSupportedForFunction',
+          {
+            defaultMessage: 'Using wildcards (*) in {name} is not allowed',
+            values: {
+              name: out.name,
+            },
+          }
+        ),
+      };
+    case 'unsupportedFieldType':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedFieldType',
+          {
+            defaultMessage:
+              'Field [{field}] cannot be retrieved, it is unsupported or not indexed; returning null',
+            values: {
+              field: out.field,
+            },
+          }
+        ),
+        type: 'warning',
+      };
+    case 'unsupportedSetting':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedSetting',
+          {
+            defaultMessage: 'Unsupported setting [{setting}], expected [{expected}]',
+            values: {
+              setting: out.setting,
+              expected: out.expected,
+            },
+          }
+        ),
+        type: 'error',
+      };
+    case 'unsupportedSettingCommandValue':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.unsupportedSettingValue',
+          {
+            defaultMessage:
+              'Unrecognized value [{value}] for {command}, mode needs to be one of [{expected}]',
+            values: {
+              expected: out.expected,
+              value: out.value,
+              command: out.command,
+            },
+          }
+        ),
+        type: 'error',
+      };
+    case 'expectedConstant':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.expectedConstantValue',
+          {
+            defaultMessage: 'Argument of [{fn}] must be a constant, received [{given}]',
+            values: {
+              given: out.given,
+              fn: out.fn,
+            },
+          }
+        ),
+        type: 'error',
+      };
+    case 'metadataBracketsDeprecation':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.metadataBracketsDeprecation',
+          {
+            defaultMessage:
+              "Square brackets '[]' need to be removed from FROM METADATA declaration",
+          }
+        ),
+        type: 'warning',
+      };
+    case 'unknownMetadataField':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongMetadataArgumentType',
+          {
+            defaultMessage:
+              'Metadata field [{value}] is not available. Available metadata fields are: [{availableFields}]',
+            values: {
+              value: out.value,
+              availableFields: out.availableFields,
+            },
+          }
+        ),
+        type: 'error',
+      };
+    case 'wrongDissectOptionArgumentType':
+      return {
+        message: i18n.translate(
+          'kbn-esql-validation-autocomplete.esql.validation.wrongDissectOptionArgumentType',
+          {
+            defaultMessage:
+              'Invalid value for DISSECT append_separator: expected a string, but was [{value}]',
+            values: {
+              value: out.value,
+            },
+          }
+        ),
+        type: 'error',
+      };
+  }
+  return { message: '' };
+}
+
+export function getMessageFromId<K extends ErrorTypes>({
+  locations,
+  ...payload
+}: {
+  messageId: K;
+  values: ErrorValues<K>;
+  locations: ESQLLocation;
+}): ESQLMessage {
+  const { message, type = 'error' } = getMessageAndTypeFromId(payload);
+  return createMessage(type, message, locations, payload.messageId);
+}
+
+export function createMessage(
+  type: 'error' | 'warning',
+  message: string,
+  location: ESQLLocation,
+  messageId: string
+) {
+  return {
+    type,
+    text: message,
+    location,
+    code: messageId,
+  };
+}
+
+export function getUnknownTypeLabel() {
+  return i18n.translate('kbn-esql-validation-autocomplete.esql.validation.unknownColumnType', {
+    defaultMessage: 'Unknown type',
+  });
+}

packages/kbn-esql-validation-autocomplete/src/validation/helpers.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import type { ESQLAst } from '../types';
+import type { ESQLAst } from '@kbn/esql-ast';
 import type { ESQLPolicy } from './types';
 
 export function buildQueryForFieldsFromSource(queryString: string, ast: ESQLAst) {

packages/kbn-esql-validation-autocomplete/src/validation/resources.ts

@@ -6,15 +6,14 @@
  * Side Public License, v 1.
  */
 
-import { nonNullable } from '../ast_helpers';
-import { createMapFromList, isSourceItem } from '../shared/helpers';
+import type { ESQLCommand } from '@kbn/esql-ast';
+import { createMapFromList, isSourceItem, nonNullable } from '../shared/helpers';
 import {
   getFieldsByTypeHelper,
   getPolicyHelper,
   getSourcesHelper,
 } from '../shared/resources_helpers';
 import type { ESQLCallbacks } from '../shared/types';
-import type { ESQLCommand } from '../types';
 import {
   buildQueryForFieldsForStringSources,
   buildQueryForFieldsFromSource,

packages/kbn-esql-validation-autocomplete/src/validation/types.ts

@@ -6,8 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { EditorError } from '../../../../types';
-import { ESQLMessage, ESQLLocation } from '../types';
+import type { ESQLMessage, ESQLLocation } from '@kbn/esql-ast';
+import type { EditorError } from '../types';
 
 export interface ESQLVariable {
   name: string;
@@ -152,6 +152,18 @@ export interface ValidationErrors {
     message: string;
     type: { fn: string; given: string };
   };
+  metadataBracketsDeprecation: {
+    message: string;
+    type: {};
+  };
+  unknownMetadataField: {
+    message: string;
+    type: { value: string; availableFields: string };
+  };
+  wrongDissectOptionArgumentType: {
+    message: string;
+    type: { value: string | number };
+  };
 }
 
 export type ErrorTypes = keyof ValidationErrors;
@@ -161,3 +173,7 @@ export interface ValidationResult {
   errors: Array<ESQLMessage | EditorError>;
   warnings: ESQLMessage[];
 }
+
+export interface ValidationOptions {
+  ignoreOnMissingCallbacks?: boolean;
+}

packages/kbn-esql-validation-autocomplete/src/validation/validation.ts

@@ -7,6 +7,18 @@
  */
 
 import uniqBy from 'lodash/uniqBy';
+import type {
+  AstProviderFn,
+  ESQLAstItem,
+  ESQLColumn,
+  ESQLCommand,
+  ESQLCommandMode,
+  ESQLCommandOption,
+  ESQLFunction,
+  ESQLMessage,
+  ESQLSingleAstItem,
+  ESQLSource,
+} from '@kbn/esql-ast';
 import {
   CommandModeDefinition,
   CommandOptionsDefinition,
@@ -41,20 +53,15 @@ import {
   isVariable,
 } from '../shared/helpers';
 import { collectVariables } from '../shared/variables';
-import type {
-  AstProviderFn,
-  ESQLAstItem,
-  ESQLColumn,
-  ESQLCommand,
-  ESQLCommandMode,
-  ESQLCommandOption,
-  ESQLFunction,
-  ESQLMessage,
-  ESQLSingleAstItem,
-  ESQLSource,
-} from '../types';
 import { getMessageFromId, getUnknownTypeLabel } from './errors';
-import type { ESQLRealField, ESQLVariable, ReferenceMaps, ValidationResult } from './types';
+import type {
+  ErrorTypes,
+  ESQLRealField,
+  ESQLVariable,
+  ReferenceMaps,
+  ValidationOptions,
+  ValidationResult,
+} from './types';
 import type { ESQLCallbacks } from '../shared/types';
 import {
   retrieveSources,
@@ -820,13 +827,71 @@ function validateUnsupportedTypeFields(fields: Map<string, ESQLRealField>) {
   return messages;
 }
 
+export const ignoreErrorsMap: Record<keyof ESQLCallbacks, ErrorTypes[]> = {
+  getFieldsFor: ['unknownColumn', 'wrongArgumentType', 'unsupportedFieldType'],
+  getSources: ['unknownIndex'],
+  getPolicies: ['unknownPolicy'],
+  getMetaFields: ['unknownMetadataField'],
+};
+
+/**
+ * ES|QL validation public API
+ * It takes a query string and returns a list of messages (errors and warnings) after validate
+ * The astProvider is optional, but if not provided the default one from '@kbn/esql-validation-autocomplete' will be used.
+ * This is useful for async loading the ES|QL parser and reduce the bundle size, or to swap grammar version.
+ * As for the callbacks, while optional, the validation function will selectively ignore some errors types based on each callback missing.
+ */
+export async function validateQuery(
+  queryString: string,
+  astProvider: AstProviderFn,
+  options: ValidationOptions = {},
+  callbacks?: ESQLCallbacks
+): Promise<ValidationResult> {
+  const result = await validateAst(queryString, astProvider, callbacks);
+  // early return if we do not want to ignore errors
+  if (!options.ignoreOnMissingCallbacks) {
+    return result;
+  }
+  const { errors, warnings } = result;
+  const finalCallbacks = callbacks || {};
+  const errorTypoesToIgnore = Object.entries(ignoreErrorsMap).reduce((acc, [key, errorCodes]) => {
+    if (
+      !(key in finalCallbacks) ||
+      (key in finalCallbacks && finalCallbacks[key as keyof ESQLCallbacks] == null)
+    ) {
+      for (const e of errorCodes) {
+        acc[e] = true;
+      }
+    }
+    return acc;
+  }, {} as Partial<Record<ErrorTypes, boolean>>);
+  const filteredErrors = errors
+    .filter((error) => {
+      if ('severity' in error) {
+        return true;
+      }
+      return !errorTypoesToIgnore[error.code as ErrorTypes];
+    })
+    .map((error) =>
+      'severity' in error
+        ? {
+            text: error.message,
+            code: error.code!,
+            type: 'error' as const,
+            location: { min: error.startColumn, max: error.endColumn },
+          }
+        : error
+    );
+  return { errors: filteredErrors, warnings };
+}
+
 /**
  * This function will perform an high level validation of the
  * query AST. An initial syntax validation is already performed by the parser
  * while here it can detect things like function names, types correctness and potential warnings
  * @param ast A valid AST data structure
  */
-export async function validateAst(
+async function validateAst(
   queryString: string,
   astProvider: AstProviderFn,
   callbacks?: ESQLCallbacks

packages/kbn-esql-validation-autocomplete/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*",
+    "**/*.ts",
+  ],
+  "kbn_references": [
+    "@kbn/i18n",
+    "@kbn/esql-ast",
+  ],
+  "exclude": [
+    "target/**/*",
+  ]
+}

packages/kbn-esql/src/antlr/.antlr/esql_lexer.tokens

@@ -1,98 +0,0 @@
-DISSECT=1
-GROK=2
-EVAL=3
-EXPLAIN=4
-FROM=5
-ROW=6
-STATS=7
-WHERE=8
-SORT=9
-MV_EXPAND=10
-LIMIT=11
-PROJECT=12
-DROP=13
-RENAME=14
-SHOW=15
-ENRICH=16
-KEEP=17
-LINE_COMMENT=18
-MULTILINE_COMMENT=19
-WS=20
-EXPLAIN_WS=21
-EXPLAIN_LINE_COMMENT=22
-EXPLAIN_MULTILINE_COMMENT=23
-PIPE=24
-STRING=25
-INTEGER_LITERAL=26
-DECIMAL_LITERAL=27
-BY=28
-DATE_LITERAL=29
-AND=30
-ASSIGN=31
-COMMA=32
-DOT=33
-LP=34
-OPENING_BRACKET=35
-CLOSING_BRACKET=36
-NOT=37
-LIKE=38
-RLIKE=39
-IN=40
-IS=41
-AS=42
-NULL=43
-OR=44
-RP=45
-UNDERSCORE=46
-INFO=47
-FUNCTIONS=48
-BOOLEAN_VALUE=49
-COMPARISON_OPERATOR=50
-PLUS=51
-MINUS=52
-ASTERISK=53
-SLASH=54
-PERCENT=55
-TEN=56
-ORDERING=57
-NULLS_ORDERING=58
-NULLS_ORDERING_DIRECTION=59
-MATH_FUNCTION=60
-UNARY_FUNCTION=61
-WHERE_FUNCTIONS=62
-UNQUOTED_IDENTIFIER=63
-QUOTED_IDENTIFIER=64
-EXPR_LINE_COMMENT=65
-EXPR_MULTILINE_COMMENT=66
-EXPR_WS=67
-METADATA=68
-SRC_UNQUOTED_IDENTIFIER=69
-SRC_QUOTED_IDENTIFIER=70
-SRC_LINE_COMMENT=71
-SRC_MULTILINE_COMMENT=72
-SRC_WS=73
-ON=74
-WITH=75
-ENR_UNQUOTED_IDENTIFIER=76
-ENR_QUOTED_IDENTIFIER=77
-ENR_LINE_COMMENT=78
-ENR_MULTILINE_COMMENT=79
-ENR_WS=80
-EXPLAIN_PIPE=81
-'by'=28
-'and'=30
-'.'=33
-'('=34
-']'=36
-'or'=44
-')'=45
-'_'=46
-'info'=47
-'functions'=48
-'+'=51
-'-'=52
-'*'=53
-'/'=54
-'%'=55
-'10'=56
-'nulls'=58

packages/kbn-esql/src/antlr/.antlr/esql_parser.tokens

@@ -1,117 +0,0 @@
-DISSECT=1
-DROP=2
-ENRICH=3
-EVAL=4
-EXPLAIN=5
-FROM=6
-GROK=7
-INLINESTATS=8
-KEEP=9
-LIMIT=10
-MV_EXPAND=11
-PROJECT=12
-RENAME=13
-ROW=14
-SHOW=15
-SORT=16
-STATS=17
-WHERE=18
-UNKNOWN_CMD=19
-LINE_COMMENT=20
-MULTILINE_COMMENT=21
-WS=22
-EXPLAIN_WS=23
-EXPLAIN_LINE_COMMENT=24
-EXPLAIN_MULTILINE_COMMENT=25
-PIPE=26
-STRING=27
-INTEGER_LITERAL=28
-DECIMAL_LITERAL=29
-BY=30
-AND=31
-ASC=32
-ASSIGN=33
-COMMA=34
-DESC=35
-DOT=36
-FALSE=37
-FIRST=38
-LAST=39
-LP=40
-IN=41
-IS=42
-LIKE=43
-NOT=44
-NULL=45
-NULLS=46
-OR=47
-PARAM=48
-RLIKE=49
-RP=50
-TRUE=51
-EQ=52
-NEQ=53
-LT=54
-LTE=55
-GT=56
-GTE=57
-PLUS=58
-MINUS=59
-ASTERISK=60
-SLASH=61
-PERCENT=62
-OPENING_BRACKET=63
-CLOSING_BRACKET=64
-UNQUOTED_IDENTIFIER=65
-QUOTED_IDENTIFIER=66
-EXPR_LINE_COMMENT=67
-EXPR_MULTILINE_COMMENT=68
-EXPR_WS=69
-METADATA=70
-FROM_UNQUOTED_IDENTIFIER=71
-FROM_LINE_COMMENT=72
-FROM_MULTILINE_COMMENT=73
-FROM_WS=74
-PROJECT_UNQUOTED_IDENTIFIER=75
-PROJECT_LINE_COMMENT=76
-PROJECT_MULTILINE_COMMENT=77
-PROJECT_WS=78
-AS=79
-RENAME_LINE_COMMENT=80
-RENAME_MULTILINE_COMMENT=81
-RENAME_WS=82
-ON=83
-WITH=84
-ENRICH_LINE_COMMENT=85
-ENRICH_MULTILINE_COMMENT=86
-ENRICH_WS=87
-ENRICH_FIELD_LINE_COMMENT=88
-ENRICH_FIELD_MULTILINE_COMMENT=89
-ENRICH_FIELD_WS=90
-MVEXPAND_LINE_COMMENT=91
-MVEXPAND_MULTILINE_COMMENT=92
-MVEXPAND_WS=93
-INFO=94
-FUNCTIONS=95
-SHOW_LINE_COMMENT=96
-SHOW_MULTILINE_COMMENT=97
-SHOW_WS=98
-'|'=26
-'='=33
-','=34
-'.'=36
-'('=40
-'?'=48
-')'=50
-'=='=52
-'!='=53
-'<'=54
-'<='=55
-'>'=56
-'>='=57
-'+'=58
-'-'=59
-'*'=60
-'/'=61
-'%'=62
-']'=64

packages/kbn-monaco/BUILD.bazel

@@ -26,10 +26,11 @@ SRCS = glob(
 SHARED_DEPS = [
   "//packages/kbn-i18n",
   "//packages/kbn-ui-theme",
+  "//packages/kbn-esql-validation-autocomplete",
+  "//packages/kbn-esql-ast",
   "@npm//antlr4",
   "@npm//monaco-editor",
   "@npm//monaco-yaml",
-  "@npm//js-levenshtein",
 ]
 
 webpack_cli(

packages/kbn-monaco/index.ts

@@ -20,7 +20,7 @@ export {
 export { XJsonLang } from './src/xjson';
 export { SQLLang } from './src/sql';
 export { ESQL_LANG_ID, ESQL_THEME_ID, ESQLLang } from './src/esql';
-export type { ESQLCallbacks } from './src/esql';
+export type { ESQLCallbacks } from '@kbn/esql-validation-autocomplete';
 
 export * from './src/painless';
 /* eslint-disable-next-line @kbn/eslint/module_migration */

packages/kbn-monaco/package.json

@@ -5,7 +5,6 @@
   "license": "SSPL-1.0 OR Elastic License 2.0",
   "scripts": {
     "build:antlr4:painless": "antlr -Dlanguage=TypeScript ./src/painless/antlr/painless_lexer.g4 ./src/painless/antlr/painless_parser.g4 && node ./scripts/fix_generated_antlr.js painless",
-    "build:antlr4:esql": "antlr -Dlanguage=TypeScript src/esql/antlr/esql_lexer.g4 src/esql/antlr/esql_parser.g4 && node ./scripts/fix_generated_antlr.js esql && node ./scripts/esql_update_ast_script.js",
     "prebuild:antlr4": "brew bundle --file=./scripts/antlr4_tools/brewfile",
     "build:antlr4": "yarn run build:antlr4:painless && npm run build:antlr4:esql"
   }

packages/kbn-monaco/src/common/diagnostics_adapter.ts

@@ -9,9 +9,14 @@
 import { BehaviorSubject } from 'rxjs';
 
 import { monaco } from '../monaco_imports';
-import type { SyntaxErrors, LangValidation, EditorError, BaseWorkerDefinition } from '../types';
+import type {
+  SyntaxErrors,
+  LangValidation,
+  MonacoEditorError,
+  BaseWorkerDefinition,
+} from '../types';
 
-const toDiagnostics = (error: EditorError): monaco.editor.IMarkerData => {
+const toDiagnostics = (error: MonacoEditorError): monaco.editor.IMarkerData => {
   return {
     ...error,
     severity: monaco.MarkerSeverity.Error,

packages/kbn-monaco/src/common/error_listener.ts

@@ -8,10 +8,10 @@
 
 import type { Recognizer, RecognitionException } from 'antlr4';
 import { ErrorListener } from 'antlr4';
-import type { EditorError } from '../types';
+import type { MonacoEditorError } from '../types';
 
 export class ANTLRErrorListener extends ErrorListener<any> {
-  protected errors: EditorError[] = [];
+  protected errors: MonacoEditorError[] = [];
 
   syntaxError(
     recognizer: Recognizer<any>,
@@ -37,7 +37,7 @@ export class ANTLRErrorListener extends ErrorListener<any> {
     });
   }
 
-  getErrors(): EditorError[] {
+  getErrors(): MonacoEditorError[] {
     return this.errors;
   }
 }

packages/kbn-monaco/src/esql/index.ts

@@ -8,5 +8,4 @@
 
 export { ESQL_LANG_ID, ESQL_THEME_ID } from './lib/constants';
 export { ESQLLang } from './language';
-export type { ESQLCallbacks } from './lib/ast/shared/types';
-export { buildESQlTheme } from './lib/monaco/esql_theme';
+export { buildESQlTheme } from './lib/esql_theme';

packages/kbn-monaco/src/esql/language.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import type { ESQLCallbacks } from '@kbn/esql-validation-autocomplete';
 import { monaco } from '../monaco_imports';
 
 import { ESQL_LANG_ID } from './lib/constants';
@@ -14,15 +15,16 @@ import type { CustomLangModuleType } from '../types';
 import type { ESQLWorker } from './worker/esql_worker';
 
 import { WorkerProxyService } from '../common/worker_proxy';
-import type { ESQLCallbacks } from './lib/ast/shared/types';
-import { ESQLAstAdapter } from './lib/monaco/esql_ast_provider';
+import { ESQLAstAdapter } from './lib/esql_ast_provider';
+import { wrapAsMonacoSuggestions } from './lib/converters/suggestions';
+import { wrapAsMonacoCodeActions } from './lib/converters/actions';
 
 const workerProxyService = new WorkerProxyService<ESQLWorker>();
 
 export const ESQLLang: CustomLangModuleType<ESQLCallbacks> = {
   ID: ESQL_LANG_ID,
   async onLanguage() {
-    const { ESQLTokensProvider } = await import('./lib/monaco');
+    const { ESQLTokensProvider } = await import('./lib');
 
     workerProxyService.setup(ESQL_LANG_ID);
 
@@ -98,10 +100,7 @@ export const ESQLLang: CustomLangModuleType<ESQLCallbacks> = {
         );
         const suggestionEntries = await astAdapter.autocomplete(model, position, context);
         return {
-          suggestions: suggestionEntries.suggestions.map((suggestion) => ({
-            ...suggestion,
-            range: undefined as unknown as monaco.IRange,
-          })),
+          suggestions: wrapAsMonacoSuggestions(suggestionEntries.suggestions),
         };
       },
     };
@@ -121,7 +120,7 @@ export const ESQLLang: CustomLangModuleType<ESQLCallbacks> = {
         );
         const actions = await astAdapter.codeAction(model, range, context);
         return {
-          actions,
+          actions: wrapAsMonacoCodeActions(model, actions),
           dispose: () => {},
         };
       },