[Console] Replace global GET /_mapping request with GET <index>/_mapping (#147770)

## Summary

### Notes for reviewers 

- Currently, autocomplete suggestions for fields don't work with
wildcards and data streams due to the
[bug](https://github.com/elastic/kibana/issues/149496) in the `main`. It
should be addressed separately.

### How to test

In order to spot the loading behaviour, ideally you should create an
index with a heavy mappings definition.
Afterwards, write a query that requires a field from this index, e.g.:

```
GET <my-index>/_search
{
  "aggs": {
    "my_agg": {
      "terms": {
        "field": "",
        "size": 10
      }
    }
  }
}
```

Place a cursor next to the `field` property, it should trigger mappings
fetch. After that, the mappings definition for this index will be cached
and accessed synchronously.

You can also open the browser's dev tools and enable Network throttling.
It allows noticing loading behaviour for any index.

--------------------

Resolves https://github.com/elastic/kibana/issues/146855

Instead of fetching all mappings upfront, requests mapping definition on
demand per index according to the cursor position.

Considering there is a maximum response size limit of 10MB in the
`/autocomplete_entities` endpoint, field autocompletion wasn't working
at all if the overall mappings definition exceeded this size. Retrieving
mappings per index tackles this and improves the init time.

![Jan-25-2023
17-16-31](https://user-images.githubusercontent.com/5236598/214616790-4954d005-e56f-49f9-be6d-435c076270a8.gif)

### Checklist

- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

---------

Co-authored-by: Jean-Louis Leysens <jloleysens@gmail.com>

src/plugins/console/public/application/containers/editor/editor.tsx

@@ -6,14 +6,14 @@
  * Side Public License, v 1.
  */
 
-import React, { useCallback, memo } from 'react';
+import React, { useCallback, memo, useEffect, useState } from 'react';
 import { debounce } from 'lodash';
 import { EuiProgress } from '@elastic/eui';
 
 import { EditorContentSpinner } from '../../components';
 import { Panel, PanelsContainer } from '..';
 import { Editor as EditorUI, EditorOutput } from './legacy/console_editor';
-import { StorageKeys } from '../../../services';
+import { getAutocompleteInfo, StorageKeys } from '../../../services';
 import { useEditorReadContext, useServicesContext, useRequestReadContext } from '../../contexts';
 import type { SenseEditor } from '../../models';
 
@@ -33,6 +33,15 @@ export const Editor = memo(({ loading, setEditorInstance }: Props) => {
   const { currentTextObject } = useEditorReadContext();
   const { requestInFlight } = useRequestReadContext();
 
+  const [fetchingMappings, setFetchingMappings] = useState(false);
+
+  useEffect(() => {
+    const subscription = getAutocompleteInfo().mapping.isLoading$.subscribe(setFetchingMappings);
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, []);
+
   const [firstPanelWidth, secondPanelWidth] = storage.get(StorageKeys.WIDTH, [
     INITIAL_PANEL_WIDTH,
     INITIAL_PANEL_WIDTH,
@@ -50,7 +59,7 @@ export const Editor = memo(({ loading, setEditorInstance }: Props) => {
 
   return (
     <>
-      {requestInFlight ? (
+      {requestInFlight || fetchingMappings ? (
         <div className="conApp__requestProgressBarContainer">
           <EuiProgress size="xs" color="accent" position="absolute" />
         </div>

src/plugins/console/public/application/index.tsx

@@ -69,6 +69,8 @@ export function renderApp({
   const api = createApi({ http });
   const esHostService = createEsHostService({ api });
 
+  autocompleteInfo.mapping.setup(http, settings);
+
   render(
     <I18nContext>
       <KibanaThemeProvider theme$={theme$}>

src/plugins/console/public/application/models/legacy_core_editor/legacy_core_editor.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import ace from 'brace';
+import ace, { type Annotation } from 'brace';
 import { Editor as IAceEditor, IEditSession as IAceEditSession } from 'brace';
 import $ from 'jquery';
 import {
@@ -402,8 +402,7 @@ export class LegacyCoreEditor implements CoreEditor {
         getCompletions: (
           // eslint-disable-next-line @typescript-eslint/naming-convention
           DO_NOT_USE_1: IAceEditor,
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          DO_NOT_USE_2: IAceEditSession,
+          aceEditSession: IAceEditSession,
           pos: { row: number; column: number },
           prefix: string,
           callback: (...args: unknown[]) => void
@@ -412,7 +411,30 @@ export class LegacyCoreEditor implements CoreEditor {
             lineNumber: pos.row + 1,
             column: pos.column + 1,
           };
-          autocompleter(position, prefix, callback);
+
+          const getAnnotationControls = () => {
+            let customAnnotation: Annotation;
+            return {
+              setAnnotation(text: string) {
+                const annotations = aceEditSession.getAnnotations();
+                customAnnotation = {
+                  text,
+                  row: pos.row,
+                  column: pos.column,
+                  type: 'warning',
+                };
+
+                aceEditSession.setAnnotations([...annotations, customAnnotation]);
+              },
+              removeAnnotation() {
+                aceEditSession.setAnnotations(
+                  aceEditSession.getAnnotations().filter((a: Annotation) => a !== customAnnotation)
+                );
+              },
+            };
+          };
+
+          autocompleter(position, prefix, callback, getAnnotationControls());
         },
       },
     ]);

src/plugins/console/public/application/models/sense_editor/integration.test.js

@@ -13,6 +13,9 @@ import $ from 'jquery';
 
 import * as kb from '../../../lib/kb/kb';
 import { AutocompleteInfo, setAutocompleteInfo } from '../../../services';
+import { httpServiceMock } from '@kbn/core-http-browser-mocks';
+import { StorageMock } from '../../../services/storage.mock';
+import { SettingsMock } from '../../../services/settings.mock';
 
 describe('Integration', () => {
   let senseEditor;
@@ -27,6 +30,15 @@ describe('Integration', () => {
     $(senseEditor.getCoreEditor().getContainer()).show();
     senseEditor.autocomplete._test.removeChangeListener();
     autocompleteInfo = new AutocompleteInfo();
+
+    const httpMock = httpServiceMock.createSetupContract();
+    const storage = new StorageMock({}, 'test');
+    const settingsMock = new SettingsMock(storage);
+
+    settingsMock.getAutocomplete.mockReturnValue({ fields: true });
+
+    autocompleteInfo.mapping.setup(httpMock, settingsMock);
+
     setAutocompleteInfo(autocompleteInfo);
   });
   afterEach(() => {
@@ -164,7 +176,8 @@ describe('Integration', () => {
             ac('textBoxPosition', posCompare);
             ac('rangeToReplace', rangeCompare);
             done();
-          }
+          },
+          { setAnnotation: () => {}, removeAnnotation: () => {} }
         );
       });
     });

src/plugins/console/public/lib/autocomplete/autocomplete.ts

@@ -593,7 +593,10 @@ export default function ({
         return null;
     }
 
-    if (!context.autoCompleteSet) {
+    const isMappingsFetchingInProgress =
+      context.autoCompleteType === 'body' && !!context.asyncResultsState?.isLoading;
+
+    if (!context.autoCompleteSet && !isMappingsFetchingInProgress) {
       return null; // nothing to do..
     }
 
@@ -1123,80 +1126,112 @@ export default function ({
     }
   }
 
+  /**
+   * Extracts terms from the autocomplete set.
+   * @param context
+   */
+  function getTerms(context: AutoCompleteContext, autoCompleteSet: ResultTerm[]) {
+    const terms = _.map(
+      autoCompleteSet.filter((term) => Boolean(term) && term.name != null),
+      function (term) {
+        if (typeof term !== 'object') {
+          term = {
+            name: term,
+          };
+        } else {
+          term = _.clone(term);
+        }
+        const defaults: {
+          value?: string;
+          meta: string;
+          score: number;
+          context: AutoCompleteContext;
+          completer?: { insertMatch: (v: unknown) => void };
+        } = {
+          value: term.name,
+          meta: 'API',
+          score: 0,
+          context,
+        };
+        // we only need our custom insertMatch behavior for the body
+        if (context.autoCompleteType === 'body') {
+          defaults.completer = {
+            insertMatch() {
+              return applyTerm(term);
+            },
+          };
+        }
+        return _.defaults(term, defaults);
+      }
+    );
+
+    terms.sort(function (
+      t1: { score: number; name?: string },
+      t2: { score: number; name?: string }
+    ) {
+      /* score sorts from high to low */
+      if (t1.score > t2.score) {
+        return -1;
+      }
+      if (t1.score < t2.score) {
+        return 1;
+      }
+      /* names sort from low to high */
+      if (t1.name! < t2.name!) {
+        return -1;
+      }
+      if (t1.name === t2.name) {
+        return 0;
+      }
+      return 1;
+    });
+
+    return terms;
+  }
+
+  function getSuggestions(terms: ResultTerm[]) {
+    return _.map(terms, function (t, i) {
+      t.insertValue = t.insertValue || t.value;
+      t.value = '' + t.value; // normalize to strings
+      t.score = -i;
+      return t;
+    });
+  }
+
   function getCompletions(
     position: Position,
     prefix: string,
-    callback: (e: Error | null, result: ResultTerm[] | null) => void
+    callback: (e: Error | null, result: ResultTerm[] | null) => void,
+    annotationControls: {
+      setAnnotation: (text: string) => void;
+      removeAnnotation: () => void;
+    }
   ) {
     try {
       const context = getAutoCompleteContext(editor, position);
+
       if (!context) {
         callback(null, []);
       } else {
-        const terms = _.map(
-          context.autoCompleteSet!.filter((term) => Boolean(term) && term.name != null),
-          function (term) {
-            if (typeof term !== 'object') {
-              term = {
-                name: term,
-              };
-            } else {
-              term = _.clone(term);
-            }
-            const defaults: {
-              value?: string;
-              meta: string;
-              score: number;
-              context: AutoCompleteContext;
-              completer?: { insertMatch: (v: unknown) => void };
-            } = {
-              value: term.name,
-              meta: 'API',
-              score: 0,
-              context,
-            };
-            // we only need our custom insertMatch behavior for the body
-            if (context.autoCompleteType === 'body') {
-              defaults.completer = {
-                insertMatch() {
-                  return applyTerm(term);
-                },
-              };
-            }
-            return _.defaults(term, defaults);
-          }
-        );
+        if (!context.asyncResultsState?.isLoading) {
+          const terms = getTerms(context, context.autoCompleteSet!);
+          const suggestions = getSuggestions(terms);
+          callback(null, suggestions);
+        }
 
-        terms.sort(function (
-          t1: { score: number; name?: string },
-          t2: { score: number; name?: string }
-        ) {
-          /* score sorts from high to low */
-          if (t1.score > t2.score) {
-            return -1;
-          }
-          if (t1.score < t2.score) {
-            return 1;
-          }
-          /* names sort from low to high */
-          if (t1.name! < t2.name!) {
-            return -1;
-          }
-          if (t1.name === t2.name) {
-            return 0;
-          }
-          return 1;
-        });
+        if (context.asyncResultsState) {
+          annotationControls.setAnnotation(
+            i18n.translate('console.autocomplete.fieldsFetchingAnnotation', {
+              defaultMessage: 'Fields fetching is in progress',
+            })
+          );
 
-        callback(
-          null,
-          _.map(terms, function (t, i) {
-            t.insertValue = t.insertValue || t.value;
-            t.value = '' + t.value; // normalize to strings
-            t.score = -i;
-            return t;
-          })
-        );
+          context.asyncResultsState.results.then((r) => {
+            const asyncSuggestions = getSuggestions(getTerms(context, r));
+            callback(null, asyncSuggestions);
+            annotationControls.removeAnnotation();
+          });
+        }
       }
     } catch (e) {
       // eslint-disable-next-line no-console
@@ -1216,8 +1251,12 @@ export default function ({
         _editSession: unknown,
         pos: Position,
         prefix: string,
-        callback: (e: Error | null, result: ResultTerm[] | null) => void
-      ) => getCompletions(pos, prefix, callback),
+        callback: (e: Error | null, result: ResultTerm[] | null) => void,
+        annotationControls: {
+          setAnnotation: (text: string) => void;
+          removeAnnotation: () => void;
+        }
+      ) => getCompletions(pos, prefix, callback, annotationControls),
       addReplacementInfoToContext,
       addChangeListener: () => editor.on('changeSelection', editorChangeListener),
       removeChangeListener: () => editor.off('changeSelection', editorChangeListener),

src/plugins/console/public/lib/autocomplete/types.ts

@@ -13,6 +13,7 @@ export interface ResultTerm {
   insertValue?: string;
   name?: string;
   value?: string;
+  score?: number;
 }
 
 export interface DataAutoCompleteRulesOneOf {
@@ -25,6 +26,14 @@ export interface DataAutoCompleteRulesOneOf {
 
 export interface AutoCompleteContext {
   autoCompleteSet?: null | ResultTerm[];
+  /**
+   * Stores a state for async results, e.g. fields suggestions based on the mappings definition.
+   */
+  asyncResultsState?: {
+    isLoading: boolean;
+    lastFetched: number | null;
+    results: Promise<ResultTerm[]>;
+  };
   endpoint?: null | {
     paramsAutocomplete: {
       getTopLevelComponents: (method?: string | null) => unknown;

src/plugins/console/public/lib/autocomplete_entities/autocomplete_entities.test.js

@@ -9,6 +9,9 @@
 import '../../application/models/sense_editor/sense_editor.test.mocks';
 import { setAutocompleteInfo, AutocompleteInfo } from '../../services';
 import { expandAliases } from './expand_aliases';
+import { httpServiceMock } from '@kbn/core-http-browser-mocks';
+import { SettingsMock } from '../../services/settings.mock';
+import { StorageMock } from '../../services/storage.mock';
 
 function fc(f1, f2) {
   if (f1.name < f2.name) {
@@ -32,10 +35,20 @@ describe('Autocomplete entities', () => {
   let componentTemplate;
   let dataStream;
   let autocompleteInfo;
+  let settingsMock;
+  let httpMock;
+
   beforeEach(() => {
     autocompleteInfo = new AutocompleteInfo();
     setAutocompleteInfo(autocompleteInfo);
     mapping = autocompleteInfo.mapping;
+
+    httpMock = httpServiceMock.createSetupContract();
+    const storage = new StorageMock({}, 'test');
+    settingsMock = new SettingsMock(storage);
+
+    mapping.setup(httpMock, settingsMock);
+
     alias = autocompleteInfo.alias;
     legacyTemplate = autocompleteInfo.legacyTemplate;
     indexTemplate = autocompleteInfo.indexTemplate;
@@ -48,61 +61,98 @@ describe('Autocomplete entities', () => {
   });
 
   describe('Mappings', function () {
-    test('Multi fields 1.0 style', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            first_name: {
-              type: 'string',
-              index: 'analyzed',
-              path: 'just_name',
-              fields: {
-                any_name: { type: 'string', index: 'analyzed' },
-              },
-            },
-            last_name: {
-              type: 'string',
-              index: 'no',
-              fields: {
-                raw: { type: 'string', index: 'analyzed' },
+    describe('When fields autocomplete is disabled', () => {
+      beforeEach(() => {
+        settingsMock.getAutocomplete.mockReturnValue({ fields: false });
+      });
+
+      test('does not return any suggestions', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              first_name: {
+                type: 'string',
+                index: 'analyzed',
+                path: 'just_name',
+                fields: {
+                  any_name: { type: 'string', index: 'analyzed' },
+                },
               },
             },
           },
-        },
-      });
+        });
 
-      expect(mapping.getMappings('index').sort(fc)).toEqual([
-        f('any_name', 'string'),
-        f('first_name', 'string'),
-        f('last_name', 'string'),
-        f('last_name.raw', 'string'),
-      ]);
+        expect(mapping.getMappings('index').sort(fc)).toEqual([]);
+      });
     });
 
-    test('Simple fields', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            str: {
-              type: 'string',
-            },
-            number: {
-              type: 'int',
+    describe('When fields autocomplete is enabled', () => {
+      beforeEach(() => {
+        settingsMock.getAutocomplete.mockReturnValue({ fields: true });
+        httpMock.get.mockReturnValue(
+          Promise.resolve({
+            mappings: { index: { mappings: { properties: { '@timestamp': { type: 'date' } } } } },
+          })
+        );
+      });
+
+      test('attempts to fetch mappings if not loaded', async () => {
+        const autoCompleteContext = {};
+        let loadingIndicator;
+
+        mapping.isLoading$.subscribe((v) => {
+          loadingIndicator = v;
+        });
+
+        // act
+        mapping.getMappings('index', [], autoCompleteContext);
+
+        expect(autoCompleteContext.asyncResultsState.isLoading).toBe(true);
+        expect(loadingIndicator).toBe(true);
+
+        expect(httpMock.get).toHaveBeenCalled();
+
+        const fields = await autoCompleteContext.asyncResultsState.results;
+
+        expect(loadingIndicator).toBe(false);
+        expect(autoCompleteContext.asyncResultsState.isLoading).toBe(false);
+        expect(fields).toEqual([{ name: '@timestamp', type: 'date' }]);
+      });
+
+      test('Multi fields 1.0 style', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              first_name: {
+                type: 'string',
+                index: 'analyzed',
+                path: 'just_name',
+                fields: {
+                  any_name: { type: 'string', index: 'analyzed' },
+                },
+              },
+              last_name: {
+                type: 'string',
+                index: 'no',
+                fields: {
+                  raw: { type: 'string', index: 'analyzed' },
+                },
+              },
             },
           },
-        },
+        });
+
+        expect(mapping.getMappings('index').sort(fc)).toEqual([
+          f('any_name', 'string'),
+          f('first_name', 'string'),
+          f('last_name', 'string'),
+          f('last_name.raw', 'string'),
+        ]);
       });
 
-      expect(mapping.getMappings('index').sort(fc)).toEqual([
-        f('number', 'int'),
-        f('str', 'string'),
-      ]);
-    });
-
-    test('Simple fields - 1.0 style', function () {
-      mapping.loadMappings({
-        index: {
-          mappings: {
+      test('Simple fields', function () {
+        mapping.loadMappings({
+          index: {
             properties: {
               str: {
                 type: 'string',
@@ -112,108 +162,130 @@ describe('Autocomplete entities', () => {
               },
             },
           },
-        },
+        });
+
+        expect(mapping.getMappings('index').sort(fc)).toEqual([
+          f('number', 'int'),
+          f('str', 'string'),
+        ]);
       });
 
-      expect(mapping.getMappings('index').sort(fc)).toEqual([
-        f('number', 'int'),
-        f('str', 'string'),
-      ]);
-    });
-
-    test('Nested fields', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            person: {
-              type: 'object',
+      test('Simple fields - 1.0 style', function () {
+        mapping.loadMappings({
+          index: {
+            mappings: {
               properties: {
-                name: {
-                  properties: {
-                    first_name: { type: 'string' },
-                    last_name: { type: 'string' },
+                str: {
+                  type: 'string',
+                },
+                number: {
+                  type: 'int',
+                },
+              },
+            },
+          },
+        });
+
+        expect(mapping.getMappings('index').sort(fc)).toEqual([
+          f('number', 'int'),
+          f('str', 'string'),
+        ]);
+      });
+
+      test('Nested fields', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              person: {
+                type: 'object',
+                properties: {
+                  name: {
+                    properties: {
+                      first_name: { type: 'string' },
+                      last_name: { type: 'string' },
+                    },
                   },
+                  sid: { type: 'string', index: 'not_analyzed' },
                 },
-                sid: { type: 'string', index: 'not_analyzed' },
               },
+              message: { type: 'string' },
             },
-            message: { type: 'string' },
           },
-        },
+        });
+
+        expect(mapping.getMappings('index', []).sort(fc)).toEqual([
+          f('message'),
+          f('person.name.first_name'),
+          f('person.name.last_name'),
+          f('person.sid'),
+        ]);
       });
 
-      expect(mapping.getMappings('index', []).sort(fc)).toEqual([
-        f('message'),
-        f('person.name.first_name'),
-        f('person.name.last_name'),
-        f('person.sid'),
-      ]);
-    });
-
-    test('Enabled fields', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            person: {
-              type: 'object',
-              properties: {
-                name: {
-                  type: 'object',
-                  enabled: false,
+      test('Enabled fields', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              person: {
+                type: 'object',
+                properties: {
+                  name: {
+                    type: 'object',
+                    enabled: false,
+                  },
+                  sid: { type: 'string', index: 'not_analyzed' },
                 },
-                sid: { type: 'string', index: 'not_analyzed' },
               },
+              message: { type: 'string' },
             },
-            message: { type: 'string' },
           },
-        },
+        });
+
+        expect(mapping.getMappings('index', []).sort(fc)).toEqual([f('message'), f('person.sid')]);
       });
 
-      expect(mapping.getMappings('index', []).sort(fc)).toEqual([f('message'), f('person.sid')]);
-    });
-
-    test('Path tests', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            name1: {
-              type: 'object',
-              path: 'just_name',
-              properties: {
-                first1: { type: 'string' },
-                last1: { type: 'string', index_name: 'i_last_1' },
+      test('Path tests', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              name1: {
+                type: 'object',
+                path: 'just_name',
+                properties: {
+                  first1: { type: 'string' },
+                  last1: { type: 'string', index_name: 'i_last_1' },
+                },
               },
-            },
-            name2: {
-              type: 'object',
-              path: 'full',
-              properties: {
-                first2: { type: 'string' },
-                last2: { type: 'string', index_name: 'i_last_2' },
+              name2: {
+                type: 'object',
+                path: 'full',
+                properties: {
+                  first2: { type: 'string' },
+                  last2: { type: 'string', index_name: 'i_last_2' },
+                },
               },
             },
           },
-        },
+        });
+
+        expect(mapping.getMappings().sort(fc)).toEqual([
+          f('first1'),
+          f('i_last_1'),
+          f('name2.first2'),
+          f('name2.i_last_2'),
+        ]);
       });
 
-      expect(mapping.getMappings().sort(fc)).toEqual([
-        f('first1'),
-        f('i_last_1'),
-        f('name2.first2'),
-        f('name2.i_last_2'),
-      ]);
-    });
-
-    test('Use index_name tests', function () {
-      mapping.loadMappings({
-        index: {
-          properties: {
-            last1: { type: 'string', index_name: 'i_last_1' },
+      test('Use index_name tests', function () {
+        mapping.loadMappings({
+          index: {
+            properties: {
+              last1: { type: 'string', index_name: 'i_last_1' },
+            },
           },
-        },
-      });
+        });
 
-      expect(mapping.getMappings().sort(fc)).toEqual([f('i_last_1')]);
+        expect(mapping.getMappings().sort(fc)).toEqual([f('i_last_1')]);
+      });
     });
   });
 

src/plugins/console/public/lib/autocomplete_entities/mapping.ts

@@ -7,9 +7,15 @@
  */
 
 import _ from 'lodash';
+import { BehaviorSubject } from 'rxjs';
 import type { IndicesGetMappingResponse } from '@elastic/elasticsearch/lib/api/types';
+import { HttpSetup } from '@kbn/core-http-browser';
+import { type Settings } from '../../services';
+import { API_BASE_PATH } from '../../../common/constants';
+import type { ResultTerm, AutoCompleteContext } from '../autocomplete/types';
 import { expandAliases } from './expand_aliases';
 import type { Field, FieldMapping } from './types';
+import { type AutoCompleteEntitiesApiResponse } from './types';
 
 function getFieldNamesFromProperties(properties: Record<string, FieldMapping> = {}) {
   const fieldList = Object.entries(properties).flatMap(([fieldName, fieldMapping]) => {
@@ -70,22 +76,127 @@ function getFieldNamesFromFieldMapping(
 
 export interface BaseMapping {
   perIndexTypes: Record<string, object>;
-  getMappings(indices: string | string[], types?: string | string[]): Field[];
+  /**
+   * Fetches mappings definition
+   */
+  fetchMappings(index: string): Promise<IndicesGetMappingResponse>;
+
+  /**
+   * Retrieves mappings definition from cache, fetches if necessary.
+   */
+  getMappings(
+    indices: string | string[],
+    types?: string | string[],
+    autoCompleteContext?: AutoCompleteContext
+  ): Field[];
+
+  /**
+   * Stores mappings definition
+   * @param mappings
+   */
   loadMappings(mappings: IndicesGetMappingResponse): void;
   clearMappings(): void;
 }
 
 export class Mapping implements BaseMapping {
+  private http!: HttpSetup;
+
+  private settings!: Settings;
+
+  /**
+   * Map of the mappings of actual ES indices.
+   */
   public perIndexTypes: Record<string, object> = {};
 
-  getMappings = (indices: string | string[], types?: string | string[]) => {
+  private readonly _isLoading$ = new BehaviorSubject<boolean>(false);
+
+  /**
+   * Indicates if mapping fetching is in progress.
+   */
+  public readonly isLoading$ = this._isLoading$.asObservable();
+
+  /**
+   * Map of the currently loading mappings for index patterns specified by a user.
+   * @private
+   */
+  private loadingState: Record<string, boolean> = {};
+
+  public setup(http: HttpSetup, settings: Settings) {
+    this.http = http;
+    this.settings = settings;
+  }
+
+  /**
+   * Fetches mappings of the requested indices.
+   * @param index
+   */
+  async fetchMappings(index: string): Promise<IndicesGetMappingResponse> {
+    const response = await this.http.get<AutoCompleteEntitiesApiResponse>(
+      `${API_BASE_PATH}/autocomplete_entities`,
+      {
+        query: { fields: true, fieldsIndices: index },
+      }
+    );
+
+    return response.mappings;
+  }
+
+  getMappings = (
+    indices: string | string[],
+    types?: string | string[],
+    autoCompleteContext?: AutoCompleteContext
+  ) => {
     // get fields for indices and types. Both can be a list, a string or null (meaning all).
     let ret: Field[] = [];
+
+    if (!this.settings.getAutocomplete().fields) return ret;
+
     indices = expandAliases(indices);
 
     if (typeof indices === 'string') {
       const typeDict = this.perIndexTypes[indices] as Record<string, unknown>;
-      if (!typeDict) {
+
+      if (!typeDict || Object.keys(typeDict).length === 0) {
+        if (!autoCompleteContext) return ret;
+
+        // Mappings fetching for the index is already in progress
+        if (this.loadingState[indices]) return ret;
+
+        this.loadingState[indices] = true;
+
+        if (!autoCompleteContext.asyncResultsState) {
+          autoCompleteContext.asyncResultsState = {} as AutoCompleteContext['asyncResultsState'];
+        }
+
+        autoCompleteContext.asyncResultsState!.isLoading = true;
+
+        autoCompleteContext.asyncResultsState!.results = new Promise<ResultTerm[]>(
+          (resolve, reject) => {
+            this._isLoading$.next(true);
+
+            this.fetchMappings(indices as string)
+              .then((mapping) => {
+                this._isLoading$.next(false);
+
+                autoCompleteContext.asyncResultsState!.isLoading = false;
+                autoCompleteContext.asyncResultsState!.lastFetched = Date.now();
+
+                // cache mappings
+                this.loadMappings(mapping);
+
+                const mappings = this.getMappings(indices, types, autoCompleteContext);
+                delete this.loadingState[indices as string];
+                resolve(mappings);
+              })
+              .catch((error) => {
+                // eslint-disable-next-line no-console
+                console.error(error);
+                this._isLoading$.next(false);
+                delete this.loadingState[indices as string];
+              });
+          }
+        );
+
         return [];
       }
 
@@ -108,7 +219,7 @@ export class Mapping implements BaseMapping {
       // multi index mode.
       Object.keys(this.perIndexTypes).forEach((index) => {
         if (!indices || indices.length === 0 || indices.includes(index)) {
-          ret.push(this.getMappings(index, types) as unknown as Field);
+          ret.push(this.getMappings(index, types, autoCompleteContext) as unknown as Field);
         }
       });
 
@@ -121,8 +232,6 @@ export class Mapping implements BaseMapping {
   };
 
   loadMappings = (mappings: IndicesGetMappingResponse) => {
-    this.perIndexTypes = {};
-
     Object.entries(mappings).forEach(([index, indexMapping]) => {
       const normalizedIndexMappings: Record<string, object[]> = {};
       let transformedMapping: Record<string, any> = indexMapping;

src/plugins/console/public/services/autocomplete.ts

@@ -53,7 +53,11 @@ export class AutocompleteInfo {
         const collaborator = this.mapping;
         return () => this.alias.getIndices(includeAliases, collaborator);
       case ENTITIES.FIELDS:
-        return this.mapping.getMappings(context.indices, context.types);
+        return this.mapping.getMappings(
+          context.indices,
+          context.types,
+          Object.getPrototypeOf(context)
+        );
       case ENTITIES.INDEX_TEMPLATES:
         return () => this.indexTemplate.getTemplates();
       case ENTITIES.COMPONENT_TEMPLATES:
@@ -93,7 +97,6 @@ export class AutocompleteInfo {
   }
 
   private load(data: AutoCompleteEntitiesApiResponse) {
-    this.mapping.loadMappings(data.mappings);
     const collaborator = this.mapping;
     this.alias.loadAliases(data.aliases, collaborator);
     this.indexTemplate.loadTemplates(data.indexTemplates);

src/plugins/console/public/services/settings.ts

@@ -14,7 +14,12 @@ export const DEFAULT_SETTINGS = Object.freeze({
   pollInterval: 60000,
   tripleQuotes: true,
   wrapMode: true,
-  autocomplete: Object.freeze({ fields: true, indices: true, templates: true, dataStreams: true }),
+  autocomplete: Object.freeze({
+    fields: true,
+    indices: true,
+    templates: true,
+    dataStreams: true,
+  }),
   isHistoryEnabled: true,
   isKeyboardShortcutsEnabled: true,
 });

src/plugins/console/public/types/core_editor.ts

@@ -7,6 +7,7 @@
  */
 
 import type { Editor } from 'brace';
+import { ResultTerm } from '../lib/autocomplete/types';
 import { TokensProvider } from './tokens_provider';
 import { Token } from './token';
 
@@ -23,7 +24,11 @@ export type EditorEvent =
 export type AutoCompleterFunction = (
   pos: Position,
   prefix: string,
-  callback: (...args: unknown[]) => void
+  callback: (e: Error | null, result: ResultTerm[] | null) => void,
+  annotationControls: {
+    setAnnotation: (text: string) => void;
+    removeAnnotation: () => void;
+  }
 ) => void;
 
 export interface Position {

src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts

@@ -6,26 +6,25 @@
  * Side Public License, v 1.
  */
 
-import { parse } from 'query-string';
 import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 import type { RouteDependencies } from '../../..';
-interface SettingsToRetrieve {
-  indices: boolean;
-  fields: boolean;
-  templates: boolean;
-  dataStreams: boolean;
-}
+import { autoCompleteEntitiesValidationConfig, type SettingsToRetrieve } from './validation_config';
 
 const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB
 // Limit the response size to 10MB, because the response can be very large and sending it to the client
 // can cause the browser to hang.
 
 const getMappings = async (settings: SettingsToRetrieve, esClient: IScopedClusterClient) => {
-  if (settings.fields) {
-    const mappings = await esClient.asInternalUser.indices.getMapping(undefined, {
-      maxResponseSize: MAX_RESPONSE_SIZE,
-      maxCompressedResponseSize: MAX_RESPONSE_SIZE,
-    });
+  if (settings.fields && settings.fieldsIndices) {
+    const mappings = await esClient.asInternalUser.indices.getMapping(
+      {
+        index: settings.fieldsIndices,
+      },
+      {
+        maxResponseSize: MAX_RESPONSE_SIZE,
+        maxCompressedResponseSize: MAX_RESPONSE_SIZE,
+      }
+    );
     return mappings;
   }
   // If the user doesn't want autocomplete suggestions, then clear any that exist.
@@ -87,20 +86,11 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => {
       options: {
         tags: ['access:console'],
       },
-      validate: false,
+      validate: autoCompleteEntitiesValidationConfig,
     },
     async (context, request, response) => {
       const esClient = (await context.core).elasticsearch.client;
-      const settings = parse(request.url.search, {
-        parseBooleans: true,
-      }) as unknown as SettingsToRetrieve;
-
-      // If no settings are specified, then return 400.
-      if (Object.keys(settings).length === 0) {
-        return response.badRequest({
-          body: 'Request must contain at least one of the following parameters: indices, fields, templates, dataStreams',
-        });
-      }
+      const settings = request.query;
 
       // Wait for all requests to complete, in case one of them fails return the successfull ones
       const results = await Promise.allSettled([

src/plugins/console/server/routes/api/console/autocomplete_entities/validation_config.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const autoCompleteEntitiesValidationConfig = {
+  query: schema.object(
+    {
+      indices: schema.maybe(schema.boolean()),
+      fields: schema.maybe(schema.boolean()),
+      templates: schema.maybe(schema.boolean()),
+      dataStreams: schema.maybe(schema.boolean()),
+      /**
+       * Comma separated list of indices for mappings retrieval.
+       */
+      fieldsIndices: schema.maybe(schema.string()),
+    },
+    {
+      validate: (payload) => {
+        if (Object.keys(payload).length === 0) {
+          return 'The request must contain at least one of the following parameters: indices, fields, templates, dataStreams.';
+        }
+      },
+    }
+  ),
+};
+
+export type SettingsToRetrieve = TypeOf<typeof autoCompleteEntitiesValidationConfig.query>;

src/plugins/console/tsconfig.json

@@ -30,6 +30,7 @@
     "@kbn/core-http-router-server-internal",
     "@kbn/web-worker-stub",
     "@kbn/core-elasticsearch-server",
+    "@kbn/core-http-browser-mocks",
   ],
   "exclude": [
     "target/**/*",

test/api_integration/apis/console/autocomplete_entities.ts

@@ -128,7 +128,7 @@ export default ({ getService }: FtrProviderContext) => {
     return await supertest.get('/api/console/autocomplete_entities').query(query);
   };
 
-  describe('/api/console/autocomplete_entities', () => {
+  describe('/api/console/autocomplete_entities', function () {
     const indexName = 'test-index-1';
     const aliasName = 'test-alias-1';
     const indexTemplateName = 'test-index-template-1';
@@ -238,9 +238,17 @@ export default ({ getService }: FtrProviderContext) => {
       expect(body.mappings).to.eql({});
     });
 
-    it('should return mappings with fields setting is set to true', async () => {
+    it('should not return mappings with fields setting is set to true without the list of indices is provided', async () => {
       const response = await sendRequest({ fields: true });
 
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(Object.keys(body.mappings)).to.not.contain(indexName);
+    });
+
+    it('should return mappings with fields setting is set to true and the list of indices is provided', async () => {
+      const response = await sendRequest({ fields: true, fieldsIndices: indexName });
+
       const { body, status } = response;
       expect(status).to.be(200);
       expect(Object.keys(body.mappings)).to.contain(indexName);