[Console] Fix autocomplete_entities API crash when response size is too big (#140569)

Fixes https://github.com/elastic/kibana/issues/144310

### Summary

This PR addresses the issue of the Kibana instance restarting when the
response size is too big for the `autocomplete_entities` API. This
happens when a cluster has a large number of mappings and we try to
retrieve them all on the server side with
`esClient.asInternalUser.indices.getMapping()`. esClient does not handle
large responses well and throws an error that causes the Kibana instance
to restart. As node's max [string
length](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#description)
is 2^28-1 (~512MB) if the response size is over 512MB, it will throw an
error with the message "Invalid string length".

The fix is to use the raw http request to retrieve the mappings instead
of esClient and check the response size before sending it to the client.
If the response size is too big, we will return an empty object and log
the error in the server logs.

#### Proposed changes

- Remove ES JS client requests and use native Node.js HTTP client
instead
- Limit the response size to 10MB for the `autocomplete_entities` API

#### How to test this PR locally
To test this out, you will need to connect Kibana to a remote cluster
with a large number of mappings. We created a patch file that you can
apply to your local Kibana instance to test this PR. Since the patch
file contains credentials, we can't share it publicly. Please reach out
to me if you would like to test this PR. I will share the patch file and
the instructions to apply it.

Co-authored-by: Muhammad Ibragimov <muhammad.ibragimov@elastic.co>

src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js

@@ -6,12 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 export class ComponentTemplateAutocompleteComponent extends ListComponent {
   constructor(name, parent) {
-    super(name, getAutocompleteInfo().getEntityProvider('componentTemplates'), parent, true, true);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.COMPONENT_TEMPLATES),
+      parent,
+      true,
+      true
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js

@@ -7,11 +7,16 @@
  */
 
 import { ListComponent } from './list_component';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 
 export class DataStreamAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('dataStreams'), parent, multiValued);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.DATA_STREAMS),
+      parent,
+      multiValued
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js

@@ -7,11 +7,11 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function FieldGenerator(context) {
-  return _.map(getAutocompleteInfo().getEntityProvider('fields', context), function (field) {
+  return _.map(getAutocompleteInfo().getEntityProvider(ENTITIES.FIELDS, context), function (field) {
     return { name: field.name, meta: field.type };
   });
 }

src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js

@@ -7,7 +7,7 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function nonValidIndexType(token) {
@@ -16,7 +16,7 @@ function nonValidIndexType(token) {
 
 export class IndexAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued);
+    super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued);
   }
   validateTokens(tokens) {
     if (!this.multiValued && tokens.length > 1) {

src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js

@@ -6,12 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 export class IndexTemplateAutocompleteComponent extends ListComponent {
   constructor(name, parent) {
-    super(name, getAutocompleteInfo().getEntityProvider('indexTemplates'), parent, true, true);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.INDEX_TEMPLATES),
+      parent,
+      true,
+      true
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js

@@ -7,7 +7,7 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function nonValidUsernameType(token) {
@@ -16,7 +16,7 @@ function nonValidUsernameType(token) {
 
 export class UsernameAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued);
+    super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued);
   }
   validateTokens(tokens) {
     if (!this.multiValued && tokens.length > 1) {

src/plugins/console/public/lib/autocomplete_entities/types.ts

@@ -29,7 +29,7 @@ export interface FieldMapping {
   fields?: FieldMapping[];
 }
 
-export interface MappingsApiResponse {
+export interface AutoCompleteEntitiesApiResponse {
   mappings: IndicesGetMappingResponse;
   aliases: IndicesGetAliasResponse;
   dataStreams: IndicesGetDataStreamResponse;

src/plugins/console/public/services/autocomplete.ts

@@ -8,7 +8,7 @@
 
 import { createGetterSetter } from '@kbn/kibana-utils-plugin/public';
 import type { HttpSetup } from '@kbn/core/public';
-import type { MappingsApiResponse } from '../lib/autocomplete_entities/types';
+import type { AutoCompleteEntitiesApiResponse } from '../lib/autocomplete_entities/types';
 import { API_BASE_PATH } from '../../common/constants';
 import {
   Alias,
@@ -20,6 +20,15 @@ import {
 } from '../lib/autocomplete_entities';
 import { DevToolsSettings, Settings } from './settings';
 
+export enum ENTITIES {
+  INDICES = 'indices',
+  FIELDS = 'fields',
+  INDEX_TEMPLATES = 'indexTemplates',
+  COMPONENT_TEMPLATES = 'componentTemplates',
+  LEGACY_TEMPLATES = 'legacyTemplates',
+  DATA_STREAMS = 'dataStreams',
+}
+
 export class AutocompleteInfo {
   public readonly alias = new Alias();
   public readonly mapping = new Mapping();
@@ -39,19 +48,19 @@ export class AutocompleteInfo {
     context: { indices: string[]; types: string[] } = { indices: [], types: [] }
   ) {
     switch (type) {
-      case 'indices':
+      case ENTITIES.INDICES:
         const includeAliases = true;
         const collaborator = this.mapping;
         return () => this.alias.getIndices(includeAliases, collaborator);
-      case 'fields':
+      case ENTITIES.FIELDS:
         return this.mapping.getMappings(context.indices, context.types);
-      case 'indexTemplates':
+      case ENTITIES.INDEX_TEMPLATES:
         return () => this.indexTemplate.getTemplates();
-      case 'componentTemplates':
+      case ENTITIES.COMPONENT_TEMPLATES:
         return () => this.componentTemplate.getTemplates();
-      case 'legacyTemplates':
+      case ENTITIES.LEGACY_TEMPLATES:
         return () => this.legacyTemplate.getTemplates();
-      case 'dataStreams':
+      case ENTITIES.DATA_STREAMS:
         return () => this.dataStream.getDataStreams();
       default:
         throw new Error(`Unsupported type: ${type}`);
@@ -61,7 +70,7 @@ export class AutocompleteInfo {
   public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) {
     this.clearSubscriptions();
     this.http
-      .get<MappingsApiResponse>(`${API_BASE_PATH}/autocomplete_entities`, {
+      .get<AutoCompleteEntitiesApiResponse>(`${API_BASE_PATH}/autocomplete_entities`, {
         query: { ...settingsToRetrieve },
       })
       .then((data) => {
@@ -83,7 +92,7 @@ export class AutocompleteInfo {
     }
   }
 
-  private load(data: MappingsApiResponse) {
+  private load(data: AutoCompleteEntitiesApiResponse) {
     this.mapping.loadMappings(data.mappings);
     const collaborator = this.mapping;
     this.alias.loadAliases(data.aliases, collaborator);

src/plugins/console/public/services/index.ts

@@ -10,4 +10,9 @@ export { createHistory, History } from './history';
 export { createStorage, Storage, StorageKeys, setStorage, getStorage } from './storage';
 export type { DevToolsSettings } from './settings';
 export { createSettings, Settings, DEFAULT_SETTINGS } from './settings';
-export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo } from './autocomplete';
+export {
+  AutocompleteInfo,
+  getAutocompleteInfo,
+  setAutocompleteInfo,
+  ENTITIES,
+} from './autocomplete';

src/plugins/console/server/lib/proxy_request.ts

@@ -12,6 +12,7 @@ import net from 'net';
 import stream from 'stream';
 import Boom from '@hapi/boom';
 import { URL } from 'url';
+import { sanitizeHostname } from './utils';
 
 interface Args {
   method: 'get' | 'post' | 'put' | 'delete' | 'patch' | 'head';
@@ -23,13 +24,6 @@ interface Args {
   rejectUnauthorized?: boolean;
 }
 
-/**
- * Node http request library does not expect there to be trailing "[" or "]"
- * characters in ipv6 host names.
- */
-const sanitizeHostname = (hostName: string): string =>
-  hostName.trim().replace(/^\[/, '').replace(/\]$/, '');
-
 // We use a modified version of Hapi's Wreck because Hapi, Axios, and Superagent don't support GET requests
 // with bodies, but ES APIs do. Similarly with DELETE requests with bodies. Another library, `request`
 // diverged too much from current behaviour.

src/plugins/console/server/lib/utils/index.ts

@@ -8,4 +8,4 @@
 
 export { encodePath } from './encode_path';
 export { toURL } from './to_url';
-export { streamToJSON } from './stream_to_json';
+export { sanitizeHostname } from './sanitize_hostname';

src/plugins/console/server/lib/utils/sanitize_hostname.test.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { sanitizeHostname } from './sanitize_hostname';
+
+describe('sanitizeHostname', () => {
+  it('should remove leading and trailing brackets', () => {
+    expect(sanitizeHostname('[::1]')).toBe('::1');
+  });
+
+  it('should remove leading brackets', () => {
+    expect(sanitizeHostname('[::1')).toBe('::1');
+  });
+
+  it('should remove trailing brackets', () => {
+    expect(sanitizeHostname('::1]')).toBe('::1');
+  });
+
+  it('should not remove brackets in the middle of the string', () => {
+    expect(sanitizeHostname('[::1]foo')).toBe('::1]foo');
+  });
+});

src/plugins/console/server/lib/utils/sanitize_hostname.ts

@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import type { RouteDependencies } from '../../..';
-import { registerGetRoute } from './register_get_route';
-
-export function registerMappingsRoute(deps: RouteDependencies) {
-  registerGetRoute(deps);
-}
+/**
+ * Node http request library does not expect there to be trailing "[" or "]"
+ * characters in ipv6 host names.
+ */
+export const sanitizeHostname = (hostName: string): string =>
+  hostName.trim().replace(/^\[/, '').replace(/\]$/, '');

src/plugins/console/server/lib/utils/stream_to_json.test.ts

@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { Readable } from 'stream';
-import { streamToJSON } from './stream_to_json';
-import type { IncomingMessage } from 'http';
-
-describe('streamToString', () => {
-  it('should limit the response size', async () => {
-    const stream = new Readable({
-      read() {
-        this.push('a'.repeat(1000));
-      },
-    });
-    await expect(
-      streamToJSON(stream as IncomingMessage, 500)
-    ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`);
-  });
-
-  it('should parse the response', async () => {
-    const stream = new Readable({
-      read() {
-        this.push('{"test": "test"}');
-        this.push(null);
-      },
-    });
-    const result = await streamToJSON(stream as IncomingMessage, 5000);
-    expect(result).toEqual({ test: 'test' });
-  });
-});

src/plugins/console/server/lib/utils/stream_to_json.ts

@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import type { IncomingMessage } from 'http';
-
-export function streamToJSON(stream: IncomingMessage, limit: number) {
-  return new Promise<string>((resolve, reject) => {
-    const chunks: Buffer[] = [];
-    stream.on('data', (chunk) => {
-      chunks.push(chunk);
-      if (Buffer.byteLength(Buffer.concat(chunks)) > limit) {
-        stream.destroy();
-        reject(new Error('Response size limit exceeded'));
-      }
-    });
-    stream.on('end', () => {
-      const response = Buffer.concat(chunks).toString('utf8');
-      resolve(JSON.parse(response));
-    });
-    stream.on('error', reject);
-  });
-}

src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts

@@ -6,4 +6,207 @@
  * Side Public License, v 1.
  */
 
-export { registerMappingsRoute } from './register_mappings_route';
+import http from 'http';
+import https from 'https';
+import { Buffer } from 'buffer';
+import { parse } from 'query-string';
+import Boom from '@hapi/boom';
+import type { KibanaRequest } from '@kbn/core-http-server';
+import type { SemVer } from 'semver';
+import type { RouteDependencies } from '../../..';
+import { sanitizeHostname } from '../../../../lib/utils';
+import type { ESConfigForProxy } from '../../../../types';
+import { getRequestConfig } from '../proxy/create_handler';
+
+interface SettingsToRetrieve {
+  indices: boolean;
+  fields: boolean;
+  templates: boolean;
+  dataStreams: boolean;
+}
+
+type Config = ESConfigForProxy & { headers: KibanaRequest['headers'] } & { kibanaVersion: SemVer };
+
+const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB
+// Limit the response size to 10MB, because the response can be very large and sending it to the client
+// can cause the browser to hang.
+
+const getMappings = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.fields) {
+    const mappings = await getEntity('/_mapping', config);
+    return mappings;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+const getAliases = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.indices) {
+    const aliases = await getEntity('/_alias', config);
+    return aliases;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+const getDataStreams = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.dataStreams) {
+    const dataStreams = await getEntity('/_data_stream', config);
+    return dataStreams;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+const getLegacyTemplates = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.templates) {
+    const legacyTemplates = await getEntity('/_template', config);
+    return legacyTemplates;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+const getIndexTemplates = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.templates) {
+    const indexTemplates = await getEntity('/_index_template', config);
+    return indexTemplates;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+const getComponentTemplates = async (settings: SettingsToRetrieve, config: Config) => {
+  if (settings.templates) {
+    const componentTemplates = await getEntity('/_component_template', config);
+    return componentTemplates;
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+};
+
+/**
+ * Get the autocomplete suggestions for the given entity.
+ * We are using the raw http request in this function to retrieve the entities instead of esClient because
+ * the esClient does not handle large responses well. For example, the response size for
+ * the mappings can be very large(> 1GB) and the esClient will throw an 'Invalid string length'
+ * error when trying to parse the response. By using the raw http request, we can limit the
+ * response size and avoid the error.
+ * @param path  The path to the entity to retrieve. For example, '/_mapping' or '/_alias'.
+ * @param config The configuration for the request.
+ * @returns The entity retrieved from Elasticsearch.
+ */
+const getEntity = (path: string, config: Config) => {
+  return new Promise((resolve, reject) => {
+    const { hosts, kibanaVersion } = config;
+    for (let idx = 0; idx < hosts.length; idx++) {
+      const host = hosts[idx];
+      const uri = new URL(host + path);
+      const { protocol, hostname, port } = uri;
+      const { headers } = getRequestConfig(config.headers, config, uri.toString(), kibanaVersion);
+      const client = protocol === 'https:' ? https : http;
+      const options = {
+        method: 'GET',
+        headers: { ...headers },
+        host: sanitizeHostname(hostname),
+        port: port === '' ? undefined : parseInt(port, 10),
+        protocol,
+        path: `${path}?pretty=false`, // add pretty=false to compress the response by removing whitespace
+      };
+
+      try {
+        const req = client.request(options, (res) => {
+          const chunks: Buffer[] = [];
+          res.on('data', (chunk) => {
+            chunks.push(chunk);
+
+            // Destroy the request if the response is too large
+            if (Buffer.byteLength(Buffer.concat(chunks)) > MAX_RESPONSE_SIZE) {
+              req.destroy();
+              reject(Boom.badRequest(`Response size is too large for ${path}`));
+            }
+          });
+          res.on('end', () => {
+            const body = Buffer.concat(chunks).toString('utf8');
+            resolve(JSON.parse(body));
+          });
+        });
+        req.on('error', reject);
+        req.end();
+        break;
+      } catch (err) {
+        if (idx === hosts.length - 1) {
+          reject(err);
+        }
+        // Try the next host
+      }
+    }
+  });
+};
+
+export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => {
+  deps.router.get(
+    {
+      path: '/api/console/autocomplete_entities',
+      options: {
+        tags: ['access:console'],
+      },
+      validate: false,
+    },
+    async (context, request, response) => {
+      const settings = parse(request.url.search, {
+        parseBooleans: true,
+      }) as unknown as SettingsToRetrieve;
+
+      // If no settings are specified, then return 400.
+      if (Object.keys(settings).length === 0) {
+        return response.badRequest({
+          body: 'Request must contain at least one of the following parameters: indices, fields, templates, dataStreams',
+        });
+      }
+
+      const legacyConfig = await deps.proxy.readLegacyESConfig();
+      const configWithHeaders = {
+        ...legacyConfig,
+        headers: request.headers,
+        kibanaVersion: deps.kibanaVersion,
+      };
+
+      // Wait for all requests to complete, in case one of them fails return the successfull ones
+      const results = await Promise.allSettled([
+        getMappings(settings, configWithHeaders),
+        getAliases(settings, configWithHeaders),
+        getDataStreams(settings, configWithHeaders),
+        getLegacyTemplates(settings, configWithHeaders),
+        getIndexTemplates(settings, configWithHeaders),
+        getComponentTemplates(settings, configWithHeaders),
+      ]);
+
+      const [mappings, aliases, dataStreams, legacyTemplates, indexTemplates, componentTemplates] =
+        results.map((result) => {
+          // If the request was successful, return the result
+          if (result.status === 'fulfilled') {
+            return result.value;
+          }
+
+          // If the request failed, log the error and return an empty object
+          if (result.reason instanceof Error) {
+            deps.log.debug(`Failed to retrieve autocomplete suggestions: ${result.reason.message}`);
+          }
+
+          return {};
+        });
+
+      return response.ok({
+        body: {
+          mappings,
+          aliases,
+          dataStreams,
+          legacyTemplates,
+          indexTemplates,
+          componentTemplates,
+        },
+      });
+    }
+  );
+};

src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts

@@ -1,152 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-import type { IScopedClusterClient } from '@kbn/core/server';
-import { parse } from 'query-string';
-import type { IncomingMessage } from 'http';
-import type { RouteDependencies } from '../../..';
-import { API_BASE_PATH } from '../../../../../common/constants';
-import { streamToJSON } from '../../../../lib/utils';
-
-interface Settings {
-  indices: boolean;
-  fields: boolean;
-  templates: boolean;
-  dataStreams: boolean;
-}
-
-const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024;
-// Limit the response size to 10MB, because the response can be very large and sending it to the client
-// can cause the browser to hang.
-
-async function getMappings(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.fields) {
-    const stream = await esClient.asInternalUser.indices.getMapping(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-async function getAliases(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.indices) {
-    const stream = await esClient.asInternalUser.indices.getAlias(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.dataStreams) {
-    const stream = await esClient.asInternalUser.indices.getDataStream(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.templates) {
-    const stream = await esClient.asInternalUser.indices.getTemplate(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-async function getComponentTemplates(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.templates) {
-    const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-async function getIndexTemplates(esClient: IScopedClusterClient, settings: Settings) {
-  if (settings.templates) {
-    const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, {
-      asStream: true,
-    });
-    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
-  }
-  // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return {};
-}
-
-export function registerGetRoute({ router, lib: { handleEsError } }: RouteDependencies) {
-  router.get(
-    {
-      path: `${API_BASE_PATH}/autocomplete_entities`,
-      validate: false,
-    },
-    async (ctx, request, response) => {
-      try {
-        const settings = parse(request.url.search, { parseBooleans: true }) as unknown as Settings;
-
-        // If no settings are provided return 400
-        if (Object.keys(settings).length === 0) {
-          return response.badRequest({
-            body: 'Request must contain a query param of autocomplete settings',
-          });
-        }
-
-        const esClient = (await ctx.core).elasticsearch.client;
-
-        // Wait for all requests to complete, in case one of them fails return the successfull ones
-        const results = await Promise.allSettled([
-          getMappings(esClient, settings),
-          getAliases(esClient, settings),
-          getDataStreams(esClient, settings),
-          getLegacyTemplates(esClient, settings),
-          getIndexTemplates(esClient, settings),
-          getComponentTemplates(esClient, settings),
-        ]);
-
-        const [
-          mappings,
-          aliases,
-          dataStreams,
-          legacyTemplates,
-          indexTemplates,
-          componentTemplates,
-        ] = results.map((result) => {
-          // If the request was successful, return the result
-          if (result.status === 'fulfilled') {
-            return result.value;
-          }
-          // If the request failed, return an empty object
-          return {};
-        });
-
-        return response.ok({
-          body: {
-            mappings,
-            aliases,
-            dataStreams,
-            legacyTemplates,
-            indexTemplates,
-            componentTemplates,
-          },
-        });
-      } catch (e) {
-        return handleEsError({ error: e, response });
-      }
-    }
-  );
-}

src/plugins/console/server/routes/api/console/proxy/create_handler.ts

@@ -43,7 +43,7 @@ function filterHeaders(originalHeaders: object, headersToKeep: string[]): object
   return pick(originalHeaders, headersToKeepNormalized);
 }
 
-function getRequestConfig(
+export function getRequestConfig(
   headers: object,
   esConfig: ESConfigForProxy,
   uri: string,

src/plugins/console/server/routes/index.ts

@@ -17,7 +17,7 @@ import { handleEsError } from '../shared_imports';
 import { registerEsConfigRoute } from './api/console/es_config';
 import { registerProxyRoute } from './api/console/proxy';
 import { registerSpecDefinitionsRoute } from './api/console/spec_definitions';
-import { registerMappingsRoute } from './api/console/autocomplete_entities';
+import { registerAutocompleteEntitiesRoute } from './api/console/autocomplete_entities';
 
 export interface ProxyDependencies {
   readLegacyESConfig: () => Promise<ESConfigForProxy>;
@@ -43,5 +43,5 @@ export const registerRoutes = (dependencies: RouteDependencies) => {
   registerEsConfigRoute(dependencies);
   registerProxyRoute(dependencies);
   registerSpecDefinitionsRoute(dependencies);
-  registerMappingsRoute(dependencies);
+  registerAutocompleteEntitiesRoute(dependencies);
 };

test/api_integration/apis/console/autocomplete_entities.ts

@@ -7,127 +7,275 @@
  */
 
 import expect from '@kbn/expect';
-import type { Response } from 'superagent';
 import type { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
+  const client = getService('es');
 
-  function utilTest(name: string, query: object, test: (response: Response) => void) {
-    it(name, async () => {
-      const response = await supertest.get('/api/console/autocomplete_entities').query(query);
-      test(response);
+  const createIndex = async (indexName: string) => {
+    await client.indices.create({
+      index: indexName,
+      body: {
+        mappings: {
+          properties: {
+            foo: {
+              type: 'text',
+            },
+          },
+        },
+      },
     });
-  }
+  };
+
+  const createAlias = async (indexName: string, aliasName: string) => {
+    await client.indices.putAlias({
+      index: indexName,
+      name: aliasName,
+    });
+  };
+
+  const createLegacyTemplate = async (templateName: string) => {
+    await client.indices.putTemplate({
+      name: templateName,
+      body: {
+        index_patterns: ['*'],
+      },
+    });
+  };
+
+  const createComponentTemplate = async (templateName: string) => {
+    await client.cluster.putComponentTemplate({
+      name: templateName,
+      body: {
+        template: {
+          mappings: {
+            properties: {
+              '@timestamp': {
+                type: 'date',
+                format: 'date_optional_time||epoch_millis',
+              },
+              message: {
+                type: 'wildcard',
+              },
+            },
+          },
+        },
+      },
+    });
+  };
+
+  const createIndexTemplate = async (
+    templateName: string,
+    indexPatterns: string[],
+    composedOf: string[]
+  ) => {
+    await client.indices.putIndexTemplate({
+      name: templateName,
+      body: {
+        index_patterns: indexPatterns,
+        data_stream: {},
+        composed_of: composedOf,
+        priority: 500,
+      },
+    });
+  };
+
+  const createDataStream = async (dataStream: string) => {
+    await client.indices.createDataStream({
+      name: dataStream,
+    });
+  };
+
+  const deleteIndex = async (indexName: string) => {
+    await client.indices.delete({
+      index: indexName,
+    });
+  };
+
+  const deleteAlias = async (indexName: string, aliasName: string) => {
+    await client.indices.deleteAlias({
+      index: indexName,
+      name: aliasName,
+    });
+  };
+
+  const deleteIndexTemplate = async (templateName: string) => {
+    await client.indices.deleteIndexTemplate({
+      name: templateName,
+    });
+  };
+
+  const deleteComponentTemplate = async (templateName: string) => {
+    await client.cluster.deleteComponentTemplate({
+      name: templateName,
+    });
+  };
+
+  const deleteLegacyTemplate = async (templateName: string) => {
+    await client.indices.deleteTemplate({
+      name: templateName,
+    });
+  };
+
+  const deleteDataStream = async (dataStream: string) => {
+    await client.indices.deleteDataStream({
+      name: dataStream,
+    });
+  };
+
+  const sendRequest = async (query: object) => {
+    return await supertest.get('/api/console/autocomplete_entities').query(query);
+  };
 
   describe('/api/console/autocomplete_entities', () => {
-    utilTest('should not succeed if no settings are provided in query params', {}, (response) => {
+    const indexName = 'test-index-1';
+    const aliasName = 'test-alias-1';
+    const indexTemplateName = 'test-index-template-1';
+    const componentTemplateName = 'test-component-template-1';
+    const dataStreamName = 'test-data-stream-1';
+    const legacyTemplateName = 'test-legacy-template-1';
+
+    before(async () => {
+      // Setup indices, aliases, templates, and data streams
+      await createIndex(indexName);
+      await createAlias(indexName, aliasName);
+      await createComponentTemplate(componentTemplateName);
+      await createIndexTemplate(indexTemplateName, [dataStreamName], [componentTemplateName]);
+      await createDataStream(dataStreamName);
+      await createLegacyTemplate(legacyTemplateName);
+    });
+
+    after(async () => {
+      // Cleanup indices, aliases, templates, and data streams
+      await deleteAlias(indexName, aliasName);
+      await deleteIndex(indexName);
+      await deleteDataStream(dataStreamName);
+      await deleteIndexTemplate(indexTemplateName);
+      await deleteComponentTemplate(componentTemplateName);
+      await deleteLegacyTemplate(legacyTemplateName);
+    });
+
+    it('should not succeed if no settings are provided in query params', async () => {
+      const response = await sendRequest({});
       const { status } = response;
       expect(status).to.be(400);
     });
 
-    utilTest(
-      'should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"',
-      {
+    it('should return an object with properties of "mappings", "aliases", "dataStreams", "legacyTemplates", "indexTemplates", "componentTemplates"', async () => {
+      const response = await sendRequest({
         indices: true,
         fields: true,
         templates: true,
         dataStreams: true,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(Object.keys(body).sort()).to.eql([
-          'aliases',
-          'componentTemplates',
-          'dataStreams',
-          'indexTemplates',
-          'legacyTemplates',
-          'mappings',
-        ]);
-      }
-    );
+      });
 
-    utilTest(
-      'should return empty payload with all settings are set to false',
-      {
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(Object.keys(body).sort()).to.eql([
+        'aliases',
+        'componentTemplates',
+        'dataStreams',
+        'indexTemplates',
+        'legacyTemplates',
+        'mappings',
+      ]);
+    });
+
+    it('should return empty payload with all settings are set to false', async () => {
+      const response = await sendRequest({
         indices: false,
         fields: false,
         templates: false,
         dataStreams: false,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(body.legacyTemplates).to.eql({});
-        expect(body.indexTemplates).to.eql({});
-        expect(body.componentTemplates).to.eql({});
-        expect(body.aliases).to.eql({});
-        expect(body.mappings).to.eql({});
-        expect(body.dataStreams).to.eql({});
-      }
-    );
+      });
 
-    utilTest(
-      'should return empty templates with templates setting is set to false',
-      {
-        indices: true,
-        fields: true,
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.legacyTemplates).to.eql({});
+      expect(body.indexTemplates).to.eql({});
+      expect(body.componentTemplates).to.eql({});
+      expect(body.aliases).to.eql({});
+      expect(body.mappings).to.eql({});
+      expect(body.dataStreams).to.eql({});
+    });
+
+    it('should return empty templates with templates setting is set to false', async () => {
+      const response = await sendRequest({
         templates: false,
-        dataStreams: true,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(body.legacyTemplates).to.eql({});
-        expect(body.indexTemplates).to.eql({});
-        expect(body.componentTemplates).to.eql({});
-      }
-    );
+      });
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.legacyTemplates).to.eql({});
+      expect(body.indexTemplates).to.eql({});
+      expect(body.componentTemplates).to.eql({});
+    });
 
-    utilTest(
-      'should return empty data streams with dataStreams setting is set to false',
-      {
-        indices: true,
-        fields: true,
-        templates: true,
+    it('should return empty data streams with dataStreams setting is set to false', async () => {
+      const response = await sendRequest({
         dataStreams: false,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(body.dataStreams).to.eql({});
-      }
-    );
+      });
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.dataStreams).to.eql({});
+    });
 
-    utilTest(
-      'should return empty aliases with indices setting is set to false',
-      {
+    it('should return empty aliases with indices setting is set to false', async () => {
+      const response = await sendRequest({
         indices: false,
-        fields: true,
-        templates: true,
-        dataStreams: true,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(body.aliases).to.eql({});
-      }
-    );
+      });
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.aliases).to.eql({});
+    });
 
-    utilTest(
-      'should return empty mappings with fields setting is set to false',
-      {
-        indices: true,
+    it('should return empty mappings with fields setting is set to false', async () => {
+      const response = await sendRequest({
         fields: false,
-        templates: true,
-        dataStreams: true,
-      },
-      (response) => {
-        const { body, status } = response;
-        expect(status).to.be(200);
-        expect(body.mappings).to.eql({});
-      }
-    );
+      });
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.mappings).to.eql({});
+    });
+
+    it('should return mappings with fields setting is set to true', async () => {
+      const response = await sendRequest({ fields: true });
+
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(Object.keys(body.mappings)).to.contain(indexName);
+    });
+
+    it('should return aliases with indices setting is set to true', async () => {
+      const response = await sendRequest({ indices: true });
+
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.aliases[indexName].aliases).to.eql({ [aliasName]: {} });
+    });
+
+    it('should return data streams with dataStreams setting is set to true', async () => {
+      const response = await sendRequest({ dataStreams: true });
+
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(body.dataStreams.data_streams.map((ds: { name: string }) => ds.name)).to.contain(
+        dataStreamName
+      );
+    });
+
+    it('should return all templates with templates setting is set to true', async () => {
+      const response = await sendRequest({ templates: true });
+
+      const { body, status } = response;
+      expect(status).to.be(200);
+      expect(Object.keys(body.legacyTemplates)).to.contain(legacyTemplateName);
+      expect(body.indexTemplates.index_templates.map((it: { name: string }) => it.name)).to.contain(
+        indexTemplateName
+      );
+      expect(
+        body.componentTemplates.component_templates.map((ct: { name: string }) => ct.name)
+      ).to.contain(componentTemplateName);
+    });
   });
 };