[Dev tools] Fix performance issue with autocomplete suggestions (#143428)

* Fix performance issue with autocomplete suggestions

* Add unit tests for streamToString function

* Address CR change

Co-authored-by: Muhammad Ibragimov <muhammad.ibragimov@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/console/public/lib/autocomplete_entities/mapping.ts

@@ -121,23 +121,9 @@ export class Mapping implements BaseMapping {
   };
 
   loadMappings = (mappings: IndicesGetMappingResponse) => {
-    const maxMappingSize = Object.keys(mappings).length > 10 * 1024 * 1024;
-    let mappingsResponse;
-    if (maxMappingSize) {
-      // eslint-disable-next-line no-console
-      console.warn(
-        `Mapping size is larger than 10MB (${
-          Object.keys(mappings).length / 1024 / 1024
-        } MB). Ignoring...`
-      );
-      mappingsResponse = {};
-    } else {
-      mappingsResponse = mappings;
-    }
-
     this.perIndexTypes = {};
 
-    Object.entries(mappingsResponse).forEach(([index, indexMapping]) => {
+    Object.entries(mappings).forEach(([index, indexMapping]) => {
       const normalizedIndexMappings: Record<string, object[]> = {};
       let transformedMapping: Record<string, any> = indexMapping;
 

src/plugins/console/server/lib/utils/index.ts

@@ -8,3 +8,4 @@
 
 export { encodePath } from './encode_path';
 export { toURL } from './to_url';
+export { streamToJSON } from './stream_to_json';

src/plugins/console/server/lib/utils/stream_to_json.test.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Readable } from 'stream';
+import { streamToJSON } from './stream_to_json';
+import type { IncomingMessage } from 'http';
+
+describe('streamToString', () => {
+  it('should limit the response size', async () => {
+    const stream = new Readable({
+      read() {
+        this.push('a'.repeat(1000));
+      },
+    });
+    await expect(
+      streamToJSON(stream as IncomingMessage, 500)
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`);
+  });
+
+  it('should parse the response', async () => {
+    const stream = new Readable({
+      read() {
+        this.push('{"test": "test"}');
+        this.push(null);
+      },
+    });
+    const result = await streamToJSON(stream as IncomingMessage, 5000);
+    expect(result).toEqual({ test: 'test' });
+  });
+});

src/plugins/console/server/lib/utils/stream_to_json.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { IncomingMessage } from 'http';
+
+export function streamToJSON(stream: IncomingMessage, limit: number) {
+  return new Promise<string>((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    stream.on('data', (chunk) => {
+      chunks.push(chunk);
+      if (Buffer.byteLength(Buffer.concat(chunks)) > limit) {
+        stream.destroy();
+        reject(new Error('Response size limit exceeded'));
+      }
+    });
+    stream.on('end', () => {
+      const response = Buffer.concat(chunks).toString('utf8');
+      resolve(JSON.parse(response));
+    });
+    stream.on('error', reject);
+  });
+}

src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts

@@ -7,8 +7,10 @@
  */
 import type { IScopedClusterClient } from '@kbn/core/server';
 import { parse } from 'query-string';
+import type { IncomingMessage } from 'http';
 import type { RouteDependencies } from '../../..';
 import { API_BASE_PATH } from '../../../../../common/constants';
+import { streamToJSON } from '../../../../lib/utils';
 
 interface Settings {
   indices: boolean;
@@ -17,40 +19,74 @@ interface Settings {
   dataStreams: boolean;
 }
 
+const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024;
+// Limit the response size to 10MB, because the response can be very large and sending it to the client
+// can cause the browser to hang.
+
 async function getMappings(esClient: IScopedClusterClient, settings: Settings) {
   if (settings.fields) {
-    return esClient.asInternalUser.indices.getMapping();
+    const stream = await esClient.asInternalUser.indices.getMapping(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
   }
   // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return Promise.resolve({});
+  return {};
 }
 
 async function getAliases(esClient: IScopedClusterClient, settings: Settings) {
   if (settings.indices) {
-    return esClient.asInternalUser.indices.getAlias();
+    const stream = await esClient.asInternalUser.indices.getAlias(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
   }
   // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return Promise.resolve({});
+  return {};
 }
 
 async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) {
   if (settings.dataStreams) {
-    return esClient.asInternalUser.indices.getDataStream();
+    const stream = await esClient.asInternalUser.indices.getDataStream(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
   }
   // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return Promise.resolve({});
+  return {};
 }
 
-async function getTemplates(esClient: IScopedClusterClient, settings: Settings) {
+async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Settings) {
   if (settings.templates) {
-    return Promise.all([
-      esClient.asInternalUser.indices.getTemplate(),
-      esClient.asInternalUser.indices.getIndexTemplate(),
-      esClient.asInternalUser.cluster.getComponentTemplate(),
-    ]);
+    const stream = await esClient.asInternalUser.indices.getTemplate(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
   }
   // If the user doesn't want autocomplete suggestions, then clear any that exist.
-  return Promise.resolve([]);
+  return {};
+}
+
+async function getComponentTemplates(esClient: IScopedClusterClient, settings: Settings) {
+  if (settings.templates) {
+    const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
+}
+
+async function getIndexTemplates(esClient: IScopedClusterClient, settings: Settings) {
+  if (settings.templates) {
+    const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, {
+      asStream: true,
+    });
+    return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT);
+  }
+  // If the user doesn't want autocomplete suggestions, then clear any that exist.
+  return {};
 }
 
 export function registerGetRoute({ router, lib: { handleEsError } }: RouteDependencies) {
@@ -71,11 +107,32 @@ export function registerGetRoute({ router, lib: { handleEsError } }: RouteDepend
         }
 
         const esClient = (await ctx.core).elasticsearch.client;
-        const mappings = await getMappings(esClient, settings);
-        const aliases = await getAliases(esClient, settings);
-        const dataStreams = await getDataStreams(esClient, settings);
-        const [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}] =
-          await getTemplates(esClient, settings);
+
+        // Wait for all requests to complete, in case one of them fails return the successfull ones
+        const results = await Promise.allSettled([
+          getMappings(esClient, settings),
+          getAliases(esClient, settings),
+          getDataStreams(esClient, settings),
+          getLegacyTemplates(esClient, settings),
+          getIndexTemplates(esClient, settings),
+          getComponentTemplates(esClient, settings),
+        ]);
+
+        const [
+          mappings,
+          aliases,
+          dataStreams,
+          legacyTemplates,
+          indexTemplates,
+          componentTemplates,
+        ] = results.map((result) => {
+          // If the request was successful, return the result
+          if (result.status === 'fulfilled') {
+            return result.value;
+          }
+          // If the request failed, return an empty object
+          return {};
+        });
 
         return response.ok({
           body: {