[Console] Filter autocomplete endpoints by availability (#161781)

Closes https://github.com/elastic/kibana/issues/160160

## Summary

This PR adds functionality to the new autocomplete generation script for
creating an `availability` property in the spec files that is used for
filtering out endpoints that are not available in the current
environment (e.g. `serverless` or `stack`). It also adds a config
setting in the console plugin that specifies the current environment.
This setting is also configured accordingly for serverless.


**How to test**
1. Checkout the [ES specification
repo](https://github.com/elastic/elasticsearch-specification)
2. Run the command with `node scripts/generate_console_definitions.js
--source <ES_SPECIFICATION_REPO> --emptyDest` where
`<ES_SPECIFICATION_REPO>` is the absolute path to the root of the ES
specification repo
3. Start the classic Kibana and verify that Console suggests only
endpoints that are available in the `stack` environment.
4. Start Kibana in any of the serverless modes and verify that Console
suggests only endpoints that are available in the `serverless`
environment.

Here are some example endpoints that can be used for testing:
| Endpoint  | Available in Stack | Available in Serverless |
| ------------- | ------------- | ------------- |
| [POST
_bulk](https://github.com/elastic/elasticsearch-specification/blob/main/specification/_global/bulk/BulkRequest.ts)
| Yes | Yes |
| [DELETE
_security/oauth2/token](https://github.com/elastic/elasticsearch-specification/blob/main/specification/security/invalidate_token/SecurityInvalidateTokenRequest.ts)
| Yes | No |

config/serverless.yml

@@ -72,5 +72,8 @@ server.versioned.strictClientVersionCheck: false
 xpack.spaces.maxSpaces: 1
 xpack.spaces.allowFeatureVisibility: false
 
+# Only display console autocomplete suggestions for ES endpoints that are available in serverless
+console.autocompleteDefinitions.endpointsAvailability: serverless
+
 # Allow authentication via the Elasticsearch JWT realm with the `shared_secret` client authentication type.
 elasticsearch.requestHeadersWhitelist: ["authorization", "es-client-authentication"]

packages/kbn-generate-console-definitions/src/generate_availability.test.ts

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SpecificationTypes } from './types';
+import { generateAvailability } from './generate_availability';
+
+describe('generateAvailability', () => {
+  const mockEndpoint: SpecificationTypes.Endpoint = {
+    name: 'test-endpoint',
+    description: 'test-endpoint',
+    docUrl: 'test-endpoint',
+    availability: {},
+    request: null,
+    requestBodyRequired: false,
+    response: null,
+    urls: [],
+  };
+
+  it('converts empty availability to true', () => {
+    const endpoint = mockEndpoint;
+
+    const availability = generateAvailability(endpoint);
+    expect(availability).toEqual({
+      stack: true,
+      serverless: true,
+    });
+  });
+
+  describe('converts correctly stack visibility', function () {
+    it('public visibility to true', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          stack: {
+            visibility: SpecificationTypes.Visibility.public,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: true,
+      });
+    });
+
+    it('private visibility to false', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          stack: {
+            visibility: SpecificationTypes.Visibility.private,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: false,
+        serverless: true,
+      });
+    });
+
+    it('feature_flag visibility to false', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          stack: {
+            visibility: SpecificationTypes.Visibility.feature_flag,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: false,
+        serverless: true,
+      });
+    });
+
+    it('missing visibility to true', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          stack: {},
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: true,
+      });
+    });
+  });
+
+  describe('converts correctly serverless visibility', function () {
+    it('public visibility to true', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          serverless: {
+            visibility: SpecificationTypes.Visibility.public,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: true,
+      });
+    });
+
+    it('private visibility to false', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          serverless: {
+            visibility: SpecificationTypes.Visibility.private,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: false,
+      });
+    });
+
+    it('feature_flag visibility to false', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          serverless: {
+            visibility: SpecificationTypes.Visibility.feature_flag,
+          },
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: false,
+      });
+    });
+
+    it('missing visibility to true', () => {
+      const endpoint = {
+        ...mockEndpoint,
+        availability: {
+          serverless: {},
+        },
+      };
+
+      const availability = generateAvailability(endpoint);
+      expect(availability).toEqual({
+        stack: true,
+        serverless: true,
+      });
+    });
+  });
+});

packages/kbn-generate-console-definitions/src/generate_availability.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { AutocompleteAvailability } from './types';
+import type { SpecificationTypes } from './types';
+
+const DEFAULT_ENDPOINT_AVAILABILITY = true;
+
+export const generateAvailability = (
+  endpoint: SpecificationTypes.Endpoint
+): AutocompleteAvailability => {
+  const availability: AutocompleteAvailability = {
+    stack: DEFAULT_ENDPOINT_AVAILABILITY,
+    serverless: DEFAULT_ENDPOINT_AVAILABILITY,
+  };
+  if (endpoint.availability.stack?.visibility) {
+    availability.stack = endpoint.availability.stack?.visibility === 'public';
+  }
+  if (endpoint.availability.serverless?.visibility) {
+    availability.serverless = endpoint.availability.serverless?.visibility === 'public';
+  }
+  return availability;
+};

packages/kbn-generate-console-definitions/src/generate_console_definitions.ts

@@ -10,6 +10,7 @@ import fs from 'fs';
 import Path, { join } from 'path';
 import { ToolingLog } from '@kbn/tooling-log';
 import { generateQueryParams } from './generate_query_params';
+import { generateAvailability } from './generate_availability';
 import type {
   AutocompleteBodyParams,
   AutocompleteDefinition,
@@ -86,12 +87,13 @@ const generateDefinition = (
   const methods = generateMethods(endpoint);
   const patterns = generatePatterns(endpoint);
   const documentation = generateDocumentation(endpoint);
+  const availability = generateAvailability(endpoint);
   let definition: AutocompleteDefinition = {};
   const params = generateParams(endpoint, schema);
   if (params) {
     definition = addParams(definition, params);
   }
-  definition = { ...definition, methods, patterns, documentation };
+  definition = { ...definition, methods, patterns, documentation, availability };
 
   return definition;
 };

packages/kbn-generate-console-definitions/src/types/autocomplete_definition_types.ts

@@ -14,10 +14,16 @@ export interface AutocompleteBodyParams {
   [key: string]: number | string;
 }
 
+export interface AutocompleteAvailability {
+  stack: boolean;
+  serverless: boolean;
+}
+
 export interface AutocompleteDefinition {
   documentation?: string;
   methods?: string[];
   patterns?: string[];
   url_params?: AutocompleteUrlParams;
   data_autocomplete_rules?: AutocompleteBodyParams;
+  availability?: AutocompleteAvailability;
 }

packages/kbn-generate-console-definitions/src/types/index.ts

@@ -10,6 +10,7 @@ export type {
   AutocompleteDefinition,
   AutocompleteUrlParams,
   AutocompleteBodyParams,
+  AutocompleteAvailability,
 } from './autocomplete_definition_types';
 
 export * as SpecificationTypes from './specification_types';

src/plugins/console/server/config.ts

@@ -24,6 +24,11 @@ const schemaLatest = schema.object(
     ui: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
     }),
+    autocompleteDefinitions: schema.object({
+      // Only displays the endpoints that are available in the specified environment
+      // Current supported values are 'stack' and 'serverless'
+      endpointsAvailability: schema.string({ defaultValue: 'stack' }),
+    }),
   },
   { defaultValue: undefined }
 );
@@ -31,6 +36,7 @@ const schemaLatest = schema.object(
 const configLatest: PluginConfigDescriptor<ConsoleConfig> = {
   exposeToBrowser: {
     ui: true,
+    autocompleteDefinitions: true,
   },
   schema: schemaLatest,
   deprecations: () => [],

src/plugins/console/server/plugin.ts

@@ -79,7 +79,10 @@ export class ConsoleServerPlugin implements Plugin<ConsoleSetup, ConsoleStart> {
   }
 
   start() {
-    this.specDefinitionsService.start();
+    const {
+      autocompleteDefinitions: { endpointsAvailability: endpointsAvailability },
+    } = this.ctx.config.get<ConsoleConfig>();
+    this.specDefinitionsService.start({ endpointsAvailability });
   }
 
   stop() {

src/plugins/console/server/services/spec_definitions_service.ts

@@ -22,6 +22,11 @@ interface EndpointDescription {
   data_autocomplete_rules?: Record<string, unknown>;
   url_components?: Record<string, unknown>;
   priority?: number;
+  availability?: Record<string, boolean>;
+}
+
+export interface SpecDefinitionsDependencies {
+  endpointsAvailability: string;
 }
 
 export class SpecDefinitionsService {
@@ -81,9 +86,9 @@ export class SpecDefinitionsService {
     };
   }
 
-  public start() {
+  public start({ endpointsAvailability }: SpecDefinitionsDependencies) {
     if (!this.hasLoadedSpec) {
-      this.loadJsonSpec();
+      this.loadJsonSpec(endpointsAvailability);
       this.loadJSSpec();
       this.hasLoadedSpec = true;
     } else {
@@ -116,11 +121,19 @@ export class SpecDefinitionsService {
     }, {} as Record<string, EndpointDescription>);
   }
 
-  private loadJsonSpec() {
+  private loadJsonSpec(endpointsAvailability: string) {
     const result = this.loadJSONSpecInDir(AUTOCOMPLETE_DEFINITIONS_FOLDER);
 
     Object.keys(result).forEach((endpoint) => {
-      this.addEndpointDescription(endpoint, result[endpoint]);
+      const description = result[endpoint];
+      const addEndpoint =
+        // If the 'availability' property doesn't exist, display the endpoint by default
+        !description.availability ||
+        (endpointsAvailability === 'stack' && description.availability.stack) ||
+        (endpointsAvailability === 'serverless' && description.availability.serverless);
+      if (addEndpoint) {
+        this.addEndpointDescription(endpoint, description);
+      }
     });
   }
 

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -83,6 +83,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         // what types of config settings can be exposed to the browser.
         // When plugin owners make a change that exposes additional config values, the changes will be reflected in this test assertion.
         // Ensure that your change does not unintentionally expose any sensitive values!
+        'console.autocompleteDefinitions.endpointsAvailability (string)',
         'console.ui.enabled (boolean)',
         'dashboard.allowByValueEmbeddables (boolean)',
         'unifiedSearch.autocomplete.querySuggestions.enabled (boolean)',