[Console] Use ES specification for autocomplete definitions (#159241)

## Summary
Fixes https://github.com/elastic/kibana/issues/159410 

This PR adds a new package `kbn-generate-console-definitions` that will
eventually replace the package `kbn-spec-to-console`. It also adds a new
command to use the script in the new package. The new command can be
used as following:
- `node scripts/generate_console_definitions.js --source
<PATH_TO_ES_SPECIFICATION_REPO>` where `PATH_TO_ES_SPECIFICATION_FOLDER`
is the absolute path to the root of the [ES specification
repo](https://github.com/elastic/elasticsearch-specification), for
example `/Users/yulia/elastic/elasticsearch-specification`. This command
will generate autocomplete definitions in the folder
`KIBANA_ROOT/src/plugins/console/server/lib/json/generated`.
- Optionally `--dest` parameter can be passed to generate definitions in
a different folder, relative to `KIBANA_ROOT`.

Basic script functionality was implemented in this PR: 

- [x] Create the folder if doesn't exist yet
- [x] Remove all files in the folder before generating definitions
- [x] Load the specification schema and parse each endpoint 
- [x] Create a file for each endpoint with the endpoint name, methods,
patterns and doc urls.

Functionality that will be added in follow up PRs:

- Url paramaters
- Request body parameters
- Availability property
- Unit test for script functions

### How to test
1. Checkout ES specification repo
2. Run the command with `node scripts/generate_console_definitions.js
--source <ES_SPECIFICATION_REPO> --emptyDest` where
`<ES_SPECIFICATION_REPO>` is the absolute path to the root of the ES
specification repo
3. Check the changes to the generated files in the folder
`/KIBANA_REPO/src/plugins/console/server/lib/spec_definitions/json/generated`
and make sure they have a correct endpoint name, patterns, methods and
doc links. We are not generating any url params, request body params or
availability property for now.
4. Change the constant in the file
`KIBANA_REPO/src/plugins/console/common/constants/autocomplete_definitions.ts`
to a non-existent folder. Run the script `node
scripts/generate_console_definitions.js --source
<ES_SPECIFICATION_REPO>` and check that the folder has been created
successfully
5. Re-run the command without `--emptyDest` flag targeting a folder that
already contain some files. Check that the script fails and doesn't
silently remove existing files
6. Run the help command `node scripts/generate_console_definitions.js
--help` and check if the help message makes sense

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Alison Goryachev <alisonmllr20@gmail.com>

.github/CODEOWNERS

@@ -401,6 +401,7 @@ packages/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/
 packages/kbn-ftr-screenshot-filename @elastic/kibana-operations @elastic/appex-qa
 x-pack/test/functional_with_es_ssl/plugins/cases @elastic/response-ops
 packages/kbn-generate @elastic/kibana-operations
+packages/kbn-generate-console-definitions @elastic/platform-deployment-management
 packages/kbn-generate-csv @elastic/appex-sharedux
 packages/kbn-generate-csv-types @elastic/appex-sharedux
 packages/kbn-get-repo-files @elastic/kibana-operations

package.json

@@ -429,6 +429,7 @@
     "@kbn/foo-plugin": "link:x-pack/test/ui_capabilities/common/plugins/foo_plugin",
     "@kbn/ftr-apis-plugin": "link:src/plugins/ftr_apis",
     "@kbn/functional-with-es-ssl-cases-test-plugin": "link:x-pack/test/functional_with_es_ssl/plugins/cases",
+    "@kbn/generate-console-definitions": "link:packages/kbn-generate-console-definitions",
     "@kbn/generate-csv": "link:packages/kbn-generate-csv",
     "@kbn/generate-csv-types": "link:packages/kbn-generate-csv-types",
     "@kbn/global-search-bar-plugin": "link:x-pack/plugins/global_search_bar",

packages/kbn-generate-console-definitions/README.md

@@ -0,0 +1,3 @@
+# @kbn/generate-console-definitions
+
+Empty package generated by @kbn/generate

packages/kbn-generate-console-definitions/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './src/cli';

packages/kbn-generate-console-definitions/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-generate-console-definitions'],
+};

packages/kbn-generate-console-definitions/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/generate-console-definitions",
+  "owner": "@elastic/platform-deployment-management"
+}

packages/kbn-generate-console-definitions/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/generate-console-definitions",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-generate-console-definitions/src/cli.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import fs from 'fs';
+import { run } from '@kbn/dev-cli-runner';
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { AUTOCOMPLETE_DEFINITIONS_FOLDER } from '@kbn/console-plugin/common/constants';
+import { generateConsoleDefinitions } from './generate_console_definitions';
+
+export function runGenerateConsoleDefinitionsCli() {
+  run(
+    (context) => {
+      const { log, flags } = context;
+      log.info('starting console definitions generation');
+      const { source, dest, emptyDest } = flags;
+      if (!source) {
+        throw createFlagError(`Missing --source argument`);
+      }
+      let definitionsFolder = Path.resolve(REPO_ROOT, `${dest}`);
+      if (!dest) {
+        definitionsFolder = Path.resolve(AUTOCOMPLETE_DEFINITIONS_FOLDER, 'generated');
+      }
+      log.info(`autocomplete definitions folder ${definitionsFolder}`);
+      if (!fs.existsSync(definitionsFolder)) {
+        log.warning(`folder ${definitionsFolder} doesn't exist, creating a new folder`);
+        fs.mkdirSync(definitionsFolder, { recursive: true });
+        log.warning(`created a new folder ${definitionsFolder}`);
+      }
+      const files = fs.readdirSync(definitionsFolder);
+      if (files.length > 0) {
+        if (!emptyDest) {
+          throw createFlagError(
+            `Definitions folder already contain files, use --emptyDest to clean the folder before generation`
+          );
+        }
+        log.warning(`folder ${definitionsFolder} already contains files, emptying the folder`);
+        for (const file of files) {
+          fs.unlinkSync(Path.resolve(definitionsFolder, file));
+        }
+        log.warning(`folder ${definitionsFolder} has been emptied`);
+      }
+
+      const specsRepo = Path.resolve(`${source}`);
+      if (!fs.existsSync(specsRepo)) {
+        throw createFlagError(`ES specification folder ${specsRepo} doesn't exist`);
+      }
+      log.info(`ES specification repo folder ${source}`);
+      generateConsoleDefinitions({ specsRepo, definitionsFolder, log });
+      log.info('completed console definitions generation');
+    },
+    {
+      description: `Generate Console autocomplete definitions from the ES specification repo`,
+      usage: `
+node scripts/generate_console_definitions.js --help
+node scripts/generate_console_definitions.js --source <ES_SPECIFICATION_REPO>
+node scripts/generate_console_definitions.js --source <ES_SPECIFICATION_REPO> [--dest <DEFINITIONS_FOLDER] [--emptyDest]
+`,
+      flags: {
+        string: ['source', 'dest'],
+        boolean: ['emptyDest'],
+        help: `
+--source        Folder containing the root of the Elasticsearch specification repo
+--dest          Folder where console autocomplete definitions will be generated (relative to the Kibana repo root)
+--emptyDest     Flag to empty definitions folder if it already contains any files
+`,
+      },
+    }
+  );
+}

packages/kbn-generate-console-definitions/src/generate_console_definitions.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import fs from 'fs';
+import Path, { join } from 'path';
+import { ToolingLog } from '@kbn/tooling-log';
+
+interface EndpointRequest {
+  name: string;
+  namespace: string;
+}
+
+interface Endpoint {
+  name: string;
+  urls: Array<{
+    methods: string[];
+    path: string;
+  }>;
+  docUrl: string;
+  request: null | EndpointRequest;
+}
+
+interface SchemaType {
+  name: {
+    name: string;
+    namespace: string;
+  };
+}
+
+interface Schema {
+  endpoints: Endpoint[];
+  types: SchemaType[];
+}
+
+interface UrlParams {
+  [key: string]: number | string;
+}
+
+interface BodyParams {
+  [key: string]: number | string;
+}
+
+interface Definition {
+  documentation?: string;
+  methods: string[];
+  patterns: string[];
+  url_params?: UrlParams;
+  data_autocomplete_rules?: BodyParams;
+}
+
+const generateMethods = (endpoint: Endpoint): string[] => {
+  // this array consists of arrays of strings
+  const methodsArray = endpoint.urls.map((url) => url.methods);
+  // flatten to return array of strings
+  const flattenMethodsArray = ([] as string[]).concat(...methodsArray);
+  // use set to remove duplication
+  return [...new Set(flattenMethodsArray)];
+};
+
+const generatePatterns = (endpoint: Endpoint): string[] => {
+  return endpoint.urls.map(({ path }) => {
+    let pattern = path;
+    // remove leading / if present
+    if (path.startsWith('/')) {
+      pattern = path.substring(1);
+    }
+    return pattern;
+  });
+};
+
+const generateDocumentation = (endpoint: Endpoint): string => {
+  return endpoint.docUrl;
+};
+
+const generateParams = (
+  endpoint: Endpoint,
+  schema: Schema
+): { urlParams: UrlParams; bodyParams: BodyParams } | undefined => {
+  const { request } = endpoint;
+  if (!request) {
+    return;
+  }
+  const requestType = schema.types.find(
+    ({ name: { name, namespace } }) => name === request.name && namespace === request.namespace
+  );
+  if (!requestType) {
+    return;
+  }
+
+  const urlParams = generateUrlParams(requestType);
+  const bodyParams = generateBodyParams(requestType);
+  return { urlParams, bodyParams };
+};
+
+const generateUrlParams = (requestType: SchemaType): UrlParams => {
+  return {};
+};
+
+const generateBodyParams = (requestType: SchemaType): BodyParams => {
+  return {};
+};
+
+const addParams = (
+  definition: Definition,
+  params: { urlParams: UrlParams; bodyParams: BodyParams }
+): Definition => {
+  const { urlParams, bodyParams } = params;
+  if (urlParams && Object.keys(urlParams).length > 0) {
+    definition.url_params = urlParams;
+  }
+  if (bodyParams && Object.keys(bodyParams).length > 0) {
+    definition.data_autocomplete_rules = bodyParams;
+  }
+  return definition;
+};
+
+const generateDefinition = (endpoint: Endpoint, schema: Schema): Definition => {
+  const methods = generateMethods(endpoint);
+  const patterns = generatePatterns(endpoint);
+  const documentation = generateDocumentation(endpoint);
+  let definition: Definition = { methods, patterns, documentation };
+  const params = generateParams(endpoint, schema);
+  if (params) {
+    definition = addParams(definition, params);
+  }
+
+  return definition;
+};
+
+export function generateConsoleDefinitions({
+  specsRepo,
+  definitionsFolder,
+  log,
+}: {
+  specsRepo: string;
+  definitionsFolder: string;
+  log: ToolingLog;
+}) {
+  const pathToSchemaFile = Path.resolve(specsRepo, 'output/schema/schema.json');
+  log.info('loading the ES specification schema file');
+  const schema = JSON.parse(fs.readFileSync(pathToSchemaFile, 'utf8')) as Schema;
+
+  const { endpoints } = schema;
+  log.info(`iterating over endpoints array: ${endpoints.length} endpoints`);
+  endpoints.forEach((endpoint) => {
+    const { name } = endpoint;
+    log.info(name);
+    const definition = generateDefinition(endpoint, schema);
+    const fileContent: { [name: string]: Definition } = {
+      [name]: definition,
+    };
+    fs.writeFileSync(
+      join(definitionsFolder, `${name}.json`),
+      JSON.stringify(fileContent, null, 2) + '\n',
+      'utf8'
+    );
+  });
+}

packages/kbn-generate-console-definitions/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/dev-cli-runner",
+    "@kbn/dev-cli-errors",
+    "@kbn/repo-info",
+    "@kbn/console-plugin",
+    "@kbn/tooling-log",
+  ]
+}

scripts/generate_console_definitions.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('@kbn/generate-console-definitions').runGenerateConsoleDefinitionsCli();

src/plugins/console/common/constants/autocomplete_definitions.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { resolve } from 'path';
+
+export const AUTOCOMPLETE_DEFINITIONS_FOLDER = resolve(
+  __dirname,
+  '../../server/lib/spec_definitions/json'
+);

src/plugins/console/common/constants/index.ts

@@ -9,3 +9,4 @@
 export { MAJOR_VERSION } from './plugin';
 export { API_BASE_PATH, KIBANA_API_PREFIX } from './api';
 export { DEFAULT_VARIABLES } from './variables';
+export { AUTOCOMPLETE_DEFINITIONS_FOLDER } from './autocomplete_definitions';

src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts

@@ -29,7 +29,8 @@ export function getDocumentation(
     if (endpoint && endpoint.documentation && endpoint.documentation.indexOf('http') !== -1) {
       return endpoint.documentation
         .replace('/master/', `/${docLinkVersion}/`)
-        .replace('/current/', `/${docLinkVersion}/`);
+        .replace('/current/', `/${docLinkVersion}/`)
+        .replace('/{branch}/', `/${docLinkVersion}/`);
     } else {
       return null;
     }

src/plugins/console/server/services/spec_definitions_service.ts

@@ -8,14 +8,13 @@
 
 import _, { merge } from 'lodash';
 import globby from 'globby';
-import { basename, join, resolve } from 'path';
+import { basename, join } from 'path';
 import normalizePath from 'normalize-path';
 import { readFileSync } from 'fs';
 
+import { AUTOCOMPLETE_DEFINITIONS_FOLDER } from '../../common/constants';
 import { jsSpecLoaders } from '../lib';
 
-const PATH_TO_OSS_JSON_SPEC = resolve(__dirname, '../lib/spec_definitions/json');
-
 interface EndpointDescription {
   methods?: string[];
   patterns?: string | string[];
@@ -129,7 +128,7 @@ export class SpecDefinitionsService {
   }
 
   private loadJsonSpec() {
-    const result = this.loadJSONSpecInDir(PATH_TO_OSS_JSON_SPEC);
+    const result = this.loadJSONSpecInDir(AUTOCOMPLETE_DEFINITIONS_FOLDER);
     this.extensionSpecFilePaths.forEach((extensionSpecFilePath) => {
       merge(result, this.loadJSONSpecInDir(extensionSpecFilePath));
     });

tsconfig.base.json

@@ -796,6 +796,8 @@
       "@kbn/functional-with-es-ssl-cases-test-plugin/*": ["x-pack/test/functional_with_es_ssl/plugins/cases/*"],
       "@kbn/generate": ["packages/kbn-generate"],
       "@kbn/generate/*": ["packages/kbn-generate/*"],
+      "@kbn/generate-console-definitions": ["packages/kbn-generate-console-definitions"],
+      "@kbn/generate-console-definitions/*": ["packages/kbn-generate-console-definitions/*"],
       "@kbn/generate-csv": ["packages/kbn-generate-csv"],
       "@kbn/generate-csv/*": ["packages/kbn-generate-csv/*"],
       "@kbn/generate-csv-types": ["packages/kbn-generate-csv-types"],

yarn.lock

@@ -4428,6 +4428,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/generate-console-definitions@link:packages/kbn-generate-console-definitions":
+  version "0.0.0"
+  uid ""
+
 "@kbn/generate-csv-types@link:packages/kbn-generate-csv-types":
   version "0.0.0"
   uid ""