From 0a7ee0836288beb9418a3cf48083ff39b8fb9521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yulia=20=C4=8Cech?= <6585477+yuliacech@users.noreply.github.com> Date: Fri, 23 Jun 2023 21:05:25 +0200 Subject: [PATCH] [Console] Use ES specification for autocomplete definitions (#159241) ## Summary Fixes https://github.com/elastic/kibana/issues/159410 This PR adds a new package `kbn-generate-console-definitions` that will eventually replace the package `kbn-spec-to-console`. It also adds a new command to use the script in the new package. The new command can be used as following: - `node scripts/generate_console_definitions.js --source ` where `PATH_TO_ES_SPECIFICATION_FOLDER` is the absolute path to the root of the [ES specification repo](https://github.com/elastic/elasticsearch-specification), for example `/Users/yulia/elastic/elasticsearch-specification`. This command will generate autocomplete definitions in the folder `KIBANA_ROOT/src/plugins/console/server/lib/json/generated`. - Optionally `--dest` parameter can be passed to generate definitions in a different folder, relative to `KIBANA_ROOT`. Basic script functionality was implemented in this PR: - [x] Create the folder if doesn't exist yet - [x] Remove all files in the folder before generating definitions - [x] Load the specification schema and parse each endpoint - [x] Create a file for each endpoint with the endpoint name, methods, patterns and doc urls. Functionality that will be added in follow up PRs: - Url paramaters - Request body parameters - Availability property - Unit test for script functions ### How to test 1. Checkout ES specification repo 2. Run the command with `node scripts/generate_console_definitions.js --source --emptyDest` where `` is the absolute path to the root of the ES specification repo 3. Check the changes to the generated files in the folder `/KIBANA_REPO/src/plugins/console/server/lib/spec_definitions/json/generated` and make sure they have a correct endpoint name, patterns, methods and doc links. We are not generating any url params, request body params or availability property for now. 4. Change the constant in the file `KIBANA_REPO/src/plugins/console/common/constants/autocomplete_definitions.ts` to a non-existent folder. Run the script `node scripts/generate_console_definitions.js --source ` and check that the folder has been created successfully 5. Re-run the command without `--emptyDest` flag targeting a folder that already contain some files. Check that the script fails and doesn't silently remove existing files 6. Run the help command `node scripts/generate_console_definitions.js --help` and check if the help message makes sense --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Alison Goryachev --- .github/CODEOWNERS | 1 + package.json | 1 + .../README.md | 3 + .../kbn-generate-console-definitions/index.ts | 9 + .../jest.config.js | 13 ++ .../kibana.jsonc | 5 + .../package.json | 6 + .../src/cli.ts | 76 ++++++++ .../src/generate_console_definitions.ts | 163 ++++++++++++++++++ .../tsconfig.json | 23 +++ scripts/generate_console_definitions.js | 10 ++ .../constants/autocomplete_definitions.ts | 14 ++ src/plugins/console/common/constants/index.ts | 1 + .../editor/legacy/console_menu_actions.ts | 3 +- .../services/spec_definitions_service.ts | 7 +- tsconfig.base.json | 2 + yarn.lock | 4 + 17 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 packages/kbn-generate-console-definitions/README.md create mode 100644 packages/kbn-generate-console-definitions/index.ts create mode 100644 packages/kbn-generate-console-definitions/jest.config.js create mode 100644 packages/kbn-generate-console-definitions/kibana.jsonc create mode 100644 packages/kbn-generate-console-definitions/package.json create mode 100644 packages/kbn-generate-console-definitions/src/cli.ts create mode 100644 packages/kbn-generate-console-definitions/src/generate_console_definitions.ts create mode 100644 packages/kbn-generate-console-definitions/tsconfig.json create mode 100644 scripts/generate_console_definitions.js create mode 100644 src/plugins/console/common/constants/autocomplete_definitions.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 78eae8cab8d7..399b45008daa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -401,6 +401,7 @@ packages/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/ packages/kbn-ftr-screenshot-filename @elastic/kibana-operations @elastic/appex-qa x-pack/test/functional_with_es_ssl/plugins/cases @elastic/response-ops packages/kbn-generate @elastic/kibana-operations +packages/kbn-generate-console-definitions @elastic/platform-deployment-management packages/kbn-generate-csv @elastic/appex-sharedux packages/kbn-generate-csv-types @elastic/appex-sharedux packages/kbn-get-repo-files @elastic/kibana-operations diff --git a/package.json b/package.json index ca2db7fd60db..51c4c29da929 100644 --- a/package.json +++ b/package.json @@ -429,6 +429,7 @@ "@kbn/foo-plugin": "link:x-pack/test/ui_capabilities/common/plugins/foo_plugin", "@kbn/ftr-apis-plugin": "link:src/plugins/ftr_apis", "@kbn/functional-with-es-ssl-cases-test-plugin": "link:x-pack/test/functional_with_es_ssl/plugins/cases", + "@kbn/generate-console-definitions": "link:packages/kbn-generate-console-definitions", "@kbn/generate-csv": "link:packages/kbn-generate-csv", "@kbn/generate-csv-types": "link:packages/kbn-generate-csv-types", "@kbn/global-search-bar-plugin": "link:x-pack/plugins/global_search_bar", diff --git a/packages/kbn-generate-console-definitions/README.md b/packages/kbn-generate-console-definitions/README.md new file mode 100644 index 000000000000..71596b6fdf0b --- /dev/null +++ b/packages/kbn-generate-console-definitions/README.md @@ -0,0 +1,3 @@ +# @kbn/generate-console-definitions + +Empty package generated by @kbn/generate diff --git a/packages/kbn-generate-console-definitions/index.ts b/packages/kbn-generate-console-definitions/index.ts new file mode 100644 index 000000000000..db51c61b9891 --- /dev/null +++ b/packages/kbn-generate-console-definitions/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './src/cli'; diff --git a/packages/kbn-generate-console-definitions/jest.config.js b/packages/kbn-generate-console-definitions/jest.config.js new file mode 100644 index 000000000000..d11883544e21 --- /dev/null +++ b/packages/kbn-generate-console-definitions/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-generate-console-definitions'], +}; diff --git a/packages/kbn-generate-console-definitions/kibana.jsonc b/packages/kbn-generate-console-definitions/kibana.jsonc new file mode 100644 index 000000000000..25efdbcd3947 --- /dev/null +++ b/packages/kbn-generate-console-definitions/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/generate-console-definitions", + "owner": "@elastic/platform-deployment-management" +} diff --git a/packages/kbn-generate-console-definitions/package.json b/packages/kbn-generate-console-definitions/package.json new file mode 100644 index 000000000000..7f79eb540a32 --- /dev/null +++ b/packages/kbn-generate-console-definitions/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/generate-console-definitions", + "private": true, + "version": "1.0.0", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-generate-console-definitions/src/cli.ts b/packages/kbn-generate-console-definitions/src/cli.ts new file mode 100644 index 000000000000..75f7e630a2c7 --- /dev/null +++ b/packages/kbn-generate-console-definitions/src/cli.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs'; +import { run } from '@kbn/dev-cli-runner'; +import { createFlagError } from '@kbn/dev-cli-errors'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { AUTOCOMPLETE_DEFINITIONS_FOLDER } from '@kbn/console-plugin/common/constants'; +import { generateConsoleDefinitions } from './generate_console_definitions'; + +export function runGenerateConsoleDefinitionsCli() { + run( + (context) => { + const { log, flags } = context; + log.info('starting console definitions generation'); + const { source, dest, emptyDest } = flags; + if (!source) { + throw createFlagError(`Missing --source argument`); + } + let definitionsFolder = Path.resolve(REPO_ROOT, `${dest}`); + if (!dest) { + definitionsFolder = Path.resolve(AUTOCOMPLETE_DEFINITIONS_FOLDER, 'generated'); + } + log.info(`autocomplete definitions folder ${definitionsFolder}`); + if (!fs.existsSync(definitionsFolder)) { + log.warning(`folder ${definitionsFolder} doesn't exist, creating a new folder`); + fs.mkdirSync(definitionsFolder, { recursive: true }); + log.warning(`created a new folder ${definitionsFolder}`); + } + const files = fs.readdirSync(definitionsFolder); + if (files.length > 0) { + if (!emptyDest) { + throw createFlagError( + `Definitions folder already contain files, use --emptyDest to clean the folder before generation` + ); + } + log.warning(`folder ${definitionsFolder} already contains files, emptying the folder`); + for (const file of files) { + fs.unlinkSync(Path.resolve(definitionsFolder, file)); + } + log.warning(`folder ${definitionsFolder} has been emptied`); + } + + const specsRepo = Path.resolve(`${source}`); + if (!fs.existsSync(specsRepo)) { + throw createFlagError(`ES specification folder ${specsRepo} doesn't exist`); + } + log.info(`ES specification repo folder ${source}`); + generateConsoleDefinitions({ specsRepo, definitionsFolder, log }); + log.info('completed console definitions generation'); + }, + { + description: `Generate Console autocomplete definitions from the ES specification repo`, + usage: ` +node scripts/generate_console_definitions.js --help +node scripts/generate_console_definitions.js --source +node scripts/generate_console_definitions.js --source [--dest ; + docUrl: string; + request: null | EndpointRequest; +} + +interface SchemaType { + name: { + name: string; + namespace: string; + }; +} + +interface Schema { + endpoints: Endpoint[]; + types: SchemaType[]; +} + +interface UrlParams { + [key: string]: number | string; +} + +interface BodyParams { + [key: string]: number | string; +} + +interface Definition { + documentation?: string; + methods: string[]; + patterns: string[]; + url_params?: UrlParams; + data_autocomplete_rules?: BodyParams; +} + +const generateMethods = (endpoint: Endpoint): string[] => { + // this array consists of arrays of strings + const methodsArray = endpoint.urls.map((url) => url.methods); + // flatten to return array of strings + const flattenMethodsArray = ([] as string[]).concat(...methodsArray); + // use set to remove duplication + return [...new Set(flattenMethodsArray)]; +}; + +const generatePatterns = (endpoint: Endpoint): string[] => { + return endpoint.urls.map(({ path }) => { + let pattern = path; + // remove leading / if present + if (path.startsWith('/')) { + pattern = path.substring(1); + } + return pattern; + }); +}; + +const generateDocumentation = (endpoint: Endpoint): string => { + return endpoint.docUrl; +}; + +const generateParams = ( + endpoint: Endpoint, + schema: Schema +): { urlParams: UrlParams; bodyParams: BodyParams } | undefined => { + const { request } = endpoint; + if (!request) { + return; + } + const requestType = schema.types.find( + ({ name: { name, namespace } }) => name === request.name && namespace === request.namespace + ); + if (!requestType) { + return; + } + + const urlParams = generateUrlParams(requestType); + const bodyParams = generateBodyParams(requestType); + return { urlParams, bodyParams }; +}; + +const generateUrlParams = (requestType: SchemaType): UrlParams => { + return {}; +}; + +const generateBodyParams = (requestType: SchemaType): BodyParams => { + return {}; +}; + +const addParams = ( + definition: Definition, + params: { urlParams: UrlParams; bodyParams: BodyParams } +): Definition => { + const { urlParams, bodyParams } = params; + if (urlParams && Object.keys(urlParams).length > 0) { + definition.url_params = urlParams; + } + if (bodyParams && Object.keys(bodyParams).length > 0) { + definition.data_autocomplete_rules = bodyParams; + } + return definition; +}; + +const generateDefinition = (endpoint: Endpoint, schema: Schema): Definition => { + const methods = generateMethods(endpoint); + const patterns = generatePatterns(endpoint); + const documentation = generateDocumentation(endpoint); + let definition: Definition = { methods, patterns, documentation }; + const params = generateParams(endpoint, schema); + if (params) { + definition = addParams(definition, params); + } + + return definition; +}; + +export function generateConsoleDefinitions({ + specsRepo, + definitionsFolder, + log, +}: { + specsRepo: string; + definitionsFolder: string; + log: ToolingLog; +}) { + const pathToSchemaFile = Path.resolve(specsRepo, 'output/schema/schema.json'); + log.info('loading the ES specification schema file'); + const schema = JSON.parse(fs.readFileSync(pathToSchemaFile, 'utf8')) as Schema; + + const { endpoints } = schema; + log.info(`iterating over endpoints array: ${endpoints.length} endpoints`); + endpoints.forEach((endpoint) => { + const { name } = endpoint; + log.info(name); + const definition = generateDefinition(endpoint, schema); + const fileContent: { [name: string]: Definition } = { + [name]: definition, + }; + fs.writeFileSync( + join(definitionsFolder, `${name}.json`), + JSON.stringify(fileContent, null, 2) + '\n', + 'utf8' + ); + }); +} diff --git a/packages/kbn-generate-console-definitions/tsconfig.json b/packages/kbn-generate-console-definitions/tsconfig.json new file mode 100644 index 000000000000..fa878bc4843f --- /dev/null +++ b/packages/kbn-generate-console-definitions/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/dev-cli-runner", + "@kbn/dev-cli-errors", + "@kbn/repo-info", + "@kbn/console-plugin", + "@kbn/tooling-log", + ] +} diff --git a/scripts/generate_console_definitions.js b/scripts/generate_console_definitions.js new file mode 100644 index 000000000000..0b182e01e935 --- /dev/null +++ b/scripts/generate_console_definitions.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +require('../src/setup_node_env'); +require('@kbn/generate-console-definitions').runGenerateConsoleDefinitionsCli(); diff --git a/src/plugins/console/common/constants/autocomplete_definitions.ts b/src/plugins/console/common/constants/autocomplete_definitions.ts new file mode 100644 index 000000000000..9e106e1641d1 --- /dev/null +++ b/src/plugins/console/common/constants/autocomplete_definitions.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { resolve } from 'path'; + +export const AUTOCOMPLETE_DEFINITIONS_FOLDER = resolve( + __dirname, + '../../server/lib/spec_definitions/json' +); diff --git a/src/plugins/console/common/constants/index.ts b/src/plugins/console/common/constants/index.ts index 7d95d4693f7c..d8a259c4c450 100644 --- a/src/plugins/console/common/constants/index.ts +++ b/src/plugins/console/common/constants/index.ts @@ -9,3 +9,4 @@ export { MAJOR_VERSION } from './plugin'; export { API_BASE_PATH, KIBANA_API_PREFIX } from './api'; export { DEFAULT_VARIABLES } from './variables'; +export { AUTOCOMPLETE_DEFINITIONS_FOLDER } from './autocomplete_definitions'; diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts b/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts index f1bacd62289f..82f7f06bef48 100644 --- a/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts +++ b/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts @@ -29,7 +29,8 @@ export function getDocumentation( if (endpoint && endpoint.documentation && endpoint.documentation.indexOf('http') !== -1) { return endpoint.documentation .replace('/master/', `/${docLinkVersion}/`) - .replace('/current/', `/${docLinkVersion}/`); + .replace('/current/', `/${docLinkVersion}/`) + .replace('/{branch}/', `/${docLinkVersion}/`); } else { return null; } diff --git a/src/plugins/console/server/services/spec_definitions_service.ts b/src/plugins/console/server/services/spec_definitions_service.ts index 6fcf718d02bb..5da193bff8c5 100644 --- a/src/plugins/console/server/services/spec_definitions_service.ts +++ b/src/plugins/console/server/services/spec_definitions_service.ts @@ -8,14 +8,13 @@ import _, { merge } from 'lodash'; import globby from 'globby'; -import { basename, join, resolve } from 'path'; +import { basename, join } from 'path'; import normalizePath from 'normalize-path'; import { readFileSync } from 'fs'; +import { AUTOCOMPLETE_DEFINITIONS_FOLDER } from '../../common/constants'; import { jsSpecLoaders } from '../lib'; -const PATH_TO_OSS_JSON_SPEC = resolve(__dirname, '../lib/spec_definitions/json'); - interface EndpointDescription { methods?: string[]; patterns?: string | string[]; @@ -129,7 +128,7 @@ export class SpecDefinitionsService { } private loadJsonSpec() { - const result = this.loadJSONSpecInDir(PATH_TO_OSS_JSON_SPEC); + const result = this.loadJSONSpecInDir(AUTOCOMPLETE_DEFINITIONS_FOLDER); this.extensionSpecFilePaths.forEach((extensionSpecFilePath) => { merge(result, this.loadJSONSpecInDir(extensionSpecFilePath)); }); diff --git a/tsconfig.base.json b/tsconfig.base.json index 2da4fbcb830f..e6c7091676ed 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -796,6 +796,8 @@ "@kbn/functional-with-es-ssl-cases-test-plugin/*": ["x-pack/test/functional_with_es_ssl/plugins/cases/*"], "@kbn/generate": ["packages/kbn-generate"], "@kbn/generate/*": ["packages/kbn-generate/*"], + "@kbn/generate-console-definitions": ["packages/kbn-generate-console-definitions"], + "@kbn/generate-console-definitions/*": ["packages/kbn-generate-console-definitions/*"], "@kbn/generate-csv": ["packages/kbn-generate-csv"], "@kbn/generate-csv/*": ["packages/kbn-generate-csv/*"], "@kbn/generate-csv-types": ["packages/kbn-generate-csv-types"], diff --git a/yarn.lock b/yarn.lock index 2ac4dd409725..621554094fb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4428,6 +4428,10 @@ version "0.0.0" uid "" +"@kbn/generate-console-definitions@link:packages/kbn-generate-console-definitions": + version "0.0.0" + uid "" + "@kbn/generate-csv-types@link:packages/kbn-generate-csv-types": version "0.0.0" uid ""