adding spec to console utility as Kibana script (#35232) (#35270)

* adding spec to console utility as Kibana script

* fixing yarn.lock

* aligning dep versions with Kibana

* fixing test

package.json

@@ -69,7 +69,8 @@
     "kbn:watch": "node scripts/kibana --dev --logging.json=false",
     "build:types": "tsc --p tsconfig.types.json",
     "core:acceptApiChanges": "node scripts/check_core_api_changes.js --accept",
-    "kbn:bootstrap": "yarn build:types && node scripts/register_git_hook"
+    "kbn:bootstrap": "yarn build:types && node scripts/register_git_hook",
+    "spec_to_console": "node scripts/spec_to_console"
   },
   "repository": {
     "type": "git",
@@ -425,4 +426,4 @@
     "node": "10.15.2",
     "yarn": "^1.10.1"
   }
-}
+}

packages/kbn-spec-to-console/README.md

@@ -0,0 +1,28 @@
+A mini utility to convert [Elasticsearch's REST spec](https://github.com/elastic/elasticsearch/blob/master/rest-api-spec) to Console's (Kibana) autocomplete format.
+
+
+It is used to semi-manually update Console's autocompletion rules.
+
+
+
+### Retrieving the spec
+```
+cd es-spec
+git init
+git remote add origin https://github.com/elastic/elasticsearch
+git config core.sparsecheckout true
+echo "rest-api-spec/src/main/resources/rest-api-spec/api/*" > .git/info/sparse-checkout
+git pull --depth=1 origin master
+```
+
+### Usage
+```
+yarn spec_to_console \
+  -g "es-spec/rest-api-spec/src/main/resources/rest-api-spec/api/*.json" \
+  -d "../kibana/src/core_plugins/console/api_server/spec/generated"
+```
+
+### Information used in Console that is not available in the REST spec
+* Request bodies
+* Data fetched at runtime: indices, fields, snapshots, etc
+* Ad hoc additions

packages/kbn-spec-to-console/bin/spec_to_console.js

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const fs = require('fs');
+const path = require('path');
+const program = require('commander');
+const glob = require('glob');
+
+const packageJSON = require('../package.json');
+const convert = require('../lib/convert');
+
+program
+  .version(packageJSON.version)
+  .option('-g --glob []', 'Files to convert')
+  .option('-d --directory []', 'Output directory')
+  .parse(process.argv);
+
+if (!program.glob) {
+  console.error('Expected input');
+  process.exit(1);
+}
+
+const files = glob.sync(program.glob);
+console.log(files.length, files);
+files.forEach(file => {
+  const spec = JSON.parse(fs.readFileSync(file));
+  const output = JSON.stringify(convert(spec), null, 2);
+  if (program.directory) {
+    const outputName = path.basename(file);
+    const outputPath = path.resolve(program.directory, outputName);
+    fs.writeFileSync(outputPath, output + '\n');
+  } else {
+    console.log(output);
+  }
+});

packages/kbn-spec-to-console/index.js

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const convert = require('./lib/convert');
+module.exports = convert;

packages/kbn-spec-to-console/lib/convert.js

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const convertParams = require('./convert/params');
+const convertMethods = require('./convert/methods');
+const convertPaths = require('./convert/paths');
+const convertParts = require('./convert/parts');
+
+module.exports = spec => {
+  const result = {};
+  Object.keys(spec).forEach(api => {
+    const source = spec[api];
+    if (!source.url) {
+      return result;
+    }
+    const convertedSpec = result[api] = {};
+    if (source.url && source.url.params) {
+      const urlParams = convertParams(source.url.params);
+      if (Object.keys(urlParams).length > 0) {
+        convertedSpec.url_params = urlParams;
+      }
+    }
+
+    if (source.methods) {
+      convertedSpec.methods = convertMethods(source.methods);
+    }
+
+    if (source.url.paths) {
+      convertedSpec.patterns = convertPaths(source.url.paths);
+    }
+
+    if (source.url.parts) {
+      const components = convertParts(source.url.parts);
+      const hasComponents = Object.keys(components).filter(c => {
+        return Boolean(components[c]);
+      }).length > 0;
+      if (hasComponents) {
+        convertedSpec.url_components = convertParts(source.url.parts);
+      }
+    }
+    if (source.documentation) {
+      convertedSpec.documentation = source.documentation;
+    }
+  });
+
+  return result;
+};

packages/kbn-spec-to-console/lib/convert.test.js

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const convert = require('./convert');
+
+const clusterHealthSpec = require('../test/fixtures/cluster_health_spec');
+const clusterHealthAutocomplete = require('../test/fixtures/cluster_health_autocomplete');
+
+test('convert', () => {
+  expect(convert(clusterHealthSpec)).toEqual(clusterHealthAutocomplete);
+});

packages/kbn-spec-to-console/lib/convert/methods.js

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+module.exports = methods => {
+  return methods;
+};

packages/kbn-spec-to-console/lib/convert/params.js

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+module.exports = params => {
+  const result = {};
+  Object.keys(params).forEach(param => {
+    const { type, description = '', options = [] } = params[param];
+    const [, defaultValue] = description.match(/\(default: (.*)\)/) || [];
+    switch (type) {
+      case undefined:
+        // { description: 'TODO: ?' }
+        break;
+      case 'int':
+        result[param] = 0;
+        break;
+      case 'double':
+        result[param] = 0.0;
+        break;
+      case 'enum':
+        result[param] = options;
+        break;
+      case 'boolean':
+        result[param] = '__flag__';
+        break;
+      case 'time':
+      case 'date':
+      case 'string':
+      case 'number':
+        result[param] = defaultValue || '';
+        break;
+      case 'list':
+        result[param] = [];
+        break;
+      default:
+        throw new Error(`Unexpected type ${type}`);
+    }
+  });
+  return result;
+};

packages/kbn-spec-to-console/lib/convert/parts.js

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const replacePattern = require('../replace_pattern');
+
+module.exports = parts => {
+  const result = {};
+  Object.keys(parts).forEach(part => {
+    const key = replacePattern(part, { exact: true });
+    const options = parts[part].options;
+    if (options && options.length) {
+      result[key] = options.sort();
+    } else {
+      result[key] = null;
+    }
+  });
+  return result;
+};

packages/kbn-spec-to-console/lib/convert/paths.js

@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const replacePattern = require('../replace_pattern');
+
+module.exports = patterns => {
+  return patterns.map(pattern => {
+    return replacePattern(pattern, { brackets: true });
+  });
+};

packages/kbn-spec-to-console/lib/replace_pattern.js

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+const map = require('./static/map_interpolation');
+
+module.exports = (pattern, { brackets, exact } = {}) => {
+  let newPattern = pattern;
+  Object.keys(map).forEach(key => {
+    const replaceFrom = brackets ? `{${key}}` : key;
+    const replaceTo = brackets ? `{${map[key]}}` : map[key];
+    if (exact) {
+      const exactMatch = replaceFrom === newPattern;
+      newPattern = exactMatch ? replaceTo : newPattern;
+    } else {
+      newPattern = newPattern.replace(replaceFrom, replaceTo);
+    }
+  });
+
+  return newPattern.replace(/^\//, '');
+};

packages/kbn-spec-to-console/lib/static/map_interpolation.json

@@ -0,0 +1,5 @@
+{
+  "index": "indices",
+  "node_id": "nodes",
+  "metric": "metrics"
+}

packages/kbn-spec-to-console/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "spec-to-console",
+  "version": "0.0.0",
+  "description": "ES REST spec -> Console autocomplete",
+  "main": "index.js",
+  "directories": {
+    "lib": "lib"
+  },
+  "scripts": {
+    "test": "jest",
+    "format": "prettier **/*.js --write"
+  },
+  "author": "",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/jbudz/spec-to-console/issues"
+  },
+  "homepage": "https://github.com/jbudz/spec-to-console#readme",
+  "devDependencies": {
+    "jest": "^24.1.0",
+    "prettier": "^1.14.3"
+  },
+  "dependencies": {
+    "commander": "^2.11.0",
+    "glob": "^7.1.2"
+  }
+}

packages/kbn-spec-to-console/test/fixtures/cluster_health_autocomplete.json

@@ -0,0 +1,39 @@
+{
+  "cluster.health": {
+    "url_params": {
+      "level": [
+        "cluster",
+        "indices",
+        "shards"
+      ],
+      "local": "__flag__",
+      "master_timeout": "",
+      "timeout": "",
+      "wait_for_active_shards": "",
+      "wait_for_nodes": "",
+      "wait_for_events": [
+        "immediate",
+        "urgent",
+        "high",
+        "normal",
+        "low",
+        "languid"
+      ],
+      "wait_for_no_relocating_shards": "__flag__",
+      "wait_for_no_initializing_shards": "__flag__",
+      "wait_for_status": [
+        "green",
+        "yellow",
+        "red"
+      ]
+    },
+    "methods": [
+      "GET"
+    ],
+    "patterns": [
+      "_cluster/health",
+      "_cluster/health/{indices}"
+    ],
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html"
+  }
+}

packages/kbn-spec-to-console/test/fixtures/cluster_health_spec.json

@@ -0,0 +1,64 @@
+{
+  "cluster.health": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html",
+    "methods": ["GET"],
+    "url": {
+      "path": "/_cluster/health",
+      "paths": ["/_cluster/health", "/_cluster/health/{index}"],
+      "parts": {
+        "index": {
+          "type" : "list",
+          "description" : "Limit the information returned to a specific index"
+        }
+      },
+      "params": {
+        "level": {
+          "type" : "enum",
+          "options" : ["cluster","indices","shards"],
+          "default" : "cluster",
+          "description" : "Specify the level of detail for returned information"
+        },
+        "local": {
+          "type" : "boolean",
+          "description" : "Return local information, do not retrieve the state from master node (default: false)"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout for connection to master node"
+        },
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "wait_for_active_shards": {
+          "type" : "string",
+          "description" : "Wait until the specified number of shards is active"
+        },
+        "wait_for_nodes": {
+          "type" : "string",
+          "description" : "Wait until the specified number of nodes is available"
+        },
+        "wait_for_events": {
+          "type" : "enum",
+          "options" : ["immediate", "urgent", "high", "normal", "low", "languid"],
+          "description" : "Wait until all currently queued events with the given priority are processed"
+        },
+        "wait_for_no_relocating_shards": {
+          "type" : "boolean",
+          "description" : "Whether to wait until there are no relocating shards in the cluster"
+        },
+        "wait_for_no_initializing_shards": {
+          "type" : "boolean",
+          "description" : "Whether to wait until there are no initializing shards in the cluster"
+        },
+        "wait_for_status": {
+          "type" : "enum",
+          "options" : ["green","yellow","red"],
+          "default" : null,
+          "description" : "Wait until cluster is in a specific state"
+        }
+      }
+    },
+    "body": null
+  }
+}

scripts/spec_to_console.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+require('../packages/kbn-spec-to-console/bin/spec_to_console');