[Console] Console with better SQL support (#51446)

* First iteration of PoC for SQL highlighting and better completion

* Alternate implementation (no """sql markers)

* xLang markers

* - Remove 'start-sql'
- no flow :c

* Revert "- Remove 'start-sql'"

This reverts commit 2d585eef17.

* Revert "xLang markers"

This reverts commit f019616675.

* Revert "xLang markers" and add comment

This reverts commit f019616675.

* Add elasticsearch sql highlight rules

* Add links to sources of data

* Update colors

* Redo built in functions

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/elasticsearch_sql_highlight_rules.ts

@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import ace from 'brace';
+
+const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
+const oop = ace.acequire('ace/lib/oop');
+
+export const ElasticsearchSqlHighlightRules = function(this: any) {
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-commands.html
+  const keywords =
+    'describe|between|in|like|not|and|or|desc|select|from|where|having|group|by|order' +
+    'asc|desc|pivot|for|in|as|show|columns|include|frozen|tables|escape|limit|rlike|all|distinct|is';
+
+  const builtinConstants = 'true|false';
+
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-syntax-show-functions.html
+  const builtinFunctions =
+    'avg|count|first|first_value|last|last_value|max|min|sum|kurtosis|mad|percentile|percentile_rank|skewness' +
+    '|stddev_pop|sum_of_squares|var_pop|histogram|case|coalesce|greatest|ifnull|iif|isnull|least|nullif|nvl' +
+    '|curdate|current_date|current_time|current_timestamp|curtime|dateadd|datediff|datepart|datetrunc|date_add' +
+    '|date_diff|date_part|date_trunc|day|dayname|dayofmonth|dayofweek|dayofyear|day_name|day_of_month|day_of_week' +
+    '|day_of_year|dom|dow|doy|hour|hour_of_day|idow|isodayofweek|isodow|isoweek|isoweekofyear|iso_day_of_week|iso_week_of_year' +
+    '|iw|iwoy|minute|minute_of_day|minute_of_hour|month|monthname|month_name|month_of_year|now|quarter|second|second_of_minute' +
+    '|timestampadd|timestampdiff|timestamp_add|timestamp_diff|today|week|week_of_year|year|abs|acos|asin|atan|atan2|cbrt' +
+    '|ceil|ceiling|cos|cosh|cot|degrees|e|exp|expm1|floor|log|log10|mod|pi|power|radians|rand|random|round|sign|signum|sin' +
+    '|sinh|sqrt|tan|truncate|ascii|bit_length|char|character_length|char_length|concat|insert|lcase|left|length|locate' +
+    '|ltrim|octet_length|position|repeat|replace|right|rtrim|space|substring|ucase|cast|convert|database|user|st_astext|st_aswkt' +
+    '|st_distance|st_geometrytype|st_geomfromtext|st_wkttosql|st_x|st_y|st_z|score';
+
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-data-types.html
+  const dataTypes =
+    'null|boolean|byte|short|integer|long|double|float|half_float|scaled_float|keyword|text|binary|date|ip|object|nested|time' +
+    '|interval_year|interval_month|interval_day|interval_hour|interval_minute|interval_second|interval_year_to_month' +
+    'inteval_day_to_hour|interval_day_to_minute|interval_day_to_second|interval_hour_to_minute|interval_hour_to_second' +
+    'interval_minute_to_second|geo_point|geo_shape|shape';
+
+  const keywordMapper = this.createKeywordMapper(
+    {
+      keyword: [keywords, builtinFunctions, builtinConstants, dataTypes].join('|'),
+    },
+    'identifier',
+    true
+  );
+
+  this.$rules = {
+    start: [
+      {
+        token: 'comment',
+        regex: '--.*$',
+      },
+      {
+        token: 'comment',
+        start: '/\\*',
+        end: '\\*/',
+      },
+      {
+        token: 'string', // " string
+        regex: '".*?"',
+      },
+      {
+        token: 'constant', // ' string
+        regex: "'.*?'",
+      },
+      {
+        token: 'string', // ` string (apache drill)
+        regex: '`.*?`',
+      },
+      {
+        token: 'entity.name.function', // float
+        regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',
+      },
+      {
+        token: keywordMapper,
+        regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b',
+      },
+      {
+        token: 'keyword.operator',
+        regex: '⇐|<⇒|\\*|\\.|\\:\\:|\\+|\\-|\\/|\\/\\/|%|&|\\^|~|<|>|<=|=>|==|!=|<>|=',
+      },
+      {
+        token: 'paren.lparen',
+        regex: '[\\(]',
+      },
+      {
+        token: 'paren.rparen',
+        regex: '[\\)]',
+      },
+      {
+        token: 'text',
+        regex: '\\s+',
+      },
+    ],
+  };
+  this.normalizeRules();
+};
+
+oop.inherits(ElasticsearchSqlHighlightRules, TextHighlightRules);

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/input_highlight_rules.js

@@ -20,14 +20,7 @@
 const ace = require('brace');
 import { addToRules } from './x_json_highlight_rules';
 
-const oop = ace.acequire('ace/lib/oop');
+export function addEOL(tokens, reg, nextIfEOL, normalNext) {
-const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
-export function InputHighlightRules() {
-  function mergeTokens(/* ... */) {
-    return [].concat.apply([], arguments);
-  }
-
-  function addEOL(tokens, reg, nextIfEOL, normalNext) {
   if (typeof reg === 'object') {
     reg = reg.source;
   }
@@ -37,10 +30,23 @@ export function InputHighlightRules() {
   ];
 }
 
+export function mergeTokens(/* ... */) {
+  return [].concat.apply([], arguments);
+}
+
+const oop = ace.acequire('ace/lib/oop');
+const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
+
+export function InputHighlightRules() {
   // regexp must not have capturing parentheses. Use (?:) instead.
   // regexps are ordered -> the first match is used
   /*jshint -W015 */
   this.$rules = {
+    'start-sql': [
+      { token: 'whitespace', regex: '\\s+' },
+      { token: 'paren.lparen', regex: '{', next: 'json-sql', push: true },
+      { regex: '', next: 'start' }
+    ],
     'start': mergeTokens([
       { 'token': 'warning', 'regex': '#!.*$' },
       { token: 'comment', regex: /^#.*$/ },
@@ -65,6 +71,7 @@ export function InputHighlightRules() {
       addEOL(['whitespace'], /(\s+)/, 'start', 'url')
     ),
     'url': mergeTokens(
+      addEOL(['url.part'], /(_sql)/, 'start-sql', 'url-sql'),
       addEOL(['url.part'], /([^?\/,\s]+)/, 'start'),
       addEOL(['url.comma'], /(,)/, 'start'),
       addEOL(['url.slash'], /(\/)/, 'start'),
@@ -74,7 +81,17 @@ export function InputHighlightRules() {
       addEOL(['url.param', 'url.equal', 'url.value'], /([^&=]+)(=)([^&]*)/, 'start'),
       addEOL(['url.param'], /([^&=]+)/, 'start'),
       addEOL(['url.amp'], /(&)/, 'start')
-    )
+    ),
+    'url-sql': mergeTokens(
+      addEOL(['url.comma'], /(,)/, 'start-sql'),
+      addEOL(['url.slash'], /(\/)/, 'start-sql'),
+      addEOL(['url.questionmark'], /(\?)/, 'start-sql', 'urlParams-sql')
+    ),
+    'urlParams-sql': mergeTokens(
+      addEOL(['url.param', 'url.equal', 'url.value'], /([^&=]+)(=)([^&]*)/, 'start-sql'),
+      addEOL(['url.param'], /([^&=]+)/, 'start-sql'),
+      addEOL(['url.amp'], /(&)/, 'start-sql')
+    ),
   };
 
   addToRules(this);

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/x_json_highlight_rules.js

@@ -18,12 +18,14 @@
  */
 
 const _ = require('lodash');
-const ScriptHighlightRules = require('./script_highlight_rules').ScriptHighlightRules;
+
+import { ElasticsearchSqlHighlightRules } from './elasticsearch_sql_highlight_rules';
+const { ScriptHighlightRules } = require('./script_highlight_rules');
 
 const jsonRules = function (root) {
   root = root ? root : 'json';
   const rules = {};
-  rules[root] = [
+  const xJsonRules = [
     {
       token: ['variable', 'whitespace', 'ace.punctuation.colon', 'whitespace', 'punctuation.start_triple_quote'],
       regex: '("(?:[^"]*_)?script"|"inline"|"source")(\\s*?)(:)(\\s*?)(""")',
@@ -106,6 +108,16 @@ const jsonRules = function (root) {
       regex: '.+?'
     }
   ];
+
+  rules[root] = xJsonRules;
+  rules[root + '-sql'] = [{
+    token: ['variable', 'whitespace', 'ace.punctuation.colon', 'whitespace', 'punctuation.start_triple_quote'],
+    regex: '("query")(\\s*?)(:)(\\s*?)(""")',
+    next: 'sql-start',
+    merge: false,
+    push: true
+  }].concat(xJsonRules);
+
   rules.string_literal = [
     {
       token: 'punctuation.end_triple_quote',
@@ -127,4 +139,9 @@ export function addToRules(otherRules, embedUnder) {
     regex: '"""',
     next: 'pop',
   }]);
+  otherRules.embedRules(ElasticsearchSqlHighlightRules, 'sql-', [{
+    token: 'punctuation.end_triple_quote',
+    regex: '"""',
+    next: 'pop',
+  }]);
 }

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/autocomplete.ts

@@ -990,7 +990,7 @@ export default function({ coreEditor: editor, parser }: { coreEditor: CoreEditor
               score: 0,
               context,
             };
-            // we only need out custom insertMatch behavior for the body
+            // we only need our custom insertMatch behavior for the body
             if (context.autoCompleteType === 'body') {
               defaults.completer = {
                 insertMatch() {

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/components/full_request_component.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// @ts-ignore
+import { ConstantComponent } from './constant_component';
+
+export class FullRequestComponent extends ConstantComponent {
+  private readonly name: string;
+  constructor(name: string, parent: any, private readonly template: string) {
+    super(name, parent);
+    this.name = name;
+  }
+
+  getTerms() {
+    return [{ name: this.name, snippet: this.template }];
+  }
+}

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/components/url_pattern_matcher.js

@@ -25,6 +25,8 @@ import {
   SimpleParamComponent,
 } from './index';
 
+import { FullRequestComponent } from './full_request_component';
+
 /**
  * @param parametrizedComponentFactories a dict of the following structure
  * that will be used as a fall back for pattern parameters (i.e.: {indices})
@@ -54,6 +56,9 @@ export class UrlPatternMatcher {
     endpoint.methods.forEach((method) => {
       let c;
       let activeComponent = this[method].rootComponent;
+      if (endpoint.template) {
+        new FullRequestComponent(pattern + '[body]', activeComponent, endpoint.template);
+      }
       const endpointComponents = endpoint.url_components || {};
       const partList = pattern.split('/');
       _.each(

src/legacy/core_plugins/console/public/np_ready/lib/row_parser.ts

@@ -44,7 +44,10 @@ export default class RowParser {
       return MODE.BETWEEN_REQUESTS;
     } // shouldn't really happen
 
-    if (mode !== 'start') {
+    // If another "start" mode is added here because we want to allow for new language highlighting
+    // please see https://github.com/elastic/kibana/pull/51446 for a discussion on why
+    // should consider a different approach.
+    if (mode !== 'start' && mode !== 'start-sql') {
       return MODE.IN_REQUEST;
     }
     let line = (this.editor.getLineValue(lineNumber) || '').trim();

x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json

@@ -18,6 +18,7 @@
         "cbor",
         "smile"
       ]
-    }
+    },
+    "template": "_sql?format=json\n{\n  \"query\": \"\"\"\n  SELECT * FROM \"${1:TABLE}\"\n  \"\"\"\n}\n"
   }
 }