Re-enable filter editor suggestions (#13376)

* Re-enable filter editor suggestions

* Use search instead of include

* Escape query

* Show spinner

* Use include rather than search

* Add additional regex and explanation for parameters

* Add suggestions API test

* Make sure test actually runs

* Use send instead of query

* Fix suggestions API test

docs/discover/field-filter.asciidoc

@@ -62,7 +62,8 @@ The following operators can be selected:
 `is not between`:: Filter where the value for the field is not in the given range.
 `exists`:: Filter where any value is present for the field.
 `does not exist`:: Filter where no value is present for the field.
-. Choose the value(s) for your filter.
+. Choose the value(s) for your filter. Values from your indices may be suggested
+as selections if you are filtering against an aggregatable field.
 +
 image::images/add_filter_value.png[]
 . (Optional) Specify a label for the filter. If you specify a label, it will be
@@ -70,9 +71,8 @@ displayed below the query bar instead of the filter definition.
 . Click *Save*. The filter will be applied to your search and be displayed below
 the query bar.
 
-NOTE: To make the filter editor more user-friendly, you can enable the `filterEditor:suggestValues` advanced setting.
-Enabling this will cause the editor to suggest values from your indices if you are filtering against an aggregatable
-field. However, this is not recommended for extremely large datasets, as it can result in long queries.
+NOTE: If you are experiencing long-running queries as a result of the value suggestions, you can
+turn off the suggestions by setting the advanced setting, `filterEditor:suggestValues`, to `false`.
 
 [float]
 [[filter-pinning]]

docs/management/advanced-options.asciidoc

@@ -74,7 +74,7 @@ mentioned use "_default_".
 `timepicker:refreshIntervalDefaults`:: The time filter's default refresh interval.
 `dashboard:defaultDarkTheme`:: Set this property to `true` to make new dashboards use the dark theme by default.
 `filters:pinnedByDefault`:: Set this property to `true` to make filters have a global state by default.
-`filterEditor:suggestValues`:: Set this property to `true` to have the filter editor suggest values for fields, instead of just providing a text input. This may result in heavy queries to Elasticsearch.
+`filterEditor:suggestValues`:: Set this property to `false` to prevent the filter editor from suggesting values for fields.
 `notifications:banner`:: You can specify a custom banner to display temporary notices to all users. This field supports
 Markdown.
 `notifications:lifetime:banner`:: Specifies the duration in milliseconds for banner notification displays. The default value is 3000000. Set this field to `Infinity` to disable banner notifications.

src/core_plugins/kibana/server/routes/api/suggestions/register_value_suggestions.js

@@ -4,33 +4,52 @@ export function registerValueSuggestions(server) {
   server.route({
     path: '/api/kibana/suggestions/values/{index}',
     method: ['POST'],
-    handler: function (req, reply) {
+    handler: async function (req, reply) {
       const { index } = req.params;
       const { field, query } = req.payload;
-
       const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
-      const include = query ? `.*${query}.*` : undefined;
-      const body = getBody({
-        field,
-        include,
-        shard_size: 10,
-        size: 10
-      });
-
-      return callWithRequest(req, 'search', { index, body })
-      .then((res) => {
-        const suggestions = res.aggregations.suggestions.buckets.map(bucket => bucket.key);
-        return reply(suggestions);
-      })
-      .catch(error => reply(handleESError(error)));
+      const body = getBody({ field, query });
+      try {
+        const response = await callWithRequest(req, 'search', { index, body });
+        const suggestions = response.aggregations.suggestions.buckets.map(bucket => bucket.key);
+        reply(suggestions);
+      } catch (error) {
+        reply(handleESError(error));
+      }
     }
   });
 }
 
-function getBody(terms) {
+function getBody({ field, query }) {
+  // Helps ensure that the regex is not evaluated eagerly against the terms dictionary
+  const executionHint = 'map';
+
+  // Helps keep the number of buckets that need to be tracked at the shard level contained in case
+  // this is a high cardinality field
+  const terminateAfter = 100000;
+
+  // We don't care about the accuracy of the counts, just the content of the terms, so this reduces
+  // the amount of information that needs to be transmitted to the coordinating node
+  const shardSize = 10;
+
   return {
+    size: 0,
+    timeout: '1s',
+    terminate_after: terminateAfter,
     aggs: {
-      suggestions: { terms }
+      suggestions: {
+        terms: {
+          field,
+          include: `${getEscapedQuery(query)}.*`,
+          execution_hint: executionHint,
+          shard_size: shardSize
+        }
+      }
     }
   };
 }
+
+function getEscapedQuery(query = '') {
+  // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html#_standard_operators
+  return query.replace(/[.?+*|{}[\]()"\\#@&<>~]/g, (match) => `\\${match}`);
+}

src/core_plugins/kibana/ui_setting_defaults.js

@@ -265,9 +265,8 @@ export function getUiSettingDefaults() {
       description: 'Whether the filters should have a global state (be pinned) by default'
     },
     'filterEditor:suggestValues': {
-      value: false,
-      description: 'Set this property to `true` to have the filter editor suggest values for fields, ' +
-        'instead of just providing a text input. This may result in heavy queries to Elasticsearch.'
+      value: true,
+      description: 'Set this property to `false` to prevent the filter editor from suggesting values for fields.'
     },
     'notifications:banner': {
       type: 'markdown',

src/ui/public/filter_editor/params_editor/filter_params_phrase_controller.js

@@ -32,7 +32,7 @@ export function filterParamsPhraseController($http, $scope, config) {
       .catch(() => []);
   }
 
-  function getFieldQueryHash(field, query) {
+  function getFieldQueryHash(field, query = '') {
     return `${field.indexPattern.id}/${field.name}/${query}`;
   }
 }

src/ui/public/filter_editor/params_editor/filter_params_phrase_editor.html

@@ -2,6 +2,8 @@
   ng-if="filterParamsPhraseEditor.shouldSuggestValues && field.aggregatable && field.type === 'string'"
   ng-model="params.phrase"
   ui-select-focus-on="focus-params"
+  spinner-enabled="true"
+  spinner-class="kuiIcon kuiIcon--basic fa-spinner fa-spin"
 >
   <ui-select-match placeholder="Values...">
     <span

src/ui/public/filter_editor/params_editor/filter_params_phrases_editor.html

@@ -2,6 +2,8 @@
   ng-model="params.phrases"
   multiple
   ui-select-focus-on="focus-params"
+  spinner-enabled="true"
+  spinner-class="kuiIcon kuiIcon--basic fa-spinner fa-spin"
 >
   <ui-select-match
     placeholder="Values..."

test/api_integration/apis/index.js

@@ -3,5 +3,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./index_patterns'));
     loadTestFile(require.resolve('./scripts'));
     loadTestFile(require.resolve('./search'));
+    loadTestFile(require.resolve('./suggestions'));
   });
 }

test/api_integration/apis/suggestions/index.js

@@ -0,0 +1,5 @@
+export default function ({ loadTestFile }) {
+  describe('suggestions', () => {
+    loadTestFile(require.resolve('./suggestions'));
+  });
+}

test/api_integration/apis/suggestions/suggestions.js

@@ -0,0 +1,19 @@
+export default function ({ getService }) {
+  const esArchiver = getService('esArchiver');
+  const supertest = getService('supertest');
+
+  describe('Suggestions API', function () {
+    before(() => esArchiver.load('index_patterns/basic_index'));
+    after(() => esArchiver.unload('index_patterns/basic_index'));
+
+    it('should return 200 with special characters', () => (
+      supertest
+        .post('/api/kibana/suggestions/values/basic_index')
+        .send({
+          field: 'baz.keyword',
+          query: '<something?with:lots&of^ bad characters'
+        })
+        .expect(200)
+    ));
+  });
+}

test/api_integration/fixtures/es_archiver/index_patterns/basic_index/mappings.json

@@ -30,4 +30,4 @@
       }
     }
   }
-}
+}