[Maps] get max_result_window and max_inner_result_window from index settings (#53500)

* [Maps] pull ES_SIZE_LIMIT and top hits limit from index settings

* get fetch working

* get min values from indicies response

* use indexSettings.maxResultWindow in documents request size

* use max_inner_result_window to define top hits max

* update jest test

* update docs

* more docs changes for top hits

* fix line spacing

* Update docs/maps/maps-aggregations.asciidoc

Co-Authored-By: gchaps <33642766+gchaps@users.noreply.github.com>

* Update docs/maps/vector-layer.asciidoc

Co-Authored-By: gchaps <33642766+gchaps@users.noreply.github.com>

* add api integration test for indexSettings route

* eslint fixes

* review feedback

* display toast on first index settings fetch failure

* clean up

Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/maps/maps-aggregations.asciidoc

@@ -47,6 +47,7 @@ To enable top hits:
 . Set *Entity* to the field that identifies entities in your documents.
 This field will be used in the terms aggregation to group your documents into entity buckets.
 . Set *Documents per entity* to configure the maximum number of documents accumulated per entity.
+This setting is limited to the `index.max_inner_result_window` index setting, which defaults to 100.
 
 [role="screenshot"]
 image::maps/images/top_hits.png[]

docs/maps/vector-layer.asciidoc

@@ -15,7 +15,7 @@ See map.regionmap.* in <<settings>> for details.
 *Documents*:: Vector data from a Kibana index pattern.
 The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape].
 
-NOTE: Document results are limited to the first 10000 matching documents.
+NOTE: Document results are limited to the `index.max_result_window` index setting, which defaults to 10000.
 Use <<maps-aggregations, aggregations>> to plot large data sets.
 
 *Grid aggregation*:: Geospatial data grouped in grids with metrics for each gridded cell.

x-pack/legacy/plugins/maps/common/constants.js

@@ -26,6 +26,7 @@ export const APP_ICON = 'gisApp';
 
 export const MAP_APP_PATH = `app/${APP_ID}`;
 export const GIS_API_PATH = `api/${APP_ID}`;
+export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`;
 
 export const MAP_BASE_URL = `/${MAP_APP_PATH}#/${MAP_SAVED_OBJECT_TYPE}`;
 
@@ -69,7 +70,9 @@ export const MAX_ZOOM = 24;
 
 export const DECIMAL_DEGREES_PRECISION = 5; // meters precision
 export const ZOOM_PRECISION = 2;
-export const ES_SIZE_LIMIT = 10000;
+export const DEFAULT_MAX_RESULT_WINDOW = 10000;
+export const DEFAULT_MAX_INNER_RESULT_WINDOW = 100;
+export const DEFAULT_MAX_BUCKETS_LIMIT = 10000;
 
 export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__';
 export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn_isvisibleduetojoin__';

x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js

@@ -6,7 +6,11 @@
 
 import { AbstractVectorSource } from '../vector_source';
 import React from 'react';
-import { ES_GEO_FIELD_TYPE, GEOJSON_FILE, ES_SIZE_LIMIT } from '../../../../common/constants';
+import {
+  ES_GEO_FIELD_TYPE,
+  GEOJSON_FILE,
+  DEFAULT_MAX_RESULT_WINDOW,
+} from '../../../../common/constants';
 import { ClientFileCreateSourceEditor } from './create_client_file_source_editor';
 import { ESSearchSource } from '../es_search_source';
 import uuid from 'uuid/v4';
@@ -82,7 +86,7 @@ export class GeojsonFileSource extends AbstractVectorSource {
         addAndViewSource(null);
       } else {
         // Only turn on bounds filter for large doc counts
-        const filterByMapBounds = indexDataResp.docCount > ES_SIZE_LIMIT;
+        const filterByMapBounds = indexDataResp.docCount > DEFAULT_MAX_RESULT_WINDOW;
         const source = new ESSearchSource(
           {
             id: uuid(),

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js

@@ -15,7 +15,11 @@ import { NoIndexPatternCallout } from '../../../components/no_index_pattern_call
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
 import { kfetch } from 'ui/kfetch';
-import { ES_GEO_FIELD_TYPE, GIS_API_PATH, ES_SIZE_LIMIT } from '../../../../common/constants';
+import {
+  ES_GEO_FIELD_TYPE,
+  GIS_API_PATH,
+  DEFAULT_MAX_RESULT_WINDOW,
+} from '../../../../common/constants';
 import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
 
 import { npStart } from 'ui/new_platform';
@@ -96,7 +100,7 @@ export class CreateSourceEditor extends Component {
     let indexHasSmallDocCount = false;
     try {
       const indexDocCount = await this.loadIndexDocCount(indexPattern.title);
-      indexHasSmallDocCount = indexDocCount <= ES_SIZE_LIMIT;
+      indexHasSmallDocCount = indexDocCount <= DEFAULT_MAX_RESULT_WINDOW;
     } catch (error) {
       // retrieving index count is a nice to have and is not essential
       // do not interrupt user flow if unable to retrieve count

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js

@@ -17,12 +17,13 @@ import { UpdateSourceEditor } from './update_source_editor';
 import {
   ES_SEARCH,
   ES_GEO_FIELD_TYPE,
-  ES_SIZE_LIMIT,
+  DEFAULT_MAX_BUCKETS_LIMIT,
   SORT_ORDER,
 } from '../../../../common/constants';
 import { i18n } from '@kbn/i18n';
 import { getDataSourceLabel } from '../../../../common/i18n_getters';
 import { getSourceFields } from '../../../index_pattern_util';
+import { loadIndexSettings } from './load_index_settings';
 
 import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
 import { ESDocField } from '../../fields/es_doc_field';
@@ -267,8 +268,8 @@ export class ESSearchSource extends AbstractESSource {
       entitySplit: {
         terms: {
           field: topHitsSplitField,
-          size: ES_SIZE_LIMIT,
-          shard_size: ES_SIZE_LIMIT,
+          size: DEFAULT_MAX_BUCKETS_LIMIT,
+          shard_size: DEFAULT_MAX_BUCKETS_LIMIT,
         },
         aggs: {
           entityHits: {
@@ -290,7 +291,7 @@ export class ESSearchSource extends AbstractESSource {
     const entityBuckets = _.get(resp, 'aggregations.entitySplit.buckets', []);
     const totalEntities = _.get(resp, 'aggregations.totalEntities.value', 0);
     // can not compare entityBuckets.length to totalEntities because totalEntities is an approximate
-    const areEntitiesTrimmed = entityBuckets.length >= ES_SIZE_LIMIT;
+    const areEntitiesTrimmed = entityBuckets.length >= DEFAULT_MAX_BUCKETS_LIMIT;
     let areTopHitsTrimmed = false;
     entityBuckets.forEach(entityBucket => {
       const total = _.get(entityBucket, 'entityHits.hits.total', 0);
@@ -315,7 +316,7 @@ export class ESSearchSource extends AbstractESSource {
 
   // searchFilters.fieldNames contains geo field and any fields needed for styling features
   // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size
-  async _getSearchHits(layerName, searchFilters, registerCancelCallback) {
+  async _getSearchHits(layerName, searchFilters, maxResultWindow, registerCancelCallback) {
     const initialSearchContext = {
       docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames),
     };
@@ -331,7 +332,7 @@ export class ESSearchSource extends AbstractESSource {
       );
       searchSource = await this._makeSearchSource(
         searchFilters,
-        ES_SIZE_LIMIT,
+        maxResultWindow,
         initialSearchContext
       );
       searchSource.setField('source', false); // do not need anything from _source
@@ -340,7 +341,7 @@ export class ESSearchSource extends AbstractESSource {
       // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source
       searchSource = await this._makeSearchSource(
         searchFilters,
-        ES_SIZE_LIMIT,
+        maxResultWindow,
         initialSearchContext
       );
       // Setting "fields" instead of "source: { includes: []}"
@@ -382,11 +383,19 @@ export class ESSearchSource extends AbstractESSource {
   }
 
   async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
+    const indexPattern = await this.getIndexPattern();
+
+    const indexSettings = await loadIndexSettings(indexPattern.title);
+
     const { hits, meta } = this._isTopHits()
       ? await this._getTopHits(layerName, searchFilters, registerCancelCallback)
-      : await this._getSearchHits(layerName, searchFilters, registerCancelCallback);
+      : await this._getSearchHits(
+          layerName,
+          searchFilters,
+          indexSettings.maxResultWindow,
+          registerCancelCallback
+        );
 
-    const indexPattern = await this.getIndexPattern();
     const unusedMetaFields = indexPattern.metaFields.filter(metaField => {
       return !['_id', '_index'].includes(metaField);
     });

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/load_index_settings.js

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  DEFAULT_MAX_RESULT_WINDOW,
+  DEFAULT_MAX_INNER_RESULT_WINDOW,
+  INDEX_SETTINGS_API_PATH,
+} from '../../../../common/constants';
+import { kfetch } from 'ui/kfetch';
+import { toastNotifications } from 'ui/notify';
+import { i18n } from '@kbn/i18n';
+
+let toastDisplayed = false;
+const indexSettings = new Map();
+
+export async function loadIndexSettings(indexPatternTitle) {
+  if (indexSettings.has(indexPatternTitle)) {
+    return indexSettings.get(indexPatternTitle);
+  }
+
+  const fetchPromise = fetchIndexSettings(indexPatternTitle);
+  indexSettings.set(indexPatternTitle, fetchPromise);
+  return fetchPromise;
+}
+
+async function fetchIndexSettings(indexPatternTitle) {
+  try {
+    const indexSettings = await kfetch({
+      pathname: `../${INDEX_SETTINGS_API_PATH}`,
+      query: {
+        indexPatternTitle,
+      },
+    });
+    return indexSettings;
+  } catch (err) {
+    const warningMsg = i18n.translate('xpack.maps.indexSettings.fetchErrorMsg', {
+      defaultMessage: `Unable to fetch index settings for index pattern '{indexPatternTitle}'.
+      Ensure you have '{viewIndexMetaRole}' role.`,
+      values: {
+        indexPatternTitle,
+        viewIndexMetaRole: 'view_index_metadata',
+      },
+    });
+    if (!toastDisplayed) {
+      // Only show toast for first failure to avoid flooding user with warnings
+      toastDisplayed = true;
+      toastNotifications.addWarning(warningMsg);
+    }
+    console.warn(warningMsg);
+    return {
+      maxResultWindow: DEFAULT_MAX_RESULT_WINDOW,
+      maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW,
+    };
+  }
+}

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.js

@@ -22,9 +22,10 @@ import { indexPatternService } from '../../../kibana_services';
 import { i18n } from '@kbn/i18n';
 import { getTermsFields, getSourceFields } from '../../../index_pattern_util';
 import { ValidatedRange } from '../../../components/validated_range';
-import { SORT_ORDER } from '../../../../common/constants';
+import { DEFAULT_MAX_INNER_RESULT_WINDOW, SORT_ORDER } from '../../../../common/constants';
 import { ESDocField } from '../../fields/es_doc_field';
 import { FormattedMessage } from '@kbn/i18n/react';
+import { loadIndexSettings } from './load_index_settings';
 
 export class UpdateSourceEditor extends Component {
   static propTypes = {
@@ -43,17 +44,31 @@ export class UpdateSourceEditor extends Component {
     sourceFields: null,
     termFields: null,
     sortFields: null,
+    maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW,
   };
 
   componentDidMount() {
     this._isMounted = true;
     this.loadFields();
+    this.loadIndexSettings();
   }
 
   componentWillUnmount() {
     this._isMounted = false;
   }
 
+  async loadIndexSettings() {
+    try {
+      const indexPattern = await indexPatternService.get(this.props.indexPatternId);
+      const { maxInnerResultWindow } = await loadIndexSettings(indexPattern.title);
+      if (this._isMounted) {
+        this.setState({ maxInnerResultWindow });
+      }
+    } catch (err) {
+      return;
+    }
+  }
+
   async loadFields() {
     let indexPattern;
     try {
@@ -149,7 +164,7 @@ export class UpdateSourceEditor extends Component {
         >
           <ValidatedRange
             min={1}
-            max={100}
+            max={this.state.maxInnerResultWindow}
             step={1}
             value={this.props.topHitsSize}
             onChange={this.onTopHitsSizeChange}

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.test.js

@@ -6,6 +6,12 @@
 
 jest.mock('../../../kibana_services', () => ({}));
 
+jest.mock('./load_index_settings', () => ({
+  loadIndexSettings: async () => {
+    return { maxInnerResultWindow: 100 };
+  },
+}));
+
 import React from 'react';
 import { shallow } from 'enzyme';
 

x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js

@@ -9,7 +9,7 @@ import _ from 'lodash';
 import { Schemas } from 'ui/vis/editors/default/schemas';
 import { AggConfigs } from 'ui/agg_types';
 import { i18n } from '@kbn/i18n';
-import { ES_SIZE_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants';
+import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants';
 import { ESDocField } from '../fields/es_doc_field';
 import { AbstractESAggSource } from './es_agg_source';
 
@@ -170,7 +170,7 @@ export class ESTermSource extends AbstractESAggSource {
         schema: 'segment',
         params: {
           field: this._termField.getName(),
-          size: ES_SIZE_LIMIT,
+          size: DEFAULT_MAX_BUCKETS_LIMIT,
         },
       },
     ];

x-pack/legacy/plugins/maps/public/layers/vector_layer.js

@@ -373,7 +373,6 @@ export class VectorLayer extends AbstractLayer {
     const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`);
     const searchFilters = this._getSearchFilters(dataFilters);
     const prevDataRequest = this.getSourceDataRequest();
-
     const canSkipFetch = await canSkipSourceUpdate({
       source: this._source,
       prevDataRequest,

x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.js

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import { DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../common/constants';
+
+export function getIndexPatternSettings(indicesSettingsResp) {
+  let maxResultWindow = Infinity;
+  let maxInnerResultWindow = Infinity;
+  Object.values(indicesSettingsResp).forEach(indexSettings => {
+    const indexMaxResultWindow = _.get(
+      indexSettings,
+      'settings.index.max_result_window',
+      DEFAULT_MAX_RESULT_WINDOW
+    );
+    maxResultWindow = Math.min(maxResultWindow, indexMaxResultWindow);
+
+    const indexMaxInnerResultWindow = _.get(
+      indexSettings,
+      'settings.index.max_inner_result_window',
+      DEFAULT_MAX_INNER_RESULT_WINDOW
+    );
+    maxInnerResultWindow = Math.min(indexMaxInnerResultWindow, indexMaxResultWindow);
+  });
+
+  return { maxResultWindow, maxInnerResultWindow };
+}

x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.test.js

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getIndexPatternSettings } from './get_index_pattern_settings';
+import { DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../common/constants';
+
+describe('max_result_window and max_inner_result_window are not set', () => {
+  test('Should provide default values when values not set', () => {
+    const indicesSettingsResp = {
+      kibana_sample_data_logs: {
+        settings: {
+          index: {},
+        },
+      },
+    };
+    const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp);
+    expect(maxResultWindow).toBe(DEFAULT_MAX_RESULT_WINDOW);
+    expect(maxInnerResultWindow).toBe(DEFAULT_MAX_INNER_RESULT_WINDOW);
+  });
+
+  test('Should include default values when providing minimum values for indices in index pattern', () => {
+    const indicesSettingsResp = {
+      kibana_sample_data_logs: {
+        settings: {
+          index: {
+            max_result_window: '15000',
+            max_inner_result_window: '200',
+          },
+        },
+      },
+      kibana_sample_data_flights: {
+        settings: {
+          index: {},
+        },
+      },
+    };
+    const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp);
+    expect(maxResultWindow).toBe(DEFAULT_MAX_RESULT_WINDOW);
+    expect(maxInnerResultWindow).toBe(DEFAULT_MAX_INNER_RESULT_WINDOW);
+  });
+});
+
+describe('max_result_window and max_inner_result_window are set', () => {
+  test('Should provide values from settings', () => {
+    const indicesSettingsResp = {
+      kibana_sample_data_logs: {
+        settings: {
+          index: {
+            max_result_window: '15000', // value is returned as string API
+            max_inner_result_window: '200',
+          },
+        },
+      },
+    };
+    const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp);
+    expect(maxResultWindow).toBe(15000);
+    expect(maxInnerResultWindow).toBe(200);
+  });
+
+  test('Should provide minimum values for indices in index pattern', () => {
+    const indicesSettingsResp = {
+      kibana_sample_data_logs: {
+        settings: {
+          index: {
+            max_result_window: '15000',
+            max_inner_result_window: '200',
+          },
+        },
+      },
+      kibana_sample_data_flights: {
+        settings: {
+          index: {
+            max_result_window: '7000',
+            max_inner_result_window: '75',
+          },
+        },
+      },
+    };
+    const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp);
+    expect(maxResultWindow).toBe(7000);
+    expect(maxInnerResultWindow).toBe(75);
+  });
+});

x-pack/legacy/plugins/maps/server/routes.js

@@ -17,10 +17,12 @@ import {
   EMS_TILES_VECTOR_TILE_PATH,
   GIS_API_PATH,
   EMS_SPRITES_PATH,
+  INDEX_SETTINGS_API_PATH,
 } from '../common/constants';
 import { EMSClient } from '@elastic/ems-client';
 import fetch from 'node-fetch';
 import { i18n } from '@kbn/i18n';
+import { getIndexPatternSettings } from './lib/get_index_pattern_settings';
 
 import Boom from 'boom';
 
@@ -414,6 +416,33 @@ export function initRoutes(server, licenseUid) {
     },
   });
 
+  server.route({
+    method: 'GET',
+    path: `/${INDEX_SETTINGS_API_PATH}`,
+    handler: async (request, h) => {
+      const { server, query } = request;
+
+      if (!query.indexPatternTitle) {
+        server.log('warning', `Required query parameter 'indexPatternTitle' not provided.`);
+        return h.response().code(400);
+      }
+
+      const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
+      try {
+        const resp = await callWithRequest(request, 'indices.getSettings', {
+          index: query.indexPatternTitle,
+        });
+        return getIndexPatternSettings(resp);
+      } catch (error) {
+        server.log(
+          'warning',
+          `Cannot load index settings for index pattern '${query.indexPatternTitle}', error: ${error.message}.`
+        );
+        return h.response().code(400);
+      }
+    },
+  });
+
   function checkEMSProxyConfig() {
     if (!mapConfig.proxyElasticMapsServiceInMaps) {
       server.log(

x-pack/test/api_integration/apis/maps/index.js

@@ -4,8 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export default function({ loadTestFile }) {
+export default function({ loadTestFile, getService }) {
+  const esArchiver = getService('esArchiver');
+
   describe('Maps endpoints', () => {
-    loadTestFile(require.resolve('./migrations'));
+    before(async () => {
+      await esArchiver.loadIfNeeded('logstash_functional');
+    });
+
+    describe('', () => {
+      loadTestFile(require.resolve('./index_settings'));
+      loadTestFile(require.resolve('./migrations'));
+    });
   });
 }

x-pack/test/api_integration/apis/maps/index_settings.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService }) {
+  const supertest = getService('supertest');
+
+  describe('index settings', () => {
+    it('should return index settings', async () => {
+      const resp = await supertest
+        .get(`/api/maps/indexSettings?indexPatternTitle=logstash*`)
+        .set('kbn-xsrf', 'kibana')
+        .expect(200);
+
+      expect(resp.body.maxResultWindow).to.be(10000);
+      expect(resp.body.maxInnerResultWindow).to.be(100);
+    });
+  });
+}