[Maps] Use geohash precision, not zoom-level, to rerequest data (#28537) (#29103)

Use a change in geohash precision to determine if data needs to be rerequested, instead of a change in zoom. Also add tests for generic zoom-level change behavior for es_geohash_grids.

x-pack/plugins/gis/public/shared/layers/heatmap_layer.js

@@ -9,7 +9,7 @@ import React from 'react';
 import { ALayer } from './layer';
 import { EuiIcon } from '@elastic/eui';
 import { HeatmapStyle } from './styles/heatmap_style';
-import { ZOOM_TO_PRECISION } from '../utils/zoom_to_precision';
+import { getGeohashPrecisionForZoom } from '../utils/zoom_to_precision';
 
 const SCALED_PROPERTY_NAME = '__kbn_heatmap_weight__';//unique name to store scaled value for weighting
 
@@ -105,7 +105,8 @@ export class HeatmapLayer extends ALayer {
     const sourceDataRequest = this.getSourceDataRequest();
     const dataMeta = sourceDataRequest ? sourceDataRequest.getMeta() : {};
 
-    const targetPrecision = ZOOM_TO_PRECISION[Math.round(dataFilters.zoom)] + this._style.getPrecisionRefinementDelta();
+    const targetPrecisionUnadjusted = getGeohashPrecisionForZoom(dataFilters.zoom);
+    const targetPrecision = targetPrecisionUnadjusted + this._style.getPrecisionRefinementDelta();
     const isSamePrecision = dataMeta.precision === targetPrecision;
 
     const isSameTime = _.isEqual(dataMeta.timeFilters, dataFilters.timeFilters);
@@ -139,13 +140,13 @@ export class HeatmapLayer extends ALayer {
   }
 
   async _fetchNewData({ startLoading, stopLoading, onLoadError, dataMeta }) {
-    const { precision, timeFilters, buffer, query } = dataMeta;
+    const { precision: geohashPrecision, timeFilters, buffer, query } = dataMeta;
     const requestToken = Symbol(`layer-source-refresh: this.getId()`);
     startLoading('source', requestToken, dataMeta);
     try {
       const layerName = await this.getDisplayName();
       const data = await this._source.getGeoJsonPoints({
-        precision,
+        geohashPrecision: geohashPrecision,
         extent: buffer,
         timeFilters,
         layerName,

x-pack/plugins/gis/public/shared/layers/layer.js

@@ -130,11 +130,11 @@ export class ALayer {
 
   renderSourceDetails = () => {
     return this._source.renderDetails();
-  }
+  };
 
   renderSourceSettingsEditor = ({ onChange }) => {
     return this._source.renderSourceSettingsEditor({ onChange });
-  }
+  };
 
   isLayerLoading() {
     return this._dataRequests.some(dataRequest => dataRequest.isLoading());
@@ -191,7 +191,9 @@ export class ALayer {
       newBuffer.maxLat
     ]);
     const doesPreviousBufferContainNewBuffer = turfBooleanContains(previousBufferGeometry, newBufferGeometry);
-    return doesPreviousBufferContainNewBuffer && !_.get(meta, 'areResultsTrimmed', false)
+
+    const isTrimmed = _.get(meta, 'areResultsTrimmed', false);
+    return doesPreviousBufferContainNewBuffer && !isTrimmed
       ? NO_SOURCE_UPDATE_REQUIRED
       : SOURCE_UPDATE_REQUIRED;
   }

x-pack/plugins/gis/public/shared/layers/sources/es_geohashgrid_source/es_geohashgrid_source.js

@@ -24,7 +24,6 @@ import { AggConfigs } from 'ui/vis/agg_configs';
 import { tabifyAggResponse } from 'ui/agg_response/tabify';
 import { convertToGeoJson } from './convert_to_geojson';
 import { ESSourceDetails } from '../../../components/es_source_details';
-import { ZOOM_TO_PRECISION } from '../../../utils/zoom_to_precision';
 import { VectorStyle } from '../../styles/vector_style';
 import { RENDER_AS } from './render_as';
 import { CreateSourceEditor } from './create_source_editor';
@@ -108,32 +107,6 @@ export class ESGeohashGridSource extends VectorSource {
     inspectorAdapters.requests.resetRequest(this._descriptor.id);
   }
 
-  async getGeoJsonWithMeta({ layerName }, searchFilters) {
-    let targetPrecision = ZOOM_TO_PRECISION[Math.round(searchFilters.zoom)];
-    targetPrecision += 0;//should have refinement param, similar to heatmap style
-    const featureCollection = await this.getGeoJsonPoints({
-      precision: targetPrecision,
-      extent: searchFilters.buffer,
-      timeFilters: searchFilters.timeFilters,
-      layerName,
-      query: searchFilters.query,
-    });
-
-    if (this._descriptor.requestType === RENDER_AS.GRID) {
-      featureCollection.features.forEach((feature) => {
-        //replace geometries with the polygon
-        feature.geometry = makeGeohashGridPolygon(feature);
-      });
-    }
-
-    return {
-      data: featureCollection,
-      meta: {
-        areResultsTrimmed: true
-      }
-    };
-  }
-
   isFieldAware() {
     return true;
   }
@@ -156,13 +129,41 @@ export class ESGeohashGridSource extends VectorSource {
     return  [this._descriptor.indexPatternId];
   }
 
+  isGeohashPrecisionAware() {
+    return true;
+  }
+
+  async getGeoJsonWithMeta({ layerName }, searchFilters) {
+    const featureCollection = await this.getGeoJsonPoints({
+      geohashPrecision: searchFilters.geohashPrecision,
+      extent: searchFilters.buffer,
+      timeFilters: searchFilters.timeFilters,
+      layerName,
+      query: searchFilters.query,
+    });
+
+    if (this._descriptor.requestType === RENDER_AS.GRID) {
+      featureCollection.features.forEach((feature) => {
+        //replace geometries with the polygon
+        feature.geometry = makeGeohashGridPolygon(feature);
+      });
+    }
+
+    return {
+      data: featureCollection,
+      meta: {
+        areResultsTrimmed: false
+      }
+    };
+  }
+
   async getNumberFields() {
     return this.getMetricFields().map(({ propertyKey: name, propertyLabel: label }) => {
       return { label, name };
     });
   }
 
-  async getGeoJsonPoints({ precision, extent, timeFilters, layerName, query }) {
+  async getGeoJsonPoints({ geohashPrecision, extent, timeFilters, layerName, query }) {
 
     let indexPattern;
     try {
@@ -176,7 +177,7 @@ export class ESGeohashGridSource extends VectorSource {
       throw new Error(`Index pattern ${indexPattern.title} no longer contains the geo field ${this._descriptor.geoField}`);
     }
 
-    const aggConfigs = new AggConfigs(indexPattern, this._makeAggConfigs(precision), aggSchemas.all);
+    const aggConfigs = new AggConfigs(indexPattern, this._makeAggConfigs(geohashPrecision), aggSchemas.all);
 
     let resp;
     try {

x-pack/plugins/gis/public/shared/layers/sources/source.js

@@ -60,6 +60,10 @@ export class ASource {
     return false;
   }
 
+  isGeohashPrecisionAware() {
+    return false;
+  }
+
   isQueryAware() {
     return false;
   }

x-pack/plugins/gis/public/shared/layers/util/data_request.js

@@ -10,7 +10,7 @@ export class DataRequest {
   }
 
   hasLoadError() {
-    return this._descriptor.dataHasLoadError;
+    return !!this._descriptor.dataHasLoadError;
   }
 
   getLoadError() {

x-pack/plugins/gis/public/shared/layers/vector_layer.js

@@ -16,6 +16,7 @@ import { FeatureTooltip } from 'plugins/gis/components/map/feature_tooltip';
 import { store } from '../../store/store';
 import { getMapColors } from '../../selectors/map_selectors';
 import _ from 'lodash';
+import { getGeohashPrecisionForZoom } from '../utils/zoom_to_precision';
 
 const EMPTY_FEATURE_COLLECTION = {
   type: 'FeatureCollection',
@@ -142,19 +143,26 @@ export class VectorLayer extends ALayer {
     return this._dataRequests.find(dataRequest => dataRequest.getDataId() === sourceDataId);
   }
 
-  async _canSkipSourceUpdate(source, sourceDataId, filters) {
+  async _canSkipSourceUpdate(source, sourceDataId, searchFilters) {
     const timeAware = await source.isTimeAware();
     const refreshTimerAware = await source.isRefreshTimerAware();
     const extentAware = source.isFilterByMapBounds();
     const isFieldAware = source.isFieldAware();
     const isQueryAware = source.isQueryAware();
+    const isGeohashPrecisionAware = source.isGeohashPrecisionAware();
 
-    if (!timeAware && !refreshTimerAware && !extentAware && !isFieldAware && !isQueryAware) {
+    if (
+      !timeAware &&
+      !refreshTimerAware &&
+      !extentAware &&
+      !isFieldAware &&
+      !isQueryAware &&
+      !isGeohashPrecisionAware
+    ) {
       const sourceDataRequest = this._findDataRequestForSource(sourceDataId);
       if (sourceDataRequest && sourceDataRequest.hasDataOrRequestInProgress()) {
         return true;
       }
-
       return false;
     }
 
@@ -169,29 +177,37 @@ export class VectorLayer extends ALayer {
 
     let updateDueToTime = false;
     if (timeAware) {
-      updateDueToTime = !_.isEqual(meta.timeFilters, filters.timeFilters);
+      updateDueToTime = !_.isEqual(meta.timeFilters, searchFilters.timeFilters);
     }
 
     let updateDueToRefreshTimer = false;
-    if (refreshTimerAware && filters.refreshTimerLastTriggeredAt) {
-      updateDueToRefreshTimer = !_.isEqual(meta.refreshTimerLastTriggeredAt, filters.refreshTimerLastTriggeredAt);
+    if (refreshTimerAware && searchFilters.refreshTimerLastTriggeredAt) {
+      updateDueToRefreshTimer = !_.isEqual(meta.refreshTimerLastTriggeredAt, searchFilters.refreshTimerLastTriggeredAt);
     }
 
     let updateDueToFields = false;
     if (isFieldAware) {
-      updateDueToFields = !_.isEqual(meta.fieldNames, filters.fieldNames);
+      updateDueToFields = !_.isEqual(meta.fieldNames, searchFilters.fieldNames);
     }
 
     let updateDueToQuery = false;
     if (isQueryAware) {
-      updateDueToQuery = !_.isEqual(meta.query, filters.query);
+      updateDueToQuery = !_.isEqual(meta.query, searchFilters.query);
     }
 
+    let updateDueToPrecisionChange = false;
+    if (isGeohashPrecisionAware) {
+      updateDueToPrecisionChange = !_.isEqual(meta.geohashPrecision, searchFilters.geohashPrecision);
+    }
+
+    const updateDueToExtentChange = this.updateDueToExtent(source, meta, searchFilters);
+
     return !updateDueToTime
       && !updateDueToRefreshTimer
-      && !this.updateDueToExtent(source, meta, filters)
+      && !updateDueToExtentChange
       && !updateDueToFields
-      && !updateDueToQuery;
+      && !updateDueToQuery
+      && !updateDueToPrecisionChange;
   }
 
   async _syncJoin(join, { startLoading, stopLoading, onLoadError, dataFilters }) {
@@ -239,34 +255,44 @@ export class VectorLayer extends ALayer {
   }
 
 
+  _getSearchFilters(dataFilters) {
+    const fieldNames = [
+      ...this._source.getFieldNames(),
+      ...this._style.getSourceFieldNames(),
+      ...this.getValidJoins().map(join => {
+        return join.getLeftFieldName();
+      })
+    ];
+
+    const targetPrecision = getGeohashPrecisionForZoom(dataFilters.zoom);
+    return {
+      ...dataFilters,
+      fieldNames: _.uniq(fieldNames).sort(),
+      geohashPrecision: targetPrecision
+    };
+  }
+
   async _syncSource({ startLoading, stopLoading, onLoadError, dataFilters }) {
+
     const sourceDataId = 'source';
     const requestToken = Symbol(`layer-source-refresh:${ this.getId()} - source`);
-    try {
-      const fieldNames = [
-        ...this._source.getFieldNames(),
-        ...this._style.getSourceFieldNames(),
-        ...this.getValidJoins().map(join => {
-          return join.getLeftFieldName();
-        })
-      ];
-      const filters = {
-        ...dataFilters,
-        fieldNames: _.uniq(fieldNames).sort()
+
+    const searchFilters = this._getSearchFilters(dataFilters);
+    const canSkip = await this._canSkipSourceUpdate(this._source, sourceDataId, searchFilters);
+    if (canSkip) {
+      const sourceDataRequest = this.getSourceDataRequest();
+      return {
+        refreshed: false,
+        featureCollection: sourceDataRequest.getData()
       };
-      const canSkip = await this._canSkipSourceUpdate(this._source, sourceDataId, filters);
-      if (canSkip) {
-        const sourceDataRequest = this.getSourceDataRequest();
-        return {
-          refreshed: false,
-          featureCollection: sourceDataRequest.getData()
-        };
-      }
-      startLoading(sourceDataId, requestToken, filters);
+    }
+
+    try {
+      startLoading(sourceDataId, requestToken, searchFilters);
       const layerName = await this.getDisplayName();
       const { data, meta } = await this._source.getGeoJsonWithMeta({
         layerName,
-      }, filters);
+      }, searchFilters);
       stopLoading(sourceDataId, requestToken, data, meta);
       return {
         refreshed: true,

x-pack/plugins/gis/public/shared/utils/zoom_to_precision.js

@@ -5,7 +5,7 @@
  */
 
 
-export const ZOOM_TO_PRECISION = {
+const ZOOM_TO_PRECISION = {
   "0": 1,
   "1": 2,
   "2": 2,
@@ -38,3 +38,9 @@ export const ZOOM_TO_PRECISION = {
   "29": 12,
   "30": 12
 };
+
+export const getGeohashPrecisionForZoom = (zoom) => {
+  let zoomNormalized = Math.round(zoom);
+  zoomNormalized = Math.max(0, Math.min(zoomNormalized, 30));
+  return ZOOM_TO_PRECISION[zoomNormalized];
+};

x-pack/plugins/gis/public/shared/utils/zoom_to_precision.test.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getGeohashPrecisionForZoom } from './zoom_to_precision';
+
+describe('zoom_to_precision', () => {
+
+  it('.getPrecision should clamp correctly', () => {
+
+    let geohashPrecision = getGeohashPrecisionForZoom(-1);
+    expect(geohashPrecision).toEqual(1);
+
+    geohashPrecision = getGeohashPrecisionForZoom(40);
+    expect(geohashPrecision).toEqual(12);
+
+    geohashPrecision = getGeohashPrecisionForZoom(20);
+    expect(geohashPrecision).toEqual(9);
+    geohashPrecision = getGeohashPrecisionForZoom(19);
+    expect(geohashPrecision).toEqual(9);
+
+  });
+
+});

x-pack/test/functional/apps/gis/es_geohashgrid_source.js

@@ -7,6 +7,7 @@
 import expect from 'expect.js';
 
 export default function ({ getPageObjects, getService }) {
+
   const PageObjects = getPageObjects(['gis']);
   const queryBar = getService('queryBar');
   const inspector = getService('inspector');
@@ -14,6 +15,10 @@ export default function ({ getPageObjects, getService }) {
 
   describe('layer geohashgrid aggregation source', () => {
 
+    const EXPECTED_NUMBER_FEATURES = 6;
+    const DATA_CENTER_LON = -98;
+    const DATA_CENTER_LAT = 38;
+
     async function getRequestTimestamp() {
       await PageObjects.gis.openInspectorRequestsView();
       const requestStats = await inspector.getTableData();
@@ -22,13 +27,60 @@ export default function ({ getPageObjects, getService }) {
       return requestTimestamp;
     }
 
+    function makeRequestTestsForGeoPrecision(LAYER_ID) {
+
+      describe('geoprecision - requests', async () => {
+        let beforeTimestamp;
+        beforeEach(async () => {
+          await PageObjects.gis.setView(DATA_CENTER_LAT, DATA_CENTER_LON, 1);
+          beforeTimestamp = await getRequestTimestamp();
+        });
+
+        it('should not rerequest when zoom changes do not cause geohash precision to change', async () => {
+          await PageObjects.gis.setView(DATA_CENTER_LAT, DATA_CENTER_LON, 2);
+          const afterTimestamp = await getRequestTimestamp();
+          expect(afterTimestamp).to.equal(beforeTimestamp);
+        });
+
+        it('should rerequest when zoom changes causes the geohash precision to change', async () => {
+          await PageObjects.gis.setView(DATA_CENTER_LAT, DATA_CENTER_LON, 4);
+          const afterTimestamp = await getRequestTimestamp();
+          expect(afterTimestamp).not.to.equal(beforeTimestamp);
+        });
+      });
+
+      describe('geoprecision - data', async ()=> {
+
+        beforeEach(async () => {
+          await PageObjects.gis.setView(DATA_CENTER_LAT, DATA_CENTER_LON, 1);
+        });
+
+        it ('should not return any data when the extent does not cover the data bounds', async () => {
+          await PageObjects.gis.setView(64, 179, 5);
+          const mapboxStyle = await PageObjects.gis.getMapboxStyle();
+          expect(mapboxStyle.sources[LAYER_ID].data.features.length).to.equal(0);
+        });
+
+        it ('should request the data when the map covers the databounds', async () => {
+          const mapboxStyle = await PageObjects.gis.getMapboxStyle();
+          expect(mapboxStyle.sources[LAYER_ID].data.features.length).to.equal(EXPECTED_NUMBER_FEATURES);
+        });
+
+        it ('should request only partial data when the map only covers part of the databounds', async () => {
+          //todo this verifies the extent-filtering behavior (not really the correct application of geohash-precision), and should ideally be moved to its own section
+          await PageObjects.gis.setView(DATA_CENTER_LAT, DATA_CENTER_LON, 6);
+          const mapboxStyle = await PageObjects.gis.getMapboxStyle();
+          expect(mapboxStyle.sources[LAYER_ID].data.features.length).to.equal(2);
+        });
+      });
+    }
+
     describe('heatmap', () => {
       before(async () => {
         await PageObjects.gis.loadSavedMap('geohashgrid heatmap example');
       });
 
       const LAYER_ID = '3xlvm';
-      const EXPECTED_NUMBER_FEATURES = 6;
       const HEATMAP_PROP_NAME = '__kbn_heatmap_weight__';
 
       it('should re-fetch geohashgrid aggregation with refresh timer', async () => {
@@ -49,10 +101,13 @@ export default function ({ getPageObjects, getService }) {
         });
       });
 
+      makeRequestTestsForGeoPrecision(LAYER_ID);
+
       describe('query bar', () => {
         before(async () => {
           await queryBar.setQuery('machine.os.raw : "win 8"');
           await queryBar.submitQuery();
+          await PageObjects.gis.setView(0, 0, 0);
         });
 
         after(async () => {
@@ -99,7 +154,7 @@ export default function ({ getPageObjects, getService }) {
       });
 
       const LAYER_ID = 'g1xkv';
-      const EXPECTED_NUMBER_FEATURES = 6;
+
       const MAX_OF_BYTES_PROP_NAME = 'max_of_bytes';
 
       it('should re-fetch geohashgrid aggregation with refresh timer', async () => {
@@ -120,8 +175,12 @@ export default function ({ getPageObjects, getService }) {
         });
       });
 
+      makeRequestTestsForGeoPrecision(LAYER_ID);
+
+
       describe('query bar', () => {
         before(async () => {
+          await PageObjects.gis.setView(0, 0, 0);
           await queryBar.setQuery('machine.os.raw : "win 8"');
           await queryBar.submitQuery();
         });
@@ -162,6 +221,7 @@ export default function ({ getPageObjects, getService }) {
           expect(noRequests).to.equal(true);
         });
       });
+
     });
 
   });

x-pack/test/functional/page_objects/gis_page.js

@@ -105,11 +105,11 @@ export function GisPageProvider({ getService, getPageObjects }) {
      * Layer TOC (table to contents) utility functions
      */
     async setView(lat, lon, zoom) {
-      log.debug(`Set view lat: ${lat}, lon: ${lon}, zoom: ${zoom}`);
+      log.debug(`Set view lat: ${lat.toString()}, lon: ${lon.toString()}, zoom: ${zoom.toString()}`);
       await testSubjects.click('toggleSetViewVisibilityButton');
-      await testSubjects.setValue('latitudeInput', lat);
-      await testSubjects.setValue('longitudeInput', lon);
-      await testSubjects.setValue('zoomInput', zoom);
+      await testSubjects.setValue('latitudeInput', lat.toString());
+      await testSubjects.setValue('longitudeInput', lon.toString());
+      await testSubjects.setValue('zoomInput', zoom.toString());
       await testSubjects.click('submitViewButton');
       await PageObjects.header.waitUntilLoadingHasFinished();
     }