[Maps] Convert HeatmapLayer to vector tiles and add support for high resolution grids (#119070)

* use interpolate instead of modifing geojson

* update mbMap._queryForMeta to support HeatmapLayer

* syncMvtSourceData

* vector source

* gridPrecision

* allow seleting super fine

* removeStaleMbSourcesAndLayers

* clean up

* fix tests

* linter

* i18n fixes and update jest snapshots

* move mtv sync source tests from mvt_vector_layer to sync_source_data test suite

* clean up test

* eslint

* review feedback

* update test expects

* properly fix test expects

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Nathan Reese 2021-11-22 14:19:09 -07:00 committed by GitHub
parent aa3a6bc777
commit 6c710ffedf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 721 additions and 646 deletions

View file

@ -5,19 +5,16 @@
* 2.0.
*/
import type { Map as MbMap, GeoJSONSource as MbGeoJSONSource } from '@kbn/mapbox-gl';
import { FeatureCollection } from 'geojson';
import type { Map as MbMap, VectorSource as MbVectorSource } from '@kbn/mapbox-gl';
import { AbstractLayer } from '../layer';
import { HeatmapStyle } from '../../styles/heatmap/heatmap_style';
import { EMPTY_FEATURE_COLLECTION, LAYER_TYPE } from '../../../../common/constants';
import { LAYER_TYPE } from '../../../../common/constants';
import { HeatmapLayerDescriptor } from '../../../../common/descriptor_types';
import { ESGeoGridSource } from '../../sources/es_geo_grid_source';
import { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from '../vector_layer';
import { getVectorSourceBounds, MvtSourceData, syncMvtSourceData } from '../vector_layer';
import { DataRequestContext } from '../../../actions';
import { DataRequestAbortError } from '../../util/data_request';
import { buildVectorRequestMeta } from '../build_vector_request_meta';
const SCALED_PROPERTY_NAME = '__kbn_heatmap_weight__'; // unique name to store scaled value for weighting
import { ITiledSingleLayerVectorSource } from '../../sources/tiled_single_layer_vector_source';
export class HeatmapLayer extends AbstractLayer {
private readonly _style: HeatmapStyle;
@ -67,11 +64,6 @@ export class HeatmapLayer extends AbstractLayer {
return this._style;
}
_getPropKeyOfSelectedMetric() {
const metricfields = this.getSource().getMetricFields();
return metricfields[0].getName();
}
_getHeatmapLayerId() {
return this.makeMbLayerId('heatmap');
}
@ -89,81 +81,95 @@ export class HeatmapLayer extends AbstractLayer {
}
async syncData(syncContext: DataRequestContext) {
if (this.isLoadingBounds()) {
return;
await syncMvtSourceData({
layerId: this.getId(),
prevDataRequest: this.getSourceDataRequest(),
requestMeta: buildVectorRequestMeta(
this.getSource(),
this.getSource().getFieldNames(),
syncContext.dataFilters,
this.getQuery(),
syncContext.isForceRefresh
),
source: this.getSource() as ITiledSingleLayerVectorSource,
syncContext,
});
}
_requiresPrevSourceCleanup(mbMap: MbMap): boolean {
const mbSource = mbMap.getSource(this.getMbSourceId()) as MbVectorSource;
if (!mbSource) {
return false;
}
try {
await syncVectorSource({
layerId: this.getId(),
layerName: await this.getDisplayName(this.getSource()),
prevDataRequest: this.getSourceDataRequest(),
requestMeta: buildVectorRequestMeta(
this.getSource(),
this.getSource().getFieldNames(),
syncContext.dataFilters,
this.getQuery(),
syncContext.isForceRefresh
),
syncContext,
source: this.getSource(),
getUpdateDueToTimeslice: () => {
return true;
},
});
} catch (error) {
if (!(error instanceof DataRequestAbortError)) {
throw error;
}
const sourceDataRequest = this.getSourceDataRequest();
if (!sourceDataRequest) {
return false;
}
const sourceData = sourceDataRequest.getData() as MvtSourceData | undefined;
if (!sourceData) {
return false;
}
return mbSource.tiles?.[0] !== sourceData.urlTemplate;
}
syncLayerWithMB(mbMap: MbMap) {
addGeoJsonMbSource(this._getMbSourceId(), this.getMbLayerIds(), mbMap);
this._removeStaleMbSourcesAndLayers(mbMap);
const sourceDataRequest = this.getSourceDataRequest();
const sourceData = sourceDataRequest
? (sourceDataRequest.getData() as MvtSourceData)
: undefined;
if (!sourceData) {
return;
}
const mbSourceId = this.getMbSourceId();
const mbSource = mbMap.getSource(mbSourceId);
if (!mbSource) {
mbMap.addSource(mbSourceId, {
type: 'vector',
tiles: [sourceData.urlTemplate],
minzoom: sourceData.minSourceZoom,
maxzoom: sourceData.maxSourceZoom,
});
}
const heatmapLayerId = this._getHeatmapLayerId();
if (!mbMap.getLayer(heatmapLayerId)) {
mbMap.addLayer({
id: heatmapLayerId,
type: 'heatmap',
source: this.getId(),
source: mbSourceId,
['source-layer']: sourceData.layerName,
paint: {},
});
}
const mbGeoJSONSource = mbMap.getSource(this.getId()) as MbGeoJSONSource;
const sourceDataRequest = this.getSourceDataRequest();
const featureCollection = sourceDataRequest
? (sourceDataRequest.getData() as FeatureCollection)
: null;
if (!featureCollection) {
mbGeoJSONSource.setData(EMPTY_FEATURE_COLLECTION);
const metricFields = this.getSource().getMetricFields();
if (!metricFields.length) {
return;
}
const metricField = metricFields[0];
const propertyKey = this._getPropKeyOfSelectedMetric();
const dataBoundToMap = AbstractLayer.getBoundDataForSource(mbMap, this.getId());
if (featureCollection !== dataBoundToMap) {
let max = 1; // max will be at least one, since counts or sums will be at least one.
for (let i = 0; i < featureCollection.features.length; i++) {
max = Math.max(featureCollection.features[i].properties?.[propertyKey], max);
const tileMetaFeatures = this._getMetaFromTiles();
let max = 0;
for (let i = 0; i < tileMetaFeatures.length; i++) {
const range = metricField.pluckRangeFromTileMetaFeature(tileMetaFeatures[i]);
if (range) {
max = Math.max(range.max, max);
}
for (let i = 0; i < featureCollection.features.length; i++) {
if (featureCollection.features[i].properties) {
featureCollection.features[i].properties![SCALED_PROPERTY_NAME] =
featureCollection.features[i].properties![propertyKey] / max;
}
}
mbGeoJSONSource.setData(featureCollection);
}
this.syncVisibilityWithMb(mbMap, heatmapLayerId);
this.getCurrentStyle().setMBPaintProperties({
mbMap,
layerId: heatmapLayerId,
propertyName: SCALED_PROPERTY_NAME,
propertyName: metricField.getMbFieldName(),
max,
resolution: this.getSource().getGridResolution(),
});
this.syncVisibilityWithMb(mbMap, heatmapLayerId);
mbMap.setPaintProperty(heatmapLayerId, 'heatmap-opacity', this.getAlpha());
mbMap.setLayerZoomRange(heatmapLayerId, this.getMinZoom(), this.getMaxZoom());
}

View file

@ -29,6 +29,7 @@ import {
LayerDescriptor,
MapExtent,
StyleDescriptor,
TileMetaFeature,
Timeslice,
StyleMetaDescriptor,
} from '../../../common/descriptor_types';
@ -75,6 +76,11 @@ export interface ILayer {
*/
getMbLayerIds(): string[];
/*
* ILayer.getMbSourceId returns mapbox source id assoicated with this layer.
*/
getMbSourceId(): string;
ownsMbLayerId(mbLayerId: string): boolean;
ownsMbSourceId(mbSourceId: string): boolean;
syncLayerWithMB(mbMap: MbMap, timeslice?: Timeslice): void;
@ -295,7 +301,7 @@ export class AbstractLayer implements ILayer {
return this._source.getMinZoom();
}
_getMbSourceId() {
getMbSourceId() {
return this.getId();
}
@ -475,4 +481,8 @@ export class AbstractLayer implements ILayer {
isBasemap(order: number): boolean {
return false;
}
_getMetaFromTiles(): TileMetaFeature[] {
return this._descriptor.__metaFromTiles || [];
}
}

View file

@ -102,7 +102,7 @@ export class TileLayer extends AbstractLayer {
return;
}
const mbSourceId = this._getMbSourceId();
const mbSourceId = this.getMbSourceId();
mbMap.addSource(mbSourceId, {
type: 'raster',
tiles: [tmsSourceData.url],

View file

@ -139,7 +139,7 @@ export class GeoJsonVectorLayer extends AbstractVectorLayer {
}
syncLayerWithMB(mbMap: MbMap, timeslice?: Timeslice) {
addGeoJsonMbSource(this._getMbSourceId(), this.getMbLayerIds(), mbMap);
addGeoJsonMbSource(this.getMbSourceId(), this.getMbLayerIds(), mbMap);
this._syncFeatureCollectionWithMb(mbMap);

View file

@ -15,4 +15,5 @@ export { isVectorLayer, NO_RESULTS_ICON_AND_TOOLTIPCONTENT } from './vector_laye
export { BlendedVectorLayer } from './blended_vector_layer';
export { GeoJsonVectorLayer } from './geojson_vector_layer';
export { MvtVectorLayer } from './mvt_vector_layer';
export { MvtVectorLayer, syncMvtSourceData } from './mvt_vector_layer';
export type { MvtSourceData } from './mvt_vector_layer';

View file

@ -6,3 +6,5 @@
*/
export { MvtVectorLayer } from './mvt_vector_layer';
export { syncMvtSourceData } from './mvt_source_data';
export type { MvtSourceData } from './mvt_source_data';

View file

@ -0,0 +1,292 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
jest.mock('uuid/v4', () => {
return function () {
return '12345';
};
});
import sinon from 'sinon';
import { MockSyncContext } from '../../__fixtures__/mock_sync_context';
import { ITiledSingleLayerVectorSource } from '../../../sources/tiled_single_layer_vector_source';
import { DataRequest } from '../../../util/data_request';
import { syncMvtSourceData } from './mvt_source_data';
const mockSource = {
getLayerName: () => {
return 'aggs';
},
getMinZoom: () => {
return 4;
},
getMaxZoom: () => {
return 14;
},
getUrlTemplateWithMeta: () => {
return {
refreshTokenParamName: 'token',
layerName: 'aggs',
minSourceZoom: 4,
maxSourceZoom: 14,
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf',
};
},
isTimeAware: () => {
return true;
},
isFieldAware: () => {
return false;
},
isQueryAware: () => {
return false;
},
isGeoGridPrecisionAware: () => {
return false;
},
} as unknown as ITiledSingleLayerVectorSource;
describe('syncMvtSourceData', () => {
test('Should sync source data when there are no previous data request', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: undefined,
requestMeta: {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
},
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
// @ts-expect-error
const call = syncContext.stopLoading.getCall(0);
const sourceData = call.args[2];
expect(sourceData).toEqual({
minSourceZoom: 4,
maxSourceZoom: 14,
layerName: 'aggs',
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
});
});
test('Should not re-sync when there are no changes in source state or search state', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
const prevRequestMeta = {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
};
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: {
getMeta: () => {
return prevRequestMeta;
},
getData: () => {
return {
minSourceZoom: 4,
maxSourceZoom: 14,
layerName: 'aggs',
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
};
},
} as unknown as DataRequest,
requestMeta: { ...prevRequestMeta },
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.notCalled(syncContext.startLoading);
// @ts-expect-error
sinon.assert.notCalled(syncContext.stopLoading);
});
test('Should re-sync when there are changes to search state', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
const prevRequestMeta = {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
};
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: {
getMeta: () => {
return prevRequestMeta;
},
getData: () => {
return {
minSourceZoom: 4,
maxSourceZoom: 14,
layerName: 'aggs',
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
};
},
} as unknown as DataRequest,
requestMeta: {
...prevRequestMeta,
timeFilters: {
from: 'now',
to: '30m',
mode: 'relative',
},
},
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
});
test('Should re-sync when layerName source state changes: ', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
const prevRequestMeta = {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
};
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: {
getMeta: () => {
return prevRequestMeta;
},
getData: () => {
return {
minSourceZoom: 4,
maxSourceZoom: 14,
layerName: 'barfoo', // layerName is different then mockSource
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
};
},
} as unknown as DataRequest,
requestMeta: { ...prevRequestMeta },
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
});
test('Should re-sync when minZoom source state changes: ', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
const prevRequestMeta = {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
};
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: {
getMeta: () => {
return prevRequestMeta;
},
getData: () => {
return {
minSourceZoom: 2, // minSourceZoom is different then mockSource
maxSourceZoom: 14,
layerName: 'aggs',
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
};
},
} as unknown as DataRequest,
requestMeta: { ...prevRequestMeta },
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
});
test('Should re-sync when maxZoom source state changes: ', async () => {
const syncContext = new MockSyncContext({ dataFilters: {} });
const prevRequestMeta = {
...syncContext.dataFilters,
applyGlobalQuery: true,
applyGlobalTime: true,
applyForceRefresh: true,
fieldNames: [],
sourceMeta: {},
isForceRefresh: false,
};
await syncMvtSourceData({
layerId: 'layer1',
prevDataRequest: {
getMeta: () => {
return prevRequestMeta;
},
getData: () => {
return {
minSourceZoom: 4,
maxSourceZoom: 9, // minSourceZoom is different then mockSource
layerName: 'aggs',
refreshTokenParamName: 'token',
urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf?token=12345',
urlToken: '12345',
};
},
} as unknown as DataRequest,
requestMeta: { ...prevRequestMeta },
source: mockSource,
syncContext,
});
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
});
});

View file

@ -0,0 +1,91 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import uuid from 'uuid/v4';
import { parse as parseUrl } from 'url';
import { SOURCE_DATA_REQUEST_ID } from '../../../../../common/constants';
import { Timeslice, VectorSourceRequestMeta } from '../../../../../common/descriptor_types';
import { DataRequest } from '../../../util/data_request';
import { DataRequestContext } from '../../../../actions';
import { canSkipSourceUpdate } from '../../../util/can_skip_fetch';
import {
ITiledSingleLayerMvtParams,
ITiledSingleLayerVectorSource,
} from '../../../sources/tiled_single_layer_vector_source';
// shape of sourceDataRequest.getData()
export type MvtSourceData = ITiledSingleLayerMvtParams & {
urlTemplate: string;
urlToken: string;
};
export async function syncMvtSourceData({
layerId,
prevDataRequest,
requestMeta,
source,
syncContext,
}: {
layerId: string;
prevDataRequest: DataRequest | undefined;
requestMeta: VectorSourceRequestMeta;
source: ITiledSingleLayerVectorSource;
syncContext: DataRequestContext;
}): Promise<void> {
const requestToken: symbol = Symbol(`${layerId}-${SOURCE_DATA_REQUEST_ID}`);
const prevData = prevDataRequest ? (prevDataRequest.getData() as MvtSourceData) : undefined;
if (prevData) {
const noChangesInSourceState: boolean =
prevData.layerName === source.getLayerName() &&
prevData.minSourceZoom === source.getMinZoom() &&
prevData.maxSourceZoom === source.getMaxZoom();
const noChangesInSearchState: boolean = await canSkipSourceUpdate({
extentAware: false, // spatial extent knowledge is already fully automated by tile-loading based on pan-zooming
source,
prevDataRequest,
nextRequestMeta: requestMeta,
getUpdateDueToTimeslice: (timeslice?: Timeslice) => {
return true;
},
});
const canSkip = noChangesInSourceState && noChangesInSearchState;
if (canSkip) {
return;
}
}
syncContext.startLoading(SOURCE_DATA_REQUEST_ID, requestToken, requestMeta);
try {
const urlToken =
!prevData || (requestMeta.isForceRefresh && requestMeta.applyForceRefresh)
? uuid()
: prevData.urlToken;
const newUrlTemplateAndMeta = await source.getUrlTemplateWithMeta(requestMeta);
let urlTemplate;
if (newUrlTemplateAndMeta.refreshTokenParamName) {
const parsedUrl = parseUrl(newUrlTemplateAndMeta.urlTemplate, true);
const separator = !parsedUrl.query || Object.keys(parsedUrl.query).length === 0 ? '?' : '&';
urlTemplate = `${newUrlTemplateAndMeta.urlTemplate}${separator}${newUrlTemplateAndMeta.refreshTokenParamName}=${urlToken}`;
} else {
urlTemplate = newUrlTemplateAndMeta.urlTemplate;
}
const sourceData = {
...newUrlTemplateAndMeta,
urlToken,
urlTemplate,
};
syncContext.stopLoading(SOURCE_DATA_REQUEST_ID, requestToken, sourceData, {});
} catch (error) {
syncContext.onLoadError(SOURCE_DATA_REQUEST_ID, requestToken, error.message);
}
}

View file

@ -5,10 +5,6 @@
* 2.0.
*/
import { MockSyncContext } from '../../__fixtures__/mock_sync_context';
import sinon from 'sinon';
import url from 'url';
jest.mock('../../../../kibana_services', () => {
return {
getIsDarkMode() {
@ -22,7 +18,6 @@ import { shallow } from 'enzyme';
import { Feature } from 'geojson';
import { MVTSingleLayerVectorSource } from '../../../sources/mvt_single_layer_vector_source';
import {
DataRequestDescriptor,
TiledSingleLayerVectorSourceDescriptor,
VectorLayerDescriptor,
} from '../../../../../common/descriptor_types';
@ -112,130 +107,3 @@ describe('getFeatureById', () => {
expect(feature).toEqual(null);
});
});
describe('syncData', () => {
it('Should sync with source-params', async () => {
const layer: MvtVectorLayer = createLayer({}, {});
const syncContext = new MockSyncContext({ dataFilters: {} });
await layer.syncData(syncContext);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
// @ts-expect-error
const call = syncContext.stopLoading.getCall(0);
expect(call.args[2]!.minSourceZoom).toEqual(defaultConfig.minSourceZoom);
expect(call.args[2]!.maxSourceZoom).toEqual(defaultConfig.maxSourceZoom);
expect(call.args[2]!.layerName).toEqual(defaultConfig.layerName);
expect(call.args[2]!.urlTemplate).toEqual(defaultConfig.urlTemplate);
});
it('Should not resync when no changes to source params', async () => {
const dataRequestDescriptor: DataRequestDescriptor = {
data: { ...defaultConfig },
dataId: 'source',
};
const layer: MvtVectorLayer = createLayer(
{
__dataRequests: [dataRequestDescriptor],
},
{}
);
const syncContext = new MockSyncContext({ dataFilters: {} });
await layer.syncData(syncContext);
// @ts-expect-error
sinon.assert.notCalled(syncContext.startLoading);
// @ts-expect-error
sinon.assert.notCalled(syncContext.stopLoading);
});
it('Should resync when changes to syncContext', async () => {
const dataRequestDescriptor: DataRequestDescriptor = {
data: { ...defaultConfig },
dataId: 'source',
};
const layer: MvtVectorLayer = createLayer(
{
__dataRequests: [dataRequestDescriptor],
},
{},
true
);
const syncContext = new MockSyncContext({
dataFilters: {
timeFilters: {
from: 'now',
to: '30m',
mode: 'relative',
},
},
});
await layer.syncData(syncContext);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
});
describe('Should resync when changes to source params: ', () => {
[{ layerName: 'barfoo' }, { minSourceZoom: 1 }, { maxSourceZoom: 12 }].forEach((changes) => {
it(`change in ${Object.keys(changes).join(',')}`, async () => {
const dataRequestDescriptor: DataRequestDescriptor = {
data: defaultConfig,
dataId: 'source',
};
const layer: MvtVectorLayer = createLayer(
{
__dataRequests: [dataRequestDescriptor],
},
changes
);
const syncContext = new MockSyncContext({ dataFilters: {} });
await layer.syncData(syncContext);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
// @ts-expect-error
const call = syncContext.stopLoading.getCall(0);
const newMeta = { ...defaultConfig, ...changes };
expect(call.args[2]!.minSourceZoom).toEqual(newMeta.minSourceZoom);
expect(call.args[2]!.maxSourceZoom).toEqual(newMeta.maxSourceZoom);
expect(call.args[2]!.layerName).toEqual(newMeta.layerName);
expect(call.args[2]!.urlTemplate).toEqual(newMeta.urlTemplate);
});
});
});
describe('refresh token', () => {
const uuidRegex = /\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b/;
it(`should add token in url`, async () => {
const layer: MvtVectorLayer = createLayer({}, {}, false, true);
const syncContext = new MockSyncContext({ dataFilters: {} });
await layer.syncData(syncContext);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.startLoading);
// @ts-expect-error
sinon.assert.calledOnce(syncContext.stopLoading);
// @ts-expect-error
const call = syncContext.stopLoading.getCall(0);
expect(call.args[2]!.minSourceZoom).toEqual(defaultConfig.minSourceZoom);
expect(call.args[2]!.maxSourceZoom).toEqual(defaultConfig.maxSourceZoom);
expect(call.args[2]!.layerName).toEqual(defaultConfig.layerName);
expect(call.args[2]!.urlTemplate.startsWith(defaultConfig.urlTemplate)).toBe(true);
const parsedUrl = url.parse(call.args[2]!.urlTemplate, true);
expect(!!(parsedUrl.query.token! as string).match(uuidRegex)).toBe(true);
});
});
});

View file

@ -13,10 +13,8 @@ import type {
} from '@kbn/mapbox-gl';
import { Feature } from 'geojson';
import { i18n } from '@kbn/i18n';
import uuid from 'uuid/v4';
import { parse as parseUrl } from 'url';
import { IVectorStyle, VectorStyle } from '../../../styles/vector/vector_style';
import { LAYER_TYPE, SOURCE_DATA_REQUEST_ID, SOURCE_TYPES } from '../../../../../common/constants';
import { VectorStyle } from '../../../styles/vector/vector_style';
import { LAYER_TYPE, SOURCE_TYPES } from '../../../../../common/constants';
import {
NO_RESULTS_ICON_AND_TOOLTIPCONTENT,
AbstractVectorLayer,
@ -27,16 +25,13 @@ import { DataRequestContext } from '../../../../actions';
import {
StyleMetaDescriptor,
TileMetaFeature,
Timeslice,
VectorLayerDescriptor,
VectorSourceRequestMeta,
} from '../../../../../common/descriptor_types';
import { MVTSingleLayerVectorSourceConfig } from '../../../sources/mvt_single_layer_vector_source/types';
import { ESSearchSource } from '../../../sources/es_search_source';
import { canSkipSourceUpdate } from '../../../util/can_skip_fetch';
import { LayerIcon } from '../../layer';
import { MvtSourceData, syncMvtSourceData } from './mvt_source_data';
const ES_MVT_META_LAYER_NAME = 'meta';
export const ES_MVT_META_LAYER_NAME = 'meta';
const ES_MVT_HITS_TOTAL_RELATION = 'hits.total.relation';
const ES_MVT_HITS_TOTAL_VALUE = 'hits.total.value';
const MAX_RESULT_WINDOW_DATA_REQUEST_ID = 'maxResultWindow';
@ -74,10 +69,6 @@ export class MvtVectorLayer extends AbstractVectorLayer {
: feature.properties?._key;
}
_getMetaFromTiles(): TileMetaFeature[] {
return this._descriptor.__metaFromTiles || [];
}
getLayerIcon(isTocIcon: boolean): LayerIcon {
if (!this.getSource().isESSource()) {
// Only ES-sources can have a special meta-tile, not 3rd party vector tile sources
@ -173,90 +164,29 @@ export class MvtVectorLayer extends AbstractVectorLayer {
stopLoading(MAX_RESULT_WINDOW_DATA_REQUEST_ID, requestToken, { maxResultWindow });
}
async _syncMVTUrlTemplate({
startLoading,
stopLoading,
onLoadError,
dataFilters,
isForceRefresh,
}: DataRequestContext) {
const requestToken: symbol = Symbol(`layer-${this.getId()}-${SOURCE_DATA_REQUEST_ID}`);
const requestMeta: VectorSourceRequestMeta = await this._getVectorSourceRequestMeta(
isForceRefresh,
dataFilters,
this.getSource(),
this._style as IVectorStyle
);
const prevDataRequest = this.getSourceDataRequest();
if (prevDataRequest) {
const data: MVTSingleLayerVectorSourceConfig =
prevDataRequest.getData() as MVTSingleLayerVectorSourceConfig;
if (data) {
const noChangesInSourceState: boolean =
data.layerName === this._source.getLayerName() &&
data.minSourceZoom === this._source.getMinZoom() &&
data.maxSourceZoom === this._source.getMaxZoom();
const noChangesInSearchState: boolean = await canSkipSourceUpdate({
extentAware: false, // spatial extent knowledge is already fully automated by tile-loading based on pan-zooming
source: this.getSource(),
prevDataRequest,
nextRequestMeta: requestMeta,
getUpdateDueToTimeslice: (timeslice?: Timeslice) => {
// TODO use meta features to determine if tiles already contain features for timeslice.
return true;
},
});
const canSkip = noChangesInSourceState && noChangesInSearchState;
if (canSkip) {
return null;
}
}
}
startLoading(SOURCE_DATA_REQUEST_ID, requestToken, requestMeta);
try {
const prevData = prevDataRequest
? (prevDataRequest.getData() as MVTSingleLayerVectorSourceConfig)
: undefined;
const urlToken =
!prevData || (requestMeta.isForceRefresh && requestMeta.applyForceRefresh)
? uuid()
: prevData.urlToken;
const newUrlTemplateAndMeta = await this._source.getUrlTemplateWithMeta(requestMeta);
let urlTemplate;
if (newUrlTemplateAndMeta.refreshTokenParamName) {
const parsedUrl = parseUrl(newUrlTemplateAndMeta.urlTemplate, true);
const separator = !parsedUrl.query || Object.keys(parsedUrl.query).length === 0 ? '?' : '&';
urlTemplate = `${newUrlTemplateAndMeta.urlTemplate}${separator}${newUrlTemplateAndMeta.refreshTokenParamName}=${urlToken}`;
} else {
urlTemplate = newUrlTemplateAndMeta.urlTemplate;
}
const urlTemplateAndMetaWithToken = {
...newUrlTemplateAndMeta,
urlToken,
urlTemplate,
};
stopLoading(SOURCE_DATA_REQUEST_ID, requestToken, urlTemplateAndMetaWithToken, {});
} catch (error) {
onLoadError(SOURCE_DATA_REQUEST_ID, requestToken, error.message);
}
}
async syncData(syncContext: DataRequestContext) {
if (this.getSource().getType() === SOURCE_TYPES.ES_SEARCH) {
await this._syncMaxResultWindow(syncContext);
}
await this._syncSourceStyleMeta(syncContext, this._source, this._style as IVectorStyle);
await this._syncSourceFormatters(syncContext, this._source, this._style as IVectorStyle);
await this._syncMVTUrlTemplate(syncContext);
await this._syncSourceStyleMeta(syncContext, this.getSource(), this.getCurrentStyle());
await this._syncSourceFormatters(syncContext, this.getSource(), this.getCurrentStyle());
await syncMvtSourceData({
layerId: this.getId(),
prevDataRequest: this.getSourceDataRequest(),
requestMeta: await this._getVectorSourceRequestMeta(
syncContext.isForceRefresh,
syncContext.dataFilters,
this.getSource(),
this.getCurrentStyle()
),
source: this.getSource() as ITiledSingleLayerVectorSource,
syncContext,
});
}
_syncSourceBindingWithMb(mbMap: MbMap) {
const mbSource = mbMap.getSource(this._getMbSourceId());
const mbSource = mbMap.getSource(this.getMbSourceId());
if (mbSource) {
return;
}
@ -268,18 +198,17 @@ export class MvtVectorLayer extends AbstractVectorLayer {
return;
}
const sourceMeta: MVTSingleLayerVectorSourceConfig | null =
sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig;
if (!sourceMeta) {
const sourceData = sourceDataRequest.getData() as MvtSourceData | undefined;
if (!sourceData) {
return;
}
const mbSourceId = this._getMbSourceId();
const mbSourceId = this.getMbSourceId();
mbMap.addSource(mbSourceId, {
type: 'vector',
tiles: [sourceMeta.urlTemplate],
minzoom: sourceMeta.minSourceZoom,
maxzoom: sourceMeta.maxSourceZoom,
tiles: [sourceData.urlTemplate],
minzoom: sourceData.minSourceZoom,
maxzoom: sourceData.maxSourceZoom,
});
}
@ -288,7 +217,7 @@ export class MvtVectorLayer extends AbstractVectorLayer {
}
ownsMbSourceId(mbSourceId: string): boolean {
return this._getMbSourceId() === mbSourceId;
return this.getMbSourceId() === mbSourceId;
}
_getMbTooManyFeaturesLayerId() {
@ -297,7 +226,7 @@ export class MvtVectorLayer extends AbstractVectorLayer {
_syncStylePropertiesWithMb(mbMap: MbMap) {
// @ts-ignore
const mbSource = mbMap.getSource(this._getMbSourceId());
const mbSource = mbMap.getSource(this.getMbSourceId());
if (!mbSource) {
return;
}
@ -306,15 +235,14 @@ export class MvtVectorLayer extends AbstractVectorLayer {
if (!sourceDataRequest) {
return;
}
const sourceMeta: MVTSingleLayerVectorSourceConfig =
sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig;
if (sourceMeta.layerName === '') {
const sourceData = sourceDataRequest.getData() as MvtSourceData | undefined;
if (!sourceData || sourceData.layerName === '') {
return;
}
this._setMbLabelProperties(mbMap, sourceMeta.layerName);
this._setMbPointsProperties(mbMap, sourceMeta.layerName);
this._setMbLinePolygonProperties(mbMap, sourceMeta.layerName);
this._setMbLabelProperties(mbMap, sourceData.layerName);
this._setMbPointsProperties(mbMap, sourceData.layerName);
this._setMbLinePolygonProperties(mbMap, sourceData.layerName);
this._syncTooManyFeaturesProperties(mbMap);
}
@ -359,66 +287,8 @@ export class MvtVectorLayer extends AbstractVectorLayer {
mbMap.setLayerZoomRange(tooManyFeaturesLayerId, this.getMinZoom(), this.getMaxZoom());
}
queryTileMetaFeatures(mbMap: MbMap): TileMetaFeature[] | null {
if (!this.getSource().isESSource()) {
return null;
}
// @ts-ignore
const mbSource = mbMap.getSource(this._getMbSourceId());
if (!mbSource) {
return null;
}
const sourceDataRequest = this.getSourceDataRequest();
if (!sourceDataRequest) {
return null;
}
const sourceMeta: MVTSingleLayerVectorSourceConfig =
sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig;
if (sourceMeta.layerName === '') {
return null;
}
// querySourceFeatures can return duplicated features when features cross tile boundaries.
// Tile meta will never have duplicated features since by there nature, tile meta is a feature contained within a single tile
const mbFeatures = mbMap.querySourceFeatures(this._getMbSourceId(), {
sourceLayer: ES_MVT_META_LAYER_NAME,
});
const metaFeatures: Array<TileMetaFeature | null> = (
mbFeatures as unknown as TileMetaFeature[]
).map((mbFeature: TileMetaFeature | null) => {
const parsedProperties: Record<string, unknown> = {};
for (const key in mbFeature?.properties) {
if (mbFeature?.properties.hasOwnProperty(key)) {
parsedProperties[key] =
typeof mbFeature.properties[key] === 'string' ||
typeof mbFeature.properties[key] === 'number' ||
typeof mbFeature.properties[key] === 'boolean'
? mbFeature.properties[key]
: JSON.parse(mbFeature.properties[key]); // mvt properties cannot be nested geojson
}
}
try {
return {
type: 'Feature',
id: mbFeature?.id,
geometry: mbFeature?.geometry, // this getter might throw with non-conforming geometries
properties: parsedProperties,
} as TileMetaFeature;
} catch (e) {
return null;
}
});
const filtered = metaFeatures.filter((f) => f !== null);
return filtered as TileMetaFeature[];
}
_requiresPrevSourceCleanup(mbMap: MbMap): boolean {
const mbSource = mbMap.getSource(this._getMbSourceId()) as MbVectorSource | MbGeoJSONSource;
const mbSource = mbMap.getSource(this.getMbSourceId()) as MbVectorSource | MbGeoJSONSource;
if (!mbSource) {
return false;
}
@ -428,21 +298,19 @@ export class MvtVectorLayer extends AbstractVectorLayer {
}
const mbTileSource = mbSource as MbVectorSource;
const dataRequest = this.getSourceDataRequest();
if (!dataRequest) {
const sourceDataRequest = this.getSourceDataRequest();
if (!sourceDataRequest) {
return false;
}
const tiledSourceMeta: MVTSingleLayerVectorSourceConfig | null =
dataRequest.getData() as MVTSingleLayerVectorSourceConfig;
if (!tiledSourceMeta) {
const sourceData = sourceDataRequest.getData() as MvtSourceData | undefined;
if (!sourceData) {
return false;
}
const isSourceDifferent =
mbTileSource.tiles?.[0] !== tiledSourceMeta.urlTemplate ||
mbTileSource.minzoom !== tiledSourceMeta.minSourceZoom ||
mbTileSource.maxzoom !== tiledSourceMeta.maxSourceZoom;
mbTileSource.tiles?.[0] !== sourceData.urlTemplate ||
mbTileSource.minzoom !== sourceData.minSourceZoom ||
mbTileSource.maxzoom !== sourceData.maxSourceZoom;
if (isSourceDifferent) {
return true;
@ -456,7 +324,7 @@ export class MvtVectorLayer extends AbstractVectorLayer {
if (
mbLayer &&
// @ts-expect-error
mbLayer.sourceLayer !== tiledSourceMeta.layerName &&
mbLayer.sourceLayer !== sourceData.layerName &&
// @ts-expect-error
mbLayer.sourceLayer !== ES_MVT_META_LAYER_NAME
) {

View file

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`resolution editor should add super-fine option 1`] = `
exports[`render 1`] = `
<Fragment>
<EuiFormRow
describedByIds={Array []}
@ -39,39 +39,3 @@ exports[`resolution editor should add super-fine option 1`] = `
</EuiFormRow>
</Fragment>
`;
exports[`resolution editor should omit super-fine option 1`] = `
<Fragment>
<EuiFormRow
describedByIds={Array []}
display="columnCompressed"
fullWidth={false}
hasChildLabel={true}
hasEmptyLabelSpace={false}
label="Grid resolution"
labelType="label"
>
<EuiSelect
compressed={true}
onChange={[Function]}
options={
Array [
Object {
"text": "coarse",
"value": "COARSE",
},
Object {
"text": "fine",
"value": "FINE",
},
Object {
"text": "finest",
"value": "MOST_FINE",
},
]
}
value="COARSE"
/>
</EuiFormRow>
</Fragment>
`;

View file

@ -1,67 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`source editor geo_grid_source default vector layer config should allow super-fine option 1`] = `
<Fragment>
<EuiPanel>
<EuiTitle
size="xs"
>
<h6>
<FormattedMessage
defaultMessage="Metrics"
id="xpack.maps.source.esGrid.metricsLabel"
values={Object {}}
/>
</h6>
</EuiTitle>
<EuiSpacer
size="m"
/>
<MetricsEditor
allowMultipleMetrics={true}
fields={Array []}
key="12345"
metrics={Array []}
onChange={[Function]}
/>
</EuiPanel>
<EuiSpacer
size="s"
/>
<EuiPanel>
<EuiTitle
size="xs"
>
<h6>
<FormattedMessage
defaultMessage="Grid parameters"
id="xpack.maps.source.esGrid.geoTileGridLabel"
values={Object {}}
/>
</h6>
</EuiTitle>
<EuiSpacer
size="m"
/>
<ResolutionEditor
includeSuperFine={true}
metrics={Array []}
onChange={[Function]}
resolution="COARSE"
/>
<RenderAsSelect
isColumnCompressed={true}
onChange={[Function]}
renderAs="point"
/>
</EuiPanel>
<EuiSpacer
size="s"
/>
</Fragment>
`;
exports[`source editor geo_grid_source should put limitations based on heatmap-rendering selection should not allow super-fine option for heatmaps and should not allow multiple metrics 1`] = `
exports[`source editor geo_grid_source should not allow editing multiple metrics for heatmap 1`] = `
<Fragment>
<EuiPanel>
<EuiTitle
@ -106,7 +45,66 @@ exports[`source editor geo_grid_source should put limitations based on heatmap-r
size="m"
/>
<ResolutionEditor
includeSuperFine={false}
metrics={Array []}
onChange={[Function]}
resolution="COARSE"
/>
<RenderAsSelect
isColumnCompressed={true}
onChange={[Function]}
renderAs="point"
/>
</EuiPanel>
<EuiSpacer
size="s"
/>
</Fragment>
`;
exports[`source editor geo_grid_source should render editor 1`] = `
<Fragment>
<EuiPanel>
<EuiTitle
size="xs"
>
<h6>
<FormattedMessage
defaultMessage="Metrics"
id="xpack.maps.source.esGrid.metricsLabel"
values={Object {}}
/>
</h6>
</EuiTitle>
<EuiSpacer
size="m"
/>
<MetricsEditor
allowMultipleMetrics={true}
fields={Array []}
key="12345"
metrics={Array []}
onChange={[Function]}
/>
</EuiPanel>
<EuiSpacer
size="s"
/>
<EuiPanel>
<EuiTitle
size="xs"
>
<h6>
<FormattedMessage
defaultMessage="Grid parameters"
id="xpack.maps.source.esGrid.geoTileGridLabel"
values={Object {}}
/>
</h6>
</EuiTitle>
<EuiSpacer
size="m"
/>
<ResolutionEditor
metrics={Array []}
onChange={[Function]}
resolution="COARSE"

View file

@ -53,7 +53,7 @@ describe('ESGeoGridSource', () => {
metrics: [],
resolution: GRID_RESOLUTION.COARSE,
type: SOURCE_TYPES.ES_GEO_GRID,
requestType: RENDER_AS.HEATMAP,
requestType: RENDER_AS.POINT,
},
{}
);
@ -316,7 +316,7 @@ describe('ESGeoGridSource', () => {
expect(urlTemplateWithMeta.minSourceZoom).toBe(0);
expect(urlTemplateWithMeta.maxSourceZoom).toBe(24);
expect(urlTemplateWithMeta.urlTemplate).toEqual(
"rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=undefined&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'')),'6':('0':aggs,'1':())))&requestType=heatmap"
"rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=undefined&gridPrecision=8&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'')),'6':('0':aggs,'1':())))&requestType=point"
);
});
});

View file

@ -48,11 +48,11 @@ import { Adapters } from '../../../../../../../src/plugins/inspector/common/adap
import { isValidStringConfig } from '../../util/valid_string_config';
import { ITiledSingleLayerMvtParams } from '../tiled_single_layer_vector_source/tiled_single_layer_vector_source';
type ESGeoGridSourceSyncMeta = Pick<ESGeoGridSourceDescriptor, 'requestType'>;
type ESGeoGridSourceSyncMeta = Pick<ESGeoGridSourceDescriptor, 'requestType' | 'resolution'>;
const ES_MVT_AGGS_LAYER_NAME = 'aggs';
export const MAX_GEOTILE_LEVEL = 29;
const MAX_GEOTILE_LEVEL = 29;
export const clustersTitle = i18n.translate('xpack.maps.source.esGridClustersTitle', {
defaultMessage: 'Clusters and grids',
@ -103,6 +103,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
getSyncMeta(): ESGeoGridSourceSyncMeta {
return {
requestType: this._descriptor.requestType,
resolution: this._descriptor.resolution,
};
}
@ -134,6 +135,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
}
isMvt() {
// heatmap uses MVT regardless of resolution because heatmap only supports counting metrics
if (this._descriptor.requestType === RENDER_AS.HEATMAP) {
return true;
}
return this._descriptor.resolution === GRID_RESOLUTION.SUPER_FINE;
}
@ -142,7 +147,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
}
isGeoGridPrecisionAware(): boolean {
if (this._descriptor.resolution === GRID_RESOLUTION.SUPER_FINE) {
if (this.isMvt()) {
// MVT gridded data should not bootstrap each time the precision changes
// mapbox-gl needs to handle this
return false;
@ -183,6 +188,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
return 4;
}
if (this._descriptor.resolution === GRID_RESOLUTION.SUPER_FINE) {
return 8;
}
throw new Error(
i18n.translate('xpack.maps.source.esGrid.resolutionParamErrorMessage', {
defaultMessage: `Grid resolution param not recognized: {resolution}`,
@ -415,10 +424,12 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
} as GeoJsonWithMeta;
}
// TODO rename to getMvtSourceLayerName
getLayerName(): string {
return ES_MVT_AGGS_LAYER_NAME;
}
// TODO rename to getMvtUrlTemplateWithMeta
async getUrlTemplateWithMeta(
searchFilters: VectorSourceRequestMeta
): Promise<ITiledSingleLayerMvtParams> {
@ -433,11 +444,15 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
`/${GIS_API_PATH}/${MVT_GETGRIDTILE_API_PATH}/{z}/{x}/{y}.pbf`
);
const requestType =
this._descriptor.requestType === RENDER_AS.GRID ? RENDER_AS.GRID : RENDER_AS.POINT;
const urlTemplate = `${mvtUrlServicePath}\
?geometryFieldName=${this._descriptor.geoField}\
&index=${indexPattern.title}\
&gridPrecision=${this._getGeoGridPrecisionResolutionDelta()}\
&requestBody=${risonDsl}\
&requestType=${this._descriptor.requestType}`;
&requestType=${requestType}`;
return {
refreshTokenParamName: MVT_TOKEN_PARAM_NAME,
@ -449,7 +464,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
}
isFilterByMapBounds(): boolean {
if (this._descriptor.resolution === GRID_RESOLUTION.SUPER_FINE) {
if (this.isMvt()) {
// MVT gridded data. Should exclude bounds-filter from ES-DSL
return false;
} else {

View file

@ -14,17 +14,10 @@ import { GRID_RESOLUTION } from '../../../../common/constants';
const defaultProps = {
resolution: GRID_RESOLUTION.COARSE,
onChange: () => {},
includeSuperFine: false,
metrics: [],
};
describe('resolution editor', () => {
test('should omit super-fine option', () => {
const component = shallow(<ResolutionEditor {...defaultProps} />);
expect(component).toMatchSnapshot();
});
test('should add super-fine option', () => {
const component = shallow(<ResolutionEditor {...defaultProps} includeSuperFine={true} />);
expect(component).toMatchSnapshot();
});
test('render', () => {
const component = shallow(<ResolutionEditor {...defaultProps} />);
expect(component).toMatchSnapshot();
});

View file

@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n';
import { AggDescriptor } from '../../../../common/descriptor_types';
import { AGG_TYPE, GRID_RESOLUTION } from '../../../../common/constants';
const BASE_OPTIONS = [
const OPTIONS = [
{
value: GRID_RESOLUTION.COARSE,
text: i18n.translate('xpack.maps.source.esGrid.coarseDropdownOption', {
@ -31,6 +31,12 @@ const BASE_OPTIONS = [
defaultMessage: 'finest',
}),
},
{
value: GRID_RESOLUTION.SUPER_FINE,
text: i18n.translate('xpack.maps.source.esGrid.superFineDropDownOption', {
defaultMessage: 'super fine',
}),
},
];
function isUnsupportedVectorTileMetric(metric: AggDescriptor) {
@ -38,7 +44,6 @@ function isUnsupportedVectorTileMetric(metric: AggDescriptor) {
}
interface Props {
includeSuperFine: boolean;
resolution: GRID_RESOLUTION;
onChange: (resolution: GRID_RESOLUTION, metrics: AggDescriptor[]) => void;
metrics: AggDescriptor[];
@ -49,24 +54,9 @@ interface State {
}
export class ResolutionEditor extends Component<Props, State> {
private readonly _options = [...BASE_OPTIONS];
constructor(props: Props) {
super(props);
this.state = {
showModal: false,
};
if (props.includeSuperFine) {
this._options.push({
value: GRID_RESOLUTION.SUPER_FINE,
text: i18n.translate('xpack.maps.source.esGrid.superFineDropDownOption', {
defaultMessage: 'super fine',
}),
});
}
}
state: State = {
showModal: false,
};
_onResolutionChange = (e: ChangeEvent<HTMLSelectElement>) => {
const resolution = e.target.value as GRID_RESOLUTION;
@ -149,7 +139,7 @@ export class ResolutionEditor extends Component<Props, State> {
display="columnCompressed"
>
<EuiSelect
options={this._options}
options={OPTIONS}
value={this.props.resolution}
onChange={this._onResolutionChange}
compressed

View file

@ -27,19 +27,15 @@ const defaultProps = {
};
describe('source editor geo_grid_source', () => {
describe('default vector layer config', () => {
test('should allow super-fine option', async () => {
const component = shallow(<UpdateSourceEditor {...defaultProps} />);
expect(component).toMatchSnapshot();
});
test('should render editor', async () => {
const component = shallow(<UpdateSourceEditor {...defaultProps} />);
expect(component).toMatchSnapshot();
});
describe('should put limitations based on heatmap-rendering selection', () => {
test('should not allow super-fine option for heatmaps and should not allow multiple metrics', async () => {
const component = shallow(
<UpdateSourceEditor {...defaultProps} currentLayerType={LAYER_TYPE.HEATMAP} />
);
expect(component).toMatchSnapshot();
});
test('should not allow editing multiple metrics for heatmap', async () => {
const component = shallow(
<UpdateSourceEditor {...defaultProps} currentLayerType={LAYER_TYPE.HEATMAP} />
);
expect(component).toMatchSnapshot();
});
});

View file

@ -86,14 +86,6 @@ export class UpdateSourceEditor extends Component<Props, State> {
) {
newLayerType =
resolution === GRID_RESOLUTION.SUPER_FINE ? LAYER_TYPE.TILED_VECTOR : LAYER_TYPE.VECTOR;
} else if (this.props.currentLayerType === LAYER_TYPE.HEATMAP) {
if (resolution === GRID_RESOLUTION.SUPER_FINE) {
throw new Error('Heatmap does not support SUPER_FINE resolution');
} else {
newLayerType = LAYER_TYPE.HEATMAP;
}
} else {
throw new Error('Unexpected layer-type');
}
await this.props.onChange(
@ -163,7 +155,6 @@ export class UpdateSourceEditor extends Component<Props, State> {
</EuiTitle>
<EuiSpacer size="m" />
<ResolutionEditor
includeSuperFine={this.props.currentLayerType !== LAYER_TYPE.HEATMAP}
resolution={this.props.resolution}
onChange={this._onResolutionChange}
metrics={this.props.metrics}

View file

@ -1,18 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MVTFieldDescriptor } from '../../../../common/descriptor_types';
export interface MVTSingleLayerVectorSourceConfig {
urlTemplate: string;
layerName: string;
minSourceZoom: number;
maxSourceZoom: number;
fields?: MVTFieldDescriptor[];
tooltipProperties?: string[];
urlToken?: string;
}

View file

@ -5,4 +5,7 @@
* 2.0.
*/
export type { ITiledSingleLayerVectorSource } from './tiled_single_layer_vector_source';
export type {
ITiledSingleLayerMvtParams,
ITiledSingleLayerVectorSource,
} from './tiled_single_layer_vector_source';

View file

@ -7,7 +7,6 @@
import React from 'react';
import type { Map as MbMap } from '@kbn/mapbox-gl';
import { i18n } from '@kbn/i18n';
import { EuiIcon } from '@elastic/eui';
import { IStyle } from '../style';
import { HeatmapStyleEditor } from './components/heatmap_style_editor';
@ -68,11 +67,13 @@ export class HeatmapStyle implements IStyle {
mbMap,
layerId,
propertyName,
max,
resolution,
}: {
mbMap: MbMap;
layerId: string;
propertyName: string;
max: number;
resolution: GRID_RESOLUTION;
}) {
let radius;
@ -83,18 +84,11 @@ export class HeatmapStyle implements IStyle {
} else if (resolution === GRID_RESOLUTION.MOST_FINE) {
radius = 32;
} else {
// SUPER_FINE or any other is not supported.
const errorMessage = i18n.translate('xpack.maps.style.heatmap.resolutionStyleErrorMessage', {
defaultMessage: `Resolution param not recognized: {resolution}`,
values: { resolution },
});
throw new Error(errorMessage);
radius = 8;
}
mbMap.setPaintProperty(layerId, 'heatmap-radius', radius);
mbMap.setPaintProperty(layerId, 'heatmap-weight', {
type: 'identity',
property: propertyName,
});
const safeMax = max <= 0 ? 1 : max;
mbMap.setPaintProperty(layerId, 'heatmap-weight', ['/', ['get', propertyName], safeMax]);
const colorStops = getOrdinalMbColorRampStops(
this._descriptor.colorRampName,

View file

@ -24,6 +24,7 @@ import { clampToLatBounds, clampToLonBounds } from '../../../common/elasticsearc
import { getInitialView } from './get_initial_view';
import { getPreserveDrawingBuffer } from '../../kibana_services';
import { ILayer } from '../../classes/layers/layer';
import { IVectorSource } from '../../classes/sources/vector_source';
import { MapSettings } from '../../reducers/map';
import {
Goto,
@ -31,17 +32,13 @@ import {
TileMetaFeature,
Timeslice,
} from '../../../common/descriptor_types';
import {
DECIMAL_DEGREES_PRECISION,
LAYER_TYPE,
RawValue,
ZOOM_PRECISION,
} from '../../../common/constants';
import { DECIMAL_DEGREES_PRECISION, RawValue, ZOOM_PRECISION } from '../../../common/constants';
import { getGlyphUrl, isRetina } from '../../util';
import { syncLayerOrder } from './sort_layers';
import {
addSpriteSheetToMapFromImageData,
getTileMetaFeatures,
loadSpriteSheetImageData,
removeOrphanedSourcesAndLayers,
} from './utils';
@ -49,7 +46,6 @@ import { ResizeChecker } from '../../../../../../src/plugins/kibana_utils/public
import { RenderToolTipContent } from '../../classes/tooltips/tooltip_property';
import { TileStatusTracker } from './tile_status_tracker';
import { DrawFeatureControl } from './draw_control/draw_feature_control';
import { MvtVectorLayer } from '../../classes/layers/vector_layer';
import type { MapExtentState } from '../../reducers/map/types';
export interface Props {
@ -125,11 +121,16 @@ export class MbMap extends Component<Props, State> {
// This keeps track of the latest update calls, per layerId
_queryForMeta = (layer: ILayer) => {
if (this.state.mbMap && layer.isVisible() && layer.getType() === LAYER_TYPE.TILED_VECTOR) {
const mbFeatures = (layer as MvtVectorLayer).queryTileMetaFeatures(this.state.mbMap);
if (mbFeatures !== null) {
this.props.updateMetaFromTiles(layer.getId(), mbFeatures);
}
const source = layer.getSource();
if (
this.state.mbMap &&
layer.isVisible() &&
source.isESSource() &&
typeof (source as IVectorSource).isMvt === 'function' &&
(source as IVectorSource).isMvt()
) {
const features = getTileMetaFeatures(this.state.mbMap, layer.getMbSourceId());
this.props.updateMetaFromTiles(layer.getId(), features);
}
};

View file

@ -6,11 +6,13 @@
*/
import type { Map as MbMap } from '@kbn/mapbox-gl';
import { TileMetaFeature } from '../../../common/descriptor_types';
// @ts-expect-error
import { RGBAImage } from './image_utils';
import { isGlDrawLayer } from './sort_layers';
import { ILayer } from '../../classes/layers/layer';
import { EmsSpriteSheet } from '../../classes/layers/vector_tile_layer/vector_tile_layer';
import { ES_MVT_META_LAYER_NAME } from '../../classes/layers/vector_layer/mvt_vector_layer/mvt_vector_layer';
export function removeOrphanedSourcesAndLayers(
mbMap: MbMap,
@ -119,3 +121,26 @@ export function addSpriteSheetToMapFromImageData(
mbMap.addImage(imageId, data, { pixelRatio, sdf });
}
}
export function getTileMetaFeatures(mbMap: MbMap, mbSourceId: string): TileMetaFeature[] {
// querySourceFeatures can return duplicated features when features cross tile boundaries.
// Tile meta will never have duplicated features since by there nature, tile meta is a feature contained within a single tile
const mbFeatures = mbMap.querySourceFeatures(mbSourceId, {
sourceLayer: ES_MVT_META_LAYER_NAME,
});
return mbFeatures
.map((mbFeature) => {
try {
return {
type: 'Feature',
id: mbFeature?.id,
geometry: mbFeature?.geometry, // this getter might throw with non-conforming geometries
properties: mbFeature?.properties,
} as TileMetaFeature;
} catch (e) {
return null;
}
})
.filter((mbFeature) => mbFeature !== null) as TileMetaFeature[];
}

View file

@ -23,6 +23,7 @@ export async function getEsGridTile({
z,
requestBody = {},
requestType = RENDER_AS.POINT,
gridPrecision,
abortController,
}: {
x: number;
@ -34,13 +35,14 @@ export async function getEsGridTile({
logger: Logger;
requestBody: any;
requestType: RENDER_AS.GRID | RENDER_AS.POINT;
gridPrecision: number;
abortController: AbortController;
}): Promise<Buffer | null> {
try {
const path = `/${encodeURIComponent(index)}/_mvt/${geometryFieldName}/${z}/${x}/${y}`;
const body = {
size: 0, // no hits
grid_precision: 8,
grid_precision: gridPrecision,
exact_bounds: false,
extent: 4096, // full resolution,
query: requestBody.query,

View file

@ -90,6 +90,7 @@ export function initMVTRoutes({
index: schema.string(),
requestType: schema.string(),
token: schema.maybe(schema.string()),
gridPrecision: schema.number(),
}),
},
},
@ -117,6 +118,7 @@ export function initMVTRoutes({
index: query.index as string,
requestBody: requestBodyDSL as any,
requestType: query.requestType as RENDER_AS.POINT | RENDER_AS.GRID,
gridPrecision: parseInt(query.gridPrecision, 10),
abortController,
});

View file

@ -15437,7 +15437,6 @@
"xpack.maps.style.customColorPaletteLabel": "カスタムカラーパレット",
"xpack.maps.style.customColorRampLabel": "カスタマカラーランプ",
"xpack.maps.style.fieldSelect.OriginLabel": "{fieldOrigin} からのフィールド",
"xpack.maps.style.heatmap.resolutionStyleErrorMessage": "解像度パラメーターが認識されません:{resolution}",
"xpack.maps.styles.categorical.otherCategoryLabel": "その他",
"xpack.maps.styles.categoricalDataMapping.isEnabled.local": "無効にすると、ローカルデータからカテゴリを計算し、データが変更されたときにカテゴリを再計算します。ユーザーがパン、ズーム、フィルターを使用するときにスタイルが一貫しない場合があります。",
"xpack.maps.styles.categoricalDataMapping.isEnabled.server": "データセット全体からカテゴリを計算します。ユーザーがパン、ズーム、フィルターを使用するときにスタイルが一貫します。",

View file

@ -15636,7 +15636,6 @@
"xpack.maps.style.customColorPaletteLabel": "定制调色板",
"xpack.maps.style.customColorRampLabel": "定制颜色渐变",
"xpack.maps.style.fieldSelect.OriginLabel": "来自 {fieldOrigin} 的字段",
"xpack.maps.style.heatmap.resolutionStyleErrorMessage": "无法识别分辨率参数:{resolution}",
"xpack.maps.styles.categorical.otherCategoryLabel": "其他",
"xpack.maps.styles.categoricalDataMapping.isEnabled.local": "禁用后,从本地数据计算类别,并在数据更改时重新计算类别。在用户平移、缩放和筛选时,样式可能不一致。",
"xpack.maps.styles.categoricalDataMapping.isEnabled.server": "从整个数据集计算类别。在用户平移、缩放和筛选时,样式保持一致。",

View file

@ -19,6 +19,7 @@ export default function ({ getService }) {
`/api/maps/mvt/getGridTile/3/2/3.pbf\
?geometryFieldName=geo.coordinates\
&index=logstash-*\
&gridPrecision=8\
&requestBody=(_source:(excludes:!()),aggs:(avg_of_bytes:(avg:(field:bytes))),fields:!((field:%27@timestamp%27,format:date_time),(field:%27relatedContent.article:modified_time%27,format:date_time),(field:%27relatedContent.article:published_time%27,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27@timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))\
&requestType=point`
)
@ -80,6 +81,7 @@ export default function ({ getService }) {
`/api/maps/mvt/getGridTile/3/2/3.pbf\
?geometryFieldName=geo.coordinates\
&index=logstash-*\
&gridPrecision=8\
&requestBody=(_source:(excludes:!()),aggs:(avg_of_bytes:(avg:(field:bytes))),fields:!((field:%27@timestamp%27,format:date_time),(field:%27relatedContent.article:modified_time%27,format:date_time),(field:%27relatedContent.article:published_time%27,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27@timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))\
&requestType=grid`
)

View file

@ -13,7 +13,7 @@ export default function ({ getPageObjects, getService }) {
const DOC_COUNT_PROP_NAME = 'doc_count';
const security = getService('security');
describe('layer geo grid aggregation source', () => {
describe('geojson vector layer - es geo grid source', () => {
const DATA_CENTER_LON = -98;
const DATA_CENTER_LAT = 38;
@ -102,69 +102,7 @@ export default function ({ getPageObjects, getService }) {
});
}
describe('heatmap', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('geo grid heatmap example');
});
const LAYER_ID = '3xlvm';
const HEATMAP_PROP_NAME = '__kbn_heatmap_weight__';
it('should re-fetch geotile_grid aggregation with refresh timer', async () => {
const beforeRefreshTimerTimestamp = await getRequestTimestamp();
expect(beforeRefreshTimerTimestamp.length).to.be(24);
await PageObjects.maps.triggerSingleRefresh(1000);
const afterRefreshTimerTimestamp = await getRequestTimestamp();
expect(beforeRefreshTimerTimestamp).not.to.equal(afterRefreshTimerTimestamp);
});
it('should decorate feature properties with scaled doc_count property', async () => {
const mapboxStyle = await PageObjects.maps.getMapboxStyle();
expect(mapboxStyle.sources[LAYER_ID].data.features.length).to.equal(6);
mapboxStyle.sources[LAYER_ID].data.features.forEach(({ properties }) => {
expect(properties.hasOwnProperty(HEATMAP_PROP_NAME)).to.be(true);
expect(properties.hasOwnProperty(DOC_COUNT_PROP_NAME)).to.be(true);
});
});
makeRequestTestsForGeoPrecision(LAYER_ID, 4, 2);
describe('query bar', () => {
before(async () => {
await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"');
await PageObjects.maps.setView(0, 0, 0);
});
after(async () => {
await PageObjects.maps.setAndSubmitQuery('');
});
it('should apply query to geotile_grid aggregation request', async () => {
const { rawResponse: response } = await PageObjects.maps.getResponse();
expect(response.aggregations.gridSplit.buckets.length).to.equal(1);
});
});
describe('inspector', () => {
afterEach(async () => {
await inspector.close();
});
it('should contain geotile_grid aggregation elasticsearch request', async () => {
const { rawResponse: response } = await PageObjects.maps.getResponse();
expect(response.aggregations.gridSplit.buckets.length).to.equal(4);
});
it('should not contain any elasticsearch request after layer is deleted', async () => {
await PageObjects.maps.removeLayer('logstash-*');
const noRequests = await PageObjects.maps.doesInspectorHaveRequests();
expect(noRequests).to.equal(true);
});
});
});
describe('vector(grid)', () => {
describe('geo_point', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('geo grid vector grid example');
});
@ -227,7 +165,7 @@ export default function ({ getPageObjects, getService }) {
});
});
describe('vector grid with geo_shape', () => {
describe('geo_shape', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('geo grid vector grid example with shape');
});

View file

@ -31,12 +31,25 @@ export default function ({ getPageObjects, getService }) {
await PageObjects.maps.loadSavedMap('MVT geotile grid (style meta from ES)');
const mapboxStyle = await PageObjects.maps.getMapboxStyle();
//Source should be correct
expect(
mapboxStyle.sources[MB_VECTOR_SOURCE_ID].tiles[0].startsWith(
`/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf?geometryFieldName=geo.coordinates&index=logstash-*&requestBody=(_source:(excludes:!()),aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))&requestType=grid`
)
).to.equal(true);
const tileUrl = new URL(
mapboxStyle.sources[MB_VECTOR_SOURCE_ID].tiles[0],
'http://absolute_path'
);
const searchParams = Object.fromEntries(tileUrl.searchParams);
expect(tileUrl.pathname).to.equal('/api/maps/mvt/getGridTile/%7Bz%7D/%7Bx%7D/%7By%7D.pbf');
// token is an unique id that changes between runs
expect(typeof searchParams.token).to.equal('string');
delete searchParams.token;
expect(searchParams).to.eql({
geometryFieldName: 'geo.coordinates',
index: 'logstash-*',
gridPrecision: 8,
requestType: 'grid',
requestBody: `(_source:(excludes:!()),aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))`,
});
//Should correctly load meta for style-rule (sigma is set to 1, opacity to 1)
const fillLayer = mapboxStyle.layers.find(
@ -169,5 +182,35 @@ export default function ({ getPageObjects, getService }) {
'fill-opacity': 0.75,
});
});
it('should render heatmap layer', async () => {
await PageObjects.maps.loadSavedMap('geo grid heatmap example');
const mapboxStyle = await PageObjects.maps.getMapboxStyle();
const heatmapLayer = mapboxStyle.layers.find((layer) => layer.id === '3xlvm_heatmap');
expect(heatmapLayer.paint).to.eql({
'heatmap-radius': 128,
'heatmap-color': [
'interpolate',
['linear'],
['heatmap-density'],
0,
'rgba(0, 0, 255, 0)',
0.1,
'rgb(65, 105, 225)',
0.28,
'rgb(0, 256, 256)',
0.45999999999999996,
'rgb(0, 256, 0)',
0.64,
'rgb(256, 256, 0)',
0.82,
'rgb(256, 0, 0)',
],
'heatmap-opacity': 0.75,
'heatmap-weight': ['/', ['get', '_count'], 1],
});
});
});
}