mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
[Maps] migrate mvt routes to use data.search service instead of directly calling elasticsearch (#89904) (#90104)
* [Maps] migrate mvt routes to use data.search service instead of directly calling elasticsearch * pass search session id to mvt requests * move grid tile tests to integration tests * replace getTile unit test with integration test * add comment about request param * revert total meta change * tslint fixes * update jest tests
This commit is contained in:
Nathan Reese
GitHub
parent
1f0ecfbfc1
commit
d2c26cf28b
12 changed files with 306 additions and 414 deletions
|
|
@ -278,6 +278,17 @@ describe('ESGeoGridSource', () => {
|
|||
"rootdir/api/maps/mvt/getGridTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=undefined&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!((geo_bounding_box:(bar:(bottom_right:!(180,-82.67628),top_left:!(-180,82.67628)))))),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':aggs,'1':(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:bar))),geotile_grid:(bounds:!n,field:bar,precision:!n,shard_size:65535,size:65535))))))&requestType=heatmap&geoFieldType=geo_point"
|
||||
);
|
||||
});
|
||||
|
||||
it('should include searchSourceId in urlTemplateWithMeta', async () => {
|
||||
const urlTemplateWithMeta = await geogridSource.getUrlTemplateWithMeta({
|
||||
...vectorSourceRequestMeta,
|
||||
searchSessionId: '1',
|
||||
});
|
||||
|
||||
expect(urlTemplateWithMeta.urlTemplate).toBe(
|
||||
"rootdir/api/maps/mvt/getGridTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=undefined&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:())),'1':('0':size,'1':0),'2':('0':filter,'1':!((geo_bounding_box:(bar:(bottom_right:!(180,-82.67628),top_left:!(-180,82.67628)))))),'3':('0':query),'4':('0':index,'1':(fields:())),'5':('0':query,'1':(language:KQL,query:'',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':aggs,'1':(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:bar))),geotile_grid:(bounds:!n,field:bar,precision:!n,shard_size:65535,size:65535))))))&requestType=heatmap&geoFieldType=geo_point&searchSessionId=1"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Gold+ usage', () => {
|
||||
|
|
|
|||
|
|
@ -443,12 +443,22 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
|
|||
);
|
||||
|
||||
const geoField = await this._getGeoField();
|
||||
const urlTemplate = `${mvtUrlServicePath}?x={x}&y={y}&z={z}&geometryFieldName=${this._descriptor.geoField}&index=${indexPattern.title}&requestBody=${risonDsl}&requestType=${this._descriptor.requestType}&geoFieldType=${geoField.type}`;
|
||||
const urlTemplate = `${mvtUrlServicePath}\
|
||||
?x={x}\
|
||||
&y={y}\
|
||||
&z={z}\
|
||||
&geometryFieldName=${this._descriptor.geoField}\
|
||||
&index=${indexPattern.title}\
|
||||
&requestBody=${risonDsl}\
|
||||
&requestType=${this._descriptor.requestType}\
|
||||
&geoFieldType=${geoField.type}`;
|
||||
return {
|
||||
layerName: this.getLayerName(),
|
||||
minSourceZoom: this.getMinZoom(),
|
||||
maxSourceZoom: this.getMaxZoom(),
|
||||
urlTemplate,
|
||||
urlTemplate: searchFilters.searchSessionId
|
||||
? urlTemplate + `&searchSessionId=${searchFilters.searchSessionId}`
|
||||
: urlTemplate,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -116,6 +116,20 @@ describe('ESSearchSource', () => {
|
|||
`rootdir/api/maps/mvt/getTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=foobar-title-*&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':fieldsFromSource,'1':!(tooltipField,styleField)),'7':('0':source,'1':!(tooltipField,styleField))))&geoFieldType=geo_shape`
|
||||
);
|
||||
});
|
||||
|
||||
it('should include searchSourceId in urlTemplateWithMeta', async () => {
|
||||
const esSearchSource = new ESSearchSource({
|
||||
geoField: geoFieldName,
|
||||
indexPatternId: 'ipId',
|
||||
});
|
||||
const urlTemplateWithMeta = await esSearchSource.getUrlTemplateWithMeta({
|
||||
...searchFilters,
|
||||
searchSessionId: '1',
|
||||
});
|
||||
expect(urlTemplateWithMeta.urlTemplate).toBe(
|
||||
`rootdir/api/maps/mvt/getTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=foobar-title-*&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':fieldsFromSource,'1':!(tooltipField,styleField)),'7':('0':source,'1':!(tooltipField,styleField))))&geoFieldType=geo_shape&searchSessionId=1`
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -729,12 +729,21 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
|
|||
|
||||
const geoField = await this._getGeoField();
|
||||
|
||||
const urlTemplate = `${mvtUrlServicePath}?x={x}&y={y}&z={z}&geometryFieldName=${this._descriptor.geoField}&index=${indexPattern.title}&requestBody=${risonDsl}&geoFieldType=${geoField.type}`;
|
||||
const urlTemplate = `${mvtUrlServicePath}\
|
||||
?x={x}\
|
||||
&y={y}\
|
||||
&z={z}\
|
||||
&geometryFieldName=${this._descriptor.geoField}\
|
||||
&index=${indexPattern.title}\
|
||||
&requestBody=${risonDsl}\
|
||||
&geoFieldType=${geoField.type}`;
|
||||
return {
|
||||
layerName: this.getLayerName(),
|
||||
minSourceZoom: this.getMinZoom(),
|
||||
maxSourceZoom: this.getMaxZoom(),
|
||||
urlTemplate,
|
||||
urlTemplate: searchFilters.searchSessionId
|
||||
? urlTemplate + `&searchSessionId=${searchFilters.searchSessionId}`
|
||||
: urlTemplate,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
{"took":2,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"gridSplit":{"buckets":[{"key":"7/37/48","doc_count":42637,"avg_of_TOTAL_AV":{"value":5398920.390458991},"gridCentroid":{"location":{"lat":40.77936432658204,"lon":-73.96795676049909},"count":42637}}]}}}
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"took":0,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0,"hits":[{"_index":"poly","_id":"G7PRMXQBgyyZ-h5iYibj","_score":0,"_source":{"coordinates":{"coordinates":[[[-106.171875,36.59788913307022],[-50.625,-22.91792293614603],[4.921875,42.8115217450979],[-33.046875,63.54855223203644],[-66.796875,63.860035895395306],[-106.171875,36.59788913307022]]],"type":"polygon"}}}]}}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
|
||||
export const TILE_SEARCHES = {
|
||||
'0.0.0': {
|
||||
countResponse: {
|
||||
count: 1,
|
||||
_shards: {
|
||||
total: 1,
|
||||
successful: 1,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
},
|
||||
},
|
||||
searchResponse: loadJson('./json/0_0_0_search.json'),
|
||||
},
|
||||
};
|
||||
|
||||
export const TILE_GRIDAGGS = {
|
||||
'0.0.0': {
|
||||
gridAggResponse: loadJson('./json/0_0_0_gridagg.json'),
|
||||
},
|
||||
};
|
||||
|
||||
function loadJson(filePath: string) {
|
||||
const absolutePath = path.resolve(__dirname, filePath);
|
||||
const rawContents = fs.readFileSync(absolutePath);
|
||||
return JSON.parse((rawContents as unknown) as string);
|
||||
}
|
||||
|
|
@ -1,261 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { getGridTile, getTile } from './get_tile';
|
||||
import { TILE_GRIDAGGS, TILE_SEARCHES } from './__fixtures__/tile_es_responses';
|
||||
import { Logger } from 'src/core/server';
|
||||
import {
|
||||
ES_GEO_FIELD_TYPE,
|
||||
KBN_IS_CENTROID_FEATURE,
|
||||
MVT_SOURCE_LAYER_NAME,
|
||||
RENDER_AS,
|
||||
} from '../../common/constants';
|
||||
|
||||
// @ts-expect-error
|
||||
import { VectorTile, VectorTileLayer } from '@mapbox/vector-tile';
|
||||
// @ts-expect-error
|
||||
import Protobuf from 'pbf';
|
||||
|
||||
interface ITileLayerJsonExpectation {
|
||||
version: number;
|
||||
name: string;
|
||||
extent: number;
|
||||
features: Array<{
|
||||
id: string | number | undefined;
|
||||
type: number;
|
||||
properties: object;
|
||||
extent: number;
|
||||
pointArrays: object;
|
||||
}>;
|
||||
}
|
||||
|
||||
describe('getTile', () => {
|
||||
const mockCallElasticsearch = jest.fn();
|
||||
|
||||
const requestBody = {
|
||||
_source: { excludes: [] },
|
||||
docvalue_fields: [],
|
||||
query: { bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } },
|
||||
script_fields: {},
|
||||
size: 10000,
|
||||
stored_fields: ['*'],
|
||||
};
|
||||
const geometryFieldName = 'coordinates';
|
||||
|
||||
beforeEach(() => {
|
||||
mockCallElasticsearch.mockReset();
|
||||
});
|
||||
|
||||
test('0.0.0 - under limit', async () => {
|
||||
mockCallElasticsearch.mockImplementation((type) => {
|
||||
if (type === 'count') {
|
||||
return TILE_SEARCHES['0.0.0'].countResponse;
|
||||
} else if (type === 'search') {
|
||||
return TILE_SEARCHES['0.0.0'].searchResponse;
|
||||
} else {
|
||||
throw new Error(`${type} not recognized`);
|
||||
}
|
||||
});
|
||||
|
||||
const pbfTile = await getTile({
|
||||
x: 0,
|
||||
y: 0,
|
||||
z: 0,
|
||||
index: 'world_countries',
|
||||
requestBody,
|
||||
geometryFieldName,
|
||||
logger: ({
|
||||
info: () => {},
|
||||
} as unknown) as Logger,
|
||||
callElasticsearch: mockCallElasticsearch,
|
||||
geoFieldType: ES_GEO_FIELD_TYPE.GEO_SHAPE,
|
||||
});
|
||||
|
||||
const jsonTile = new VectorTile(new Protobuf(pbfTile));
|
||||
compareJsonTiles(jsonTile, {
|
||||
version: 2,
|
||||
name: 'source_layer',
|
||||
extent: 4096,
|
||||
features: [
|
||||
{
|
||||
id: undefined,
|
||||
type: 3,
|
||||
properties: {
|
||||
__kbn__feature_id__: 'poly:G7PRMXQBgyyZ-h5iYibj:0',
|
||||
_id: 'G7PRMXQBgyyZ-h5iYibj',
|
||||
_index: 'poly',
|
||||
},
|
||||
extent: 4096,
|
||||
pointArrays: [
|
||||
[
|
||||
{ x: 840, y: 1600 },
|
||||
{ x: 1288, y: 1096 },
|
||||
{ x: 1672, y: 1104 },
|
||||
{ x: 2104, y: 1508 },
|
||||
{ x: 1472, y: 2316 },
|
||||
{ x: 840, y: 1600 },
|
||||
],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: undefined,
|
||||
type: 1,
|
||||
properties: {
|
||||
__kbn__feature_id__: 'poly:G7PRMXQBgyyZ-h5iYibj:0',
|
||||
_id: 'G7PRMXQBgyyZ-h5iYibj',
|
||||
_index: 'poly',
|
||||
[KBN_IS_CENTROID_FEATURE]: true,
|
||||
},
|
||||
extent: 4096,
|
||||
pointArrays: [[{ x: 1470, y: 1702 }]],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGridTile', () => {
|
||||
const mockCallElasticsearch = jest.fn();
|
||||
|
||||
const geometryFieldName = 'geometry';
|
||||
|
||||
// For mock-purposes only. The ES-call response is mocked in 0_0_0_gridagg.json file
|
||||
const requestBody = {
|
||||
_source: { excludes: [] },
|
||||
aggs: {
|
||||
gridSplit: {
|
||||
aggs: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
avg_of_TOTAL_AV: { avg: { field: 'TOTAL_AV' } },
|
||||
gridCentroid: { geo_centroid: { field: geometryFieldName } },
|
||||
},
|
||||
geotile_grid: {
|
||||
bounds: null,
|
||||
field: geometryFieldName,
|
||||
precision: null,
|
||||
shard_size: 65535,
|
||||
size: 65535,
|
||||
},
|
||||
},
|
||||
},
|
||||
docvalue_fields: [],
|
||||
query: {
|
||||
bool: {
|
||||
filter: [],
|
||||
},
|
||||
},
|
||||
script_fields: {},
|
||||
size: 0,
|
||||
stored_fields: ['*'],
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockCallElasticsearch.mockReset();
|
||||
mockCallElasticsearch.mockImplementation((type) => {
|
||||
return TILE_GRIDAGGS['0.0.0'].gridAggResponse;
|
||||
});
|
||||
});
|
||||
|
||||
const defaultParams = {
|
||||
x: 0,
|
||||
y: 0,
|
||||
z: 0,
|
||||
index: 'manhattan',
|
||||
requestBody,
|
||||
geometryFieldName,
|
||||
logger: ({
|
||||
info: () => {},
|
||||
} as unknown) as Logger,
|
||||
callElasticsearch: mockCallElasticsearch,
|
||||
requestType: RENDER_AS.POINT,
|
||||
geoFieldType: ES_GEO_FIELD_TYPE.GEO_POINT,
|
||||
};
|
||||
|
||||
test('0.0.0 tile (clusters)', async () => {
|
||||
const pbfTile = await getGridTile(defaultParams);
|
||||
const jsonTile = new VectorTile(new Protobuf(pbfTile));
|
||||
compareJsonTiles(jsonTile, {
|
||||
version: 2,
|
||||
name: 'source_layer',
|
||||
extent: 4096,
|
||||
features: [
|
||||
{
|
||||
id: undefined,
|
||||
type: 1,
|
||||
properties: {
|
||||
['avg_of_TOTAL_AV']: 5398920.390458991,
|
||||
doc_count: 42637,
|
||||
},
|
||||
extent: 4096,
|
||||
pointArrays: [[{ x: 1206, y: 1539 }]],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test('0.0.0 tile (grids)', async () => {
|
||||
const pbfTile = await getGridTile({ ...defaultParams, requestType: RENDER_AS.GRID });
|
||||
const jsonTile = new VectorTile(new Protobuf(pbfTile));
|
||||
compareJsonTiles(jsonTile, {
|
||||
version: 2,
|
||||
name: 'source_layer',
|
||||
extent: 4096,
|
||||
features: [
|
||||
{
|
||||
id: undefined,
|
||||
type: 3,
|
||||
properties: {
|
||||
['avg_of_TOTAL_AV']: 5398920.390458991,
|
||||
doc_count: 42637,
|
||||
},
|
||||
extent: 4096,
|
||||
pointArrays: [
|
||||
[
|
||||
{ x: 1216, y: 1536 },
|
||||
{ x: 1216, y: 1568 },
|
||||
{ x: 1184, y: 1568 },
|
||||
{ x: 1184, y: 1536 },
|
||||
{ x: 1216, y: 1536 },
|
||||
],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: undefined,
|
||||
type: 1,
|
||||
properties: {
|
||||
['avg_of_TOTAL_AV']: 5398920.390458991,
|
||||
doc_count: 42637,
|
||||
[KBN_IS_CENTROID_FEATURE]: true,
|
||||
},
|
||||
extent: 4096,
|
||||
pointArrays: [[{ x: 1200, y: 1552 }]],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Verifies JSON-representation of tile-contents
|
||||
* @param actualTileJson
|
||||
* @param expectedLayer
|
||||
*/
|
||||
function compareJsonTiles(actualTileJson: VectorTile, expectedLayer: ITileLayerJsonExpectation) {
|
||||
const actualLayer: VectorTileLayer = actualTileJson.layers[MVT_SOURCE_LAYER_NAME];
|
||||
expect(actualLayer.version).toEqual(expectedLayer.version);
|
||||
expect(actualLayer.extent).toEqual(expectedLayer.extent);
|
||||
expect(actualLayer.name).toEqual(expectedLayer.name);
|
||||
expect(actualLayer.length).toEqual(expectedLayer.features.length);
|
||||
|
||||
expectedLayer.features.forEach((expectedFeature, index) => {
|
||||
const actualFeature = actualLayer.feature(index);
|
||||
expect(actualFeature.type).toEqual(expectedFeature.type);
|
||||
expect(actualFeature.extent).toEqual(expectedFeature.extent);
|
||||
expect(actualFeature.id).toEqual(expectedFeature.id);
|
||||
expect(actualFeature.properties).toEqual(expectedFeature.properties);
|
||||
expect(actualFeature.loadGeometry()).toEqual(expectedFeature.pointArrays);
|
||||
});
|
||||
}
|
||||
|
|
@ -8,7 +8,8 @@
|
|||
import geojsonvt from 'geojson-vt';
|
||||
// @ts-expect-error
|
||||
import vtpbf from 'vt-pbf';
|
||||
import { Logger } from 'src/core/server';
|
||||
import { Logger, RequestHandlerContext } from 'src/core/server';
|
||||
import type { DataApiRequestHandlerContext } from 'src/plugins/data/server';
|
||||
import { Feature, FeatureCollection, Polygon } from 'geojson';
|
||||
import {
|
||||
ES_GEO_FIELD_TYPE,
|
||||
|
|
@ -28,7 +29,7 @@ import { getCentroidFeatures } from '../../common/get_centroid_features';
|
|||
|
||||
export async function getGridTile({
|
||||
logger,
|
||||
callElasticsearch,
|
||||
context,
|
||||
index,
|
||||
geometryFieldName,
|
||||
x,
|
||||
|
|
@ -37,17 +38,19 @@ export async function getGridTile({
|
|||
requestBody = {},
|
||||
requestType = RENDER_AS.POINT,
|
||||
geoFieldType = ES_GEO_FIELD_TYPE.GEO_POINT,
|
||||
searchSessionId,
|
||||
}: {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
geometryFieldName: string;
|
||||
index: string;
|
||||
callElasticsearch: (type: string, ...args: any[]) => Promise<unknown>;
|
||||
context: RequestHandlerContext & { search: DataApiRequestHandlerContext };
|
||||
logger: Logger;
|
||||
requestBody: any;
|
||||
requestType: RENDER_AS;
|
||||
geoFieldType: ES_GEO_FIELD_TYPE;
|
||||
searchSessionId?: string;
|
||||
}): Promise<Buffer | null> {
|
||||
const esBbox: ESBounds = tileToESBbox(x, y, z);
|
||||
try {
|
||||
|
|
@ -79,13 +82,20 @@ export async function getGridTile({
|
|||
);
|
||||
requestBody.aggs[GEOTILE_GRID_AGG_NAME].geotile_grid.bounds = esBbox;
|
||||
|
||||
const esGeotileGridQuery = {
|
||||
index,
|
||||
body: requestBody,
|
||||
};
|
||||
|
||||
const gridAggResult = await callElasticsearch('search', esGeotileGridQuery);
|
||||
const features: Feature[] = convertRegularRespToGeoJson(gridAggResult, requestType);
|
||||
const response = await context
|
||||
.search!.search(
|
||||
{
|
||||
params: {
|
||||
index,
|
||||
body: requestBody,
|
||||
},
|
||||
},
|
||||
{
|
||||
sessionId: searchSessionId,
|
||||
}
|
||||
)
|
||||
.toPromise();
|
||||
const features: Feature[] = convertRegularRespToGeoJson(response.rawResponse, requestType);
|
||||
const featureCollection: FeatureCollection = {
|
||||
features,
|
||||
type: 'FeatureCollection',
|
||||
|
|
@ -100,7 +110,7 @@ export async function getGridTile({
|
|||
|
||||
export async function getTile({
|
||||
logger,
|
||||
callElasticsearch,
|
||||
context,
|
||||
index,
|
||||
geometryFieldName,
|
||||
x,
|
||||
|
|
@ -108,112 +118,121 @@ export async function getTile({
|
|||
z,
|
||||
requestBody = {},
|
||||
geoFieldType,
|
||||
searchSessionId,
|
||||
}: {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
geometryFieldName: string;
|
||||
index: string;
|
||||
callElasticsearch: (type: string, ...args: any[]) => Promise<unknown>;
|
||||
context: RequestHandlerContext & { search: DataApiRequestHandlerContext };
|
||||
logger: Logger;
|
||||
requestBody: any;
|
||||
geoFieldType: ES_GEO_FIELD_TYPE;
|
||||
searchSessionId?: string;
|
||||
}): Promise<Buffer | null> {
|
||||
const geojsonBbox = tileToGeoJsonPolygon(x, y, z);
|
||||
|
||||
let resultFeatures: Feature[];
|
||||
let features: Feature[];
|
||||
try {
|
||||
let result;
|
||||
try {
|
||||
const geoShapeFilter = {
|
||||
geo_shape: {
|
||||
[geometryFieldName]: {
|
||||
shape: geojsonBbox,
|
||||
relation: 'INTERSECTS',
|
||||
requestBody.query.bool.filter.push({
|
||||
geo_shape: {
|
||||
[geometryFieldName]: {
|
||||
shape: tileToGeoJsonPolygon(x, y, z),
|
||||
relation: 'INTERSECTS',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const searchOptions = {
|
||||
sessionId: searchSessionId,
|
||||
};
|
||||
|
||||
const countResponse = await context
|
||||
.search!.search(
|
||||
{
|
||||
params: {
|
||||
index,
|
||||
body: {
|
||||
size: 0,
|
||||
query: requestBody.query,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
requestBody.query.bool.filter.push(geoShapeFilter);
|
||||
searchOptions
|
||||
)
|
||||
.toPromise();
|
||||
|
||||
const esSearchQuery = {
|
||||
index,
|
||||
body: requestBody,
|
||||
};
|
||||
|
||||
const esCountQuery = {
|
||||
index,
|
||||
body: {
|
||||
query: requestBody.query,
|
||||
},
|
||||
};
|
||||
|
||||
const countResult = await callElasticsearch('count', esCountQuery);
|
||||
|
||||
// @ts-expect-error
|
||||
if (countResult.count > requestBody.size) {
|
||||
// Generate "too many features"-bounds
|
||||
const bboxAggName = 'data_bounds';
|
||||
const bboxQuery = {
|
||||
index,
|
||||
body: {
|
||||
size: 0,
|
||||
query: requestBody.query,
|
||||
aggs: {
|
||||
[bboxAggName]: {
|
||||
geo_bounds: {
|
||||
field: geometryFieldName,
|
||||
if (countResponse.rawResponse.hits.total > requestBody.size) {
|
||||
// Generate "too many features"-bounds
|
||||
const bboxResponse = await context
|
||||
.search!.search(
|
||||
{
|
||||
params: {
|
||||
index,
|
||||
body: {
|
||||
size: 0,
|
||||
query: requestBody.query,
|
||||
aggs: {
|
||||
data_bounds: {
|
||||
geo_bounds: {
|
||||
field: geometryFieldName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
searchOptions
|
||||
)
|
||||
.toPromise();
|
||||
|
||||
const bboxResult = await callElasticsearch('search', bboxQuery);
|
||||
|
||||
// @ts-expect-error
|
||||
const bboxForData = esBboxToGeoJsonPolygon(bboxResult.aggregations[bboxAggName].bounds);
|
||||
|
||||
resultFeatures = [
|
||||
features = [
|
||||
{
|
||||
type: 'Feature',
|
||||
properties: {
|
||||
[KBN_TOO_MANY_FEATURES_PROPERTY]: true,
|
||||
},
|
||||
geometry: esBboxToGeoJsonPolygon(
|
||||
bboxResponse.rawResponse.aggregations.data_bounds.bounds
|
||||
),
|
||||
},
|
||||
];
|
||||
} else {
|
||||
const documentsResponse = await context
|
||||
.search!.search(
|
||||
{
|
||||
type: 'Feature',
|
||||
properties: {
|
||||
[KBN_TOO_MANY_FEATURES_PROPERTY]: true,
|
||||
params: {
|
||||
index,
|
||||
body: requestBody,
|
||||
},
|
||||
geometry: bboxForData,
|
||||
},
|
||||
];
|
||||
} else {
|
||||
result = await callElasticsearch('search', esSearchQuery);
|
||||
searchOptions
|
||||
)
|
||||
.toPromise();
|
||||
|
||||
// Todo: pass in epochMillies-fields
|
||||
const featureCollection = hitsToGeoJson(
|
||||
// @ts-expect-error
|
||||
result.hits.hits,
|
||||
(hit: Record<string, unknown>) => {
|
||||
return flattenHit(geometryFieldName, hit);
|
||||
},
|
||||
geometryFieldName,
|
||||
geoFieldType,
|
||||
[]
|
||||
);
|
||||
// Todo: pass in epochMillies-fields
|
||||
const featureCollection = hitsToGeoJson(
|
||||
documentsResponse.rawResponse.hits.hits,
|
||||
(hit: Record<string, unknown>) => {
|
||||
return flattenHit(geometryFieldName, hit);
|
||||
},
|
||||
geometryFieldName,
|
||||
geoFieldType,
|
||||
[]
|
||||
);
|
||||
|
||||
resultFeatures = featureCollection.features;
|
||||
features = featureCollection.features;
|
||||
|
||||
// Correct system-fields.
|
||||
for (let i = 0; i < resultFeatures.length; i++) {
|
||||
const props = resultFeatures[i].properties;
|
||||
if (props !== null) {
|
||||
props[FEATURE_ID_PROPERTY_NAME] = resultFeatures[i].id;
|
||||
}
|
||||
// Correct system-fields.
|
||||
for (let i = 0; i < features.length; i++) {
|
||||
const props = features[i].properties;
|
||||
if (props !== null) {
|
||||
props[FEATURE_ID_PROPERTY_NAME] = features[i].id;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warn(e.message);
|
||||
throw e;
|
||||
}
|
||||
|
||||
const featureCollection: FeatureCollection = {
|
||||
features: resultFeatures,
|
||||
features,
|
||||
type: 'FeatureCollection',
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
RequestHandlerContext,
|
||||
} from 'src/core/server';
|
||||
import { IRouter } from 'src/core/server';
|
||||
import type { DataApiRequestHandlerContext } from 'src/plugins/data/server';
|
||||
import {
|
||||
MVT_GETTILE_API_PATH,
|
||||
API_ROOT_PATH,
|
||||
|
|
@ -24,7 +25,13 @@ import { getGridTile, getTile } from './get_tile';
|
|||
|
||||
const CACHE_TIMEOUT = 0; // Todo. determine good value. Unsure about full-implications (e.g. wrt. time-based data).
|
||||
|
||||
export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRouter }) {
|
||||
export function initMVTRoutes({
|
||||
router,
|
||||
logger,
|
||||
}: {
|
||||
router: IRouter<RequestHandlerContext & { search: DataApiRequestHandlerContext }>;
|
||||
logger: Logger;
|
||||
}) {
|
||||
router.get(
|
||||
{
|
||||
path: `${API_ROOT_PATH}/${MVT_GETTILE_API_PATH}`,
|
||||
|
|
@ -37,11 +44,12 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
requestBody: schema.string(),
|
||||
index: schema.string(),
|
||||
geoFieldType: schema.string(),
|
||||
searchSessionId: schema.maybe(schema.string()),
|
||||
}),
|
||||
},
|
||||
},
|
||||
async (
|
||||
context: RequestHandlerContext,
|
||||
context: RequestHandlerContext & { search: DataApiRequestHandlerContext },
|
||||
request: KibanaRequest<unknown, Record<string, any>, unknown>,
|
||||
response: KibanaResponseFactory
|
||||
) => {
|
||||
|
|
@ -50,7 +58,7 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
|
||||
const tile = await getTile({
|
||||
logger,
|
||||
callElasticsearch: makeCallElasticsearch(context),
|
||||
context,
|
||||
geometryFieldName: query.geometryFieldName as string,
|
||||
x: query.x as number,
|
||||
y: query.y as number,
|
||||
|
|
@ -58,6 +66,7 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
index: query.index as string,
|
||||
requestBody: requestBodyDSL as any,
|
||||
geoFieldType: query.geoFieldType as ES_GEO_FIELD_TYPE,
|
||||
searchSessionId: query.searchSessionId,
|
||||
});
|
||||
|
||||
return sendResponse(response, tile);
|
||||
|
|
@ -77,11 +86,12 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
index: schema.string(),
|
||||
requestType: schema.string(),
|
||||
geoFieldType: schema.string(),
|
||||
searchSessionId: schema.maybe(schema.string()),
|
||||
}),
|
||||
},
|
||||
},
|
||||
async (
|
||||
context: RequestHandlerContext,
|
||||
context: RequestHandlerContext & { search: DataApiRequestHandlerContext },
|
||||
request: KibanaRequest<unknown, Record<string, any>, unknown>,
|
||||
response: KibanaResponseFactory
|
||||
) => {
|
||||
|
|
@ -90,7 +100,7 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
|
||||
const tile = await getGridTile({
|
||||
logger,
|
||||
callElasticsearch: makeCallElasticsearch(context),
|
||||
context,
|
||||
geometryFieldName: query.geometryFieldName as string,
|
||||
x: query.x as number,
|
||||
y: query.y as number,
|
||||
|
|
@ -99,6 +109,7 @@ export function initMVTRoutes({ router, logger }: { logger: Logger; router: IRou
|
|||
requestBody: requestBodyDSL as any,
|
||||
requestType: query.requestType as RENDER_AS,
|
||||
geoFieldType: query.geoFieldType as ES_GEO_FIELD_TYPE,
|
||||
searchSessionId: query.searchSessionId,
|
||||
});
|
||||
|
||||
return sendResponse(response, tile);
|
||||
|
|
@ -125,9 +136,3 @@ function sendResponse(response: KibanaResponseFactory, tile: any) {
|
|||
});
|
||||
}
|
||||
}
|
||||
|
||||
function makeCallElasticsearch(context: RequestHandlerContext) {
|
||||
return async (type: string, ...args: any[]): Promise<unknown> => {
|
||||
return context.core.elasticsearch.legacy.client.callAsCurrentUser(type, ...args);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,26 +4,92 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { VectorTile } from '@mapbox/vector-tile';
|
||||
import Protobuf from 'pbf';
|
||||
import expect from '@kbn/expect';
|
||||
import {
|
||||
KBN_IS_CENTROID_FEATURE,
|
||||
MVT_SOURCE_LAYER_NAME,
|
||||
} from '../../../../plugins/maps/common/constants';
|
||||
|
||||
export default function ({ getService }) {
|
||||
const supertest = getService('supertest');
|
||||
|
||||
describe('getGridTile', () => {
|
||||
it('should validate params', async () => {
|
||||
await supertest
|
||||
it('should return vector tile containing cluster features', async () => {
|
||||
const resp = await supertest
|
||||
.get(
|
||||
`/api/maps/mvt/getGridTile?x=0&y=0&z=0&geometryFieldName=coordinates&index=logstash*&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:coordinates))),geotile_grid:(bounds:!n,field:coordinates,precision:!n,shard_size:65535,size:65535))),docvalue_fields:!((field:%27@timestamp%27,format:date_time),(field:timestamp,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(timestamp:(format:strict_date_optional_time,gte:%272020-09-16T13:57:36.734Z%27,lte:%272020-09-23T13:57:36.734Z%27)))),must:!(),must_not:!(),should:!())),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))&requestType=point&geoFieldType=geo_point`
|
||||
`/api/maps/mvt/getGridTile\
|
||||
?x=2\
|
||||
&y=3\
|
||||
&z=3\
|
||||
&geometryFieldName=geo.coordinates\
|
||||
&index=logstash-*\
|
||||
&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(avg_of_bytes:(avg:(field:bytes)),gridCentroid:(geo_centroid:(field:geo.coordinates))),geotile_grid:(bounds:!n,field:geo.coordinates,precision:!n,shard_size:65535,size:65535))),fields:!((field:%27@timestamp%27,format:date_time),(field:%27relatedContent.article:modified_time%27,format:date_time),(field:%27relatedContent.article:published_time%27,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27@timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))\
|
||||
&requestType=point\
|
||||
&geoFieldType=geo_point`
|
||||
)
|
||||
.set('kbn-xsrf', 'kibana')
|
||||
.responseType('blob')
|
||||
.expect(200);
|
||||
|
||||
const jsonTile = new VectorTile(new Protobuf(resp.body));
|
||||
const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME];
|
||||
expect(layer.length).to.be(1);
|
||||
const clusterFeature = layer.feature(0);
|
||||
expect(clusterFeature.type).to.be(1);
|
||||
expect(clusterFeature.extent).to.be(4096);
|
||||
expect(clusterFeature.id).to.be(undefined);
|
||||
expect(clusterFeature.properties).to.eql({ doc_count: 1, avg_of_bytes: 9252 });
|
||||
expect(clusterFeature.loadGeometry()).to.eql([[{ x: 87, y: 667 }]]);
|
||||
});
|
||||
|
||||
it('should not validate when required params are missing', async () => {
|
||||
await supertest
|
||||
it('should return vector tile containing grid features', async () => {
|
||||
const resp = await supertest
|
||||
.get(
|
||||
`/api/maps/mvt/getGridTile?x=0&y=0&z=0&geometryFieldName=coordinates&index=logstash*&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:coordinates))),geotile_grid:(bounds:!n,field:coordinates,precision:!n,shard_size:65535,size:65535))),docvalue_fields:!((field:%27@timestamp%27,format:date_time),(field:timestamp,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(timestamp:(format:strict_date_optional_time,gte:%272020-09-16T13:57:36.734Z%27,lte:%272020-09-23T13:57:36.734Z%27)))),must:!(),must_not:!(),should:!())),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))&requestType=point`
|
||||
`/api/maps/mvt/getGridTile\
|
||||
?x=2\
|
||||
&y=3\
|
||||
&z=3\
|
||||
&geometryFieldName=geo.coordinates\
|
||||
&index=logstash-*\
|
||||
&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(avg_of_bytes:(avg:(field:bytes)),gridCentroid:(geo_centroid:(field:geo.coordinates))),geotile_grid:(bounds:!n,field:geo.coordinates,precision:!n,shard_size:65535,size:65535))),fields:!((field:%27@timestamp%27,format:date_time),(field:%27relatedContent.article:modified_time%27,format:date_time),(field:%27relatedContent.article:published_time%27,format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:%27doc[!%27@timestamp!%27].value.getHour()%27))),size:0,stored_fields:!(%27*%27))\
|
||||
&requestType=grid\
|
||||
&geoFieldType=geo_point`
|
||||
)
|
||||
.set('kbn-xsrf', 'kibana')
|
||||
.expect(400);
|
||||
.responseType('blob')
|
||||
.expect(200);
|
||||
|
||||
const jsonTile = new VectorTile(new Protobuf(resp.body));
|
||||
const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME];
|
||||
expect(layer.length).to.be(2);
|
||||
|
||||
const gridFeature = layer.feature(0);
|
||||
expect(gridFeature.type).to.be(3);
|
||||
expect(gridFeature.extent).to.be(4096);
|
||||
expect(gridFeature.id).to.be(undefined);
|
||||
expect(gridFeature.properties).to.eql({ doc_count: 1, avg_of_bytes: 9252 });
|
||||
expect(gridFeature.loadGeometry()).to.eql([
|
||||
[
|
||||
{ x: 96, y: 640 },
|
||||
{ x: 96, y: 672 },
|
||||
{ x: 64, y: 672 },
|
||||
{ x: 64, y: 640 },
|
||||
{ x: 96, y: 640 },
|
||||
],
|
||||
]);
|
||||
|
||||
const clusterFeature = layer.feature(1);
|
||||
expect(clusterFeature.type).to.be(1);
|
||||
expect(clusterFeature.extent).to.be(4096);
|
||||
expect(clusterFeature.id).to.be(undefined);
|
||||
expect(clusterFeature.properties).to.eql({
|
||||
doc_count: 1,
|
||||
avg_of_bytes: 9252,
|
||||
[KBN_IS_CENTROID_FEATURE]: true,
|
||||
});
|
||||
expect(clusterFeature.loadGeometry()).to.eql([[{ x: 80, y: 656 }]]);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,26 +4,82 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { VectorTile } from '@mapbox/vector-tile';
|
||||
import Protobuf from 'pbf';
|
||||
import expect from '@kbn/expect';
|
||||
import { MVT_SOURCE_LAYER_NAME } from '../../../../plugins/maps/common/constants';
|
||||
|
||||
export default function ({ getService }) {
|
||||
const supertest = getService('supertest');
|
||||
|
||||
describe('getTile', () => {
|
||||
it('should validate params', async () => {
|
||||
await supertest
|
||||
it('should return vector tile containing document', async () => {
|
||||
const resp = await supertest
|
||||
.get(
|
||||
`/api/maps/mvt/getTile?x=15&y=11&z=5&geometryFieldName=coordinates&index=logstash*&requestBody=(_source:(includes:!(coordinates)),docvalue_fields:!(),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),script_fields:(),size:10000,stored_fields:!(coordinates))&geoFieldType=geo_point`
|
||||
`/api/maps/mvt/getTile\
|
||||
?x=1\
|
||||
&y=1\
|
||||
&z=2\
|
||||
&geometryFieldName=geo.coordinates\
|
||||
&index=logstash-*\
|
||||
&requestBody=(_source:!f,docvalue_fields:!(bytes,geo.coordinates,machine.os.raw),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(bytes,geo.coordinates,machine.os.raw))\
|
||||
&geoFieldType=geo_point`
|
||||
)
|
||||
.set('kbn-xsrf', 'kibana')
|
||||
.responseType('blob')
|
||||
.expect(200);
|
||||
|
||||
const jsonTile = new VectorTile(new Protobuf(resp.body));
|
||||
const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME];
|
||||
expect(layer.length).to.be(2);
|
||||
const feature = layer.feature(0);
|
||||
expect(feature.type).to.be(1);
|
||||
expect(feature.extent).to.be(4096);
|
||||
expect(feature.id).to.be(undefined);
|
||||
expect(feature.properties).to.eql({
|
||||
__kbn__feature_id__: 'logstash-2015.09.20:AU_x3_BsGFA8no6Qjjug:0',
|
||||
_id: 'AU_x3_BsGFA8no6Qjjug',
|
||||
_index: 'logstash-2015.09.20',
|
||||
bytes: 9252,
|
||||
['machine.os.raw']: 'ios',
|
||||
});
|
||||
expect(feature.loadGeometry()).to.eql([[{ x: 44, y: 2382 }]]);
|
||||
});
|
||||
|
||||
it('should not validate when required params are missing', async () => {
|
||||
await supertest
|
||||
it('should return vector tile containing bounds when count exceeds size', async () => {
|
||||
const resp = await supertest
|
||||
// requestBody sets size=1 to force count exceeded
|
||||
.get(
|
||||
`/api/maps/mvt/getTile?&index=logstash*&requestBody=(_source:(includes:!(coordinates)),docvalue_fields:!(),query:(bool:(filter:!((match_all:())),must:!(),must_not:!(),should:!())),script_fields:(),size:10000,stored_fields:!(coordinates))`
|
||||
`/api/maps/mvt/getTile\
|
||||
?x=1\
|
||||
&y=1\
|
||||
&z=2\
|
||||
&geometryFieldName=geo.coordinates\
|
||||
&index=logstash-*\
|
||||
&requestBody=(_source:!f,docvalue_fields:!(bytes,geo.coordinates,machine.os.raw),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:1,stored_fields:!(bytes,geo.coordinates,machine.os.raw))\
|
||||
&geoFieldType=geo_point`
|
||||
)
|
||||
.set('kbn-xsrf', 'kibana')
|
||||
.expect(400);
|
||||
.responseType('blob')
|
||||
.expect(200);
|
||||
|
||||
const jsonTile = new VectorTile(new Protobuf(resp.body));
|
||||
const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME];
|
||||
expect(layer.length).to.be(1);
|
||||
const feature = layer.feature(0);
|
||||
expect(feature.type).to.be(3);
|
||||
expect(feature.extent).to.be(4096);
|
||||
expect(feature.id).to.be(undefined);
|
||||
expect(feature.properties).to.eql({ __kbn_too_many_features__: true });
|
||||
expect(feature.loadGeometry()).to.eql([
|
||||
[
|
||||
{ x: 44, y: 2382 },
|
||||
{ x: 44, y: 1913 },
|
||||
{ x: 550, y: 1913 },
|
||||
{ x: 550, y: 2382 },
|
||||
{ x: 44, y: 2382 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue