[Maps] Enable gridding/clustering/heatmaps for geo_shape fields (#67886)

Enables heatmap, clusters, and grid layers for index-patterns with geo_shape field. This feature is only available for Gold+ users.
This commit is contained in:
Thomas Neirynck 2020-06-08 11:51:04 -04:00 committed by GitHub
parent e2f11e9fe9
commit 0189ae5c3f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 308 additions and 42 deletions

View file

@ -14,7 +14,7 @@ import { NoIndexPatternCallout } from '../../../components/no_index_pattern_call
import { i18n } from '@kbn/i18n';
import { EuiFormRow, EuiSpacer } from '@elastic/eui';
import { AGGREGATABLE_GEO_FIELD_TYPES, getFieldsWithGeoTileAgg } from '../../../index_pattern_util';
import { getAggregatableGeoFieldTypes, getFieldsWithGeoTileAgg } from '../../../index_pattern_util';
import { RenderAsSelect } from './render_as_select';
export class CreateSourceEditor extends Component {
@ -176,7 +176,7 @@ export class CreateSourceEditor extends Component {
placeholder={i18n.translate('xpack.maps.source.esGeoGrid.indexPatternPlaceholder', {
defaultMessage: 'Select index pattern',
})}
fieldTypes={AGGREGATABLE_GEO_FIELD_TYPES}
fieldTypes={getAggregatableGeoFieldTypes()}
onNoIndexPatterns={this._onNoIndexPatterns}
/>
</EuiFormRow>

View file

@ -21,6 +21,7 @@ import { getDataSourceLabel } from '../../../../common/i18n_getters';
import { AbstractESAggSource } from '../es_agg_source';
import { DataRequestAbortError } from '../../util/data_request';
import { registerSource } from '../source_registry';
import { makeESBbox } from '../../../elasticsearch_geo_utils';
export const MAX_GEOTILE_LEVEL = 29;
@ -146,6 +147,7 @@ export class ESGeoGridSource extends AbstractESAggSource {
registerCancelCallback,
bucketsPerGrid,
isRequestStillActive,
bufferedExtent,
}) {
const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid);
const aggs = {
@ -156,6 +158,7 @@ export class ESGeoGridSource extends AbstractESAggSource {
{
gridSplit: {
geotile_grid: {
bounds: makeESBbox(bufferedExtent),
field: this._descriptor.geoField,
precision,
},
@ -234,10 +237,12 @@ export class ESGeoGridSource extends AbstractESAggSource {
precision,
layerName,
registerCancelCallback,
bufferedExtent,
}) {
searchSource.setField('aggs', {
gridSplit: {
geotile_grid: {
bounds: makeESBbox(bufferedExtent),
field: this._descriptor.geoField,
precision,
},
@ -282,6 +287,7 @@ export class ESGeoGridSource extends AbstractESAggSource {
precision: searchFilters.geogridPrecision,
layerName,
registerCancelCallback,
bufferedExtent: searchFilters.buffer,
})
: await this._compositeAggRequest({
searchSource,
@ -291,6 +297,7 @@ export class ESGeoGridSource extends AbstractESAggSource {
registerCancelCallback,
bucketsPerGrid,
isRequestStillActive,
bufferedExtent: searchFilters.buffer,
});
return {

View file

@ -14,7 +14,8 @@ import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import { EuiFormRow, EuiCallOut } from '@elastic/eui';
import { AGGREGATABLE_GEO_FIELD_TYPES, getFieldsWithGeoTileAgg } from '../../../index_pattern_util';
import { getFieldsWithGeoTileAgg } from '../../../index_pattern_util';
import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
export class CreateSourceEditor extends Component {
static propTypes = {
@ -177,7 +178,7 @@ export class CreateSourceEditor extends Component {
placeholder={i18n.translate('xpack.maps.source.pewPew.indexPatternPlaceholder', {
defaultMessage: 'Select index pattern',
})}
fieldTypes={AGGREGATABLE_GEO_FIELD_TYPES}
fieldTypes={[ES_GEO_FIELD_TYPE.GEO_POINT]}
/>
</EuiFormRow>
);

View file

@ -225,39 +225,36 @@ export function geoShapeToGeometry(value, accumulator) {
accumulator.push(geoJson);
}
function createGeoBoundBoxFilter({ maxLat, maxLon, minLat, minLon }, geoFieldName) {
const top = clampToLatBounds(maxLat);
export function makeESBbox({ maxLat, maxLon, minLat, minLon }) {
const bottom = clampToLatBounds(minLat);
// geo_bounding_box does not support ranges outside of -180 and 180
// When the area crosses the 180° meridian,
// the value of the lower left longitude will be greater than the value of the upper right longitude.
// http://docs.opengeospatial.org/is/12-063r5/12-063r5.html#30
let boundingBox;
const top = clampToLatBounds(maxLat);
let esBbox;
if (maxLon - minLon >= 360) {
boundingBox = {
esBbox = {
top_left: [-180, top],
bottom_right: [180, bottom],
};
} else if (maxLon > 180) {
const overflow = maxLon - 180;
boundingBox = {
top_left: [minLon, top],
bottom_right: [-180 + overflow, bottom],
};
} else if (minLon < -180) {
const overflow = Math.abs(minLon) - 180;
boundingBox = {
top_left: [180 - overflow, top],
bottom_right: [maxLon, bottom],
};
} else {
boundingBox = {
top_left: [minLon, top],
bottom_right: [maxLon, bottom],
// geo_bounding_box does not support ranges outside of -180 and 180
// When the area crosses the 180° meridian,
// the value of the lower left longitude will be greater than the value of the upper right longitude.
// http://docs.opengeospatial.org/is/12-063r5/12-063r5.html#30
//
// This ensures bbox goes West->East in the happy case,
// but will be formatted East->West in case it crosses the date-line
const newMinlon = ((minLon + 180 + 360) % 360) - 180;
const newMaxlon = ((maxLon + 180 + 360) % 360) - 180;
esBbox = {
top_left: [newMinlon, top],
bottom_right: [newMaxlon, bottom],
};
}
return esBbox;
}
function createGeoBoundBoxFilter({ maxLat, maxLon, minLat, minLon }, geoFieldName) {
const boundingBox = makeESBbox({ maxLat, maxLon, minLat, minLon });
return {
geo_bounding_box: {
[geoFieldName]: boundingBox,

View file

@ -19,6 +19,7 @@ import {
createExtentFilter,
roundCoordinates,
extractFeaturesFromFilters,
makeESBbox,
} from './elasticsearch_geo_utils';
import { indexPatterns } from '../../../../src/plugins/data/public';
@ -594,3 +595,95 @@ describe('extractFeaturesFromFilters', () => {
expect(extractFeaturesFromFilters([spatialFilter])).toEqual([]);
});
});
describe('makeESBbox', () => {
it('Should invert Y-axis', () => {
const bbox = makeESBbox({
minLon: 10,
maxLon: 20,
minLat: 0,
maxLat: 1,
});
expect(bbox).toEqual({ bottom_right: [20, 0], top_left: [10, 1] });
});
it('Should snap to 360 width', () => {
const bbox = makeESBbox({
minLon: 10,
maxLon: 400,
minLat: 0,
maxLat: 1,
});
expect(bbox).toEqual({ bottom_right: [180, 0], top_left: [-180, 1] });
});
it('Should clamp latitudes', () => {
const bbox = makeESBbox({
minLon: 10,
maxLon: 400,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [180, -89], top_left: [-180, 89] });
});
it('Should swap West->East orientation to East->West orientation when crossing dateline (West extension)', () => {
const bbox = makeESBbox({
minLon: -190,
maxLon: 20,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [20, -89], top_left: [170, 89] });
});
it('Should swap West->East orientation to East->West orientation when crossing dateline (West extension) (overrated)', () => {
const bbox = makeESBbox({
minLon: -190 + 360 + 360,
maxLon: 20 + 360 + 360,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [20, -89], top_left: [170, 89] });
});
it('Should swap West->East orientation to East->West orientation when crossing dateline (east extension)', () => {
const bbox = makeESBbox({
minLon: 175,
maxLon: 190,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [-170, -89], top_left: [175, 89] });
});
it('Should preserve West->East orientation when _not_ crossing dateline', () => {
const bbox = makeESBbox({
minLon: 20,
maxLon: 170,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [170, -89], top_left: [20, 89] });
});
it('Should preserve West->East orientation when _not_ crossing dateline _and_ snap longitudes (west extension)', () => {
const bbox = makeESBbox({
minLon: -190,
maxLon: -185,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [175, -89], top_left: [170, 89] });
});
it('Should preserve West->East orientation when _not_ crossing dateline _and_ snap longitudes (east extension)', () => {
const bbox = makeESBbox({
minLon: 185,
maxLon: 190,
minLat: -100,
maxLat: 100,
});
expect(bbox).toEqual({ bottom_right: [-170, -89], top_left: [-175, 89] });
});
});

View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { getIndexPatternService } from './kibana_services';
import { getIndexPatternService, getIsGoldPlus } from './kibana_services';
import { indexPatterns } from '../../../../src/plugins/data/public';
import { ES_GEO_FIELD_TYPE } from '../common/constants';
@ -30,19 +30,24 @@ export function getTermsFields(fields) {
});
}
export const AGGREGATABLE_GEO_FIELD_TYPES = [ES_GEO_FIELD_TYPE.GEO_POINT];
export function getAggregatableGeoFieldTypes() {
const aggregatableFieldTypes = [ES_GEO_FIELD_TYPE.GEO_POINT];
if (getIsGoldPlus()) {
aggregatableFieldTypes.push(ES_GEO_FIELD_TYPE.GEO_SHAPE);
}
return aggregatableFieldTypes;
}
export function getFieldsWithGeoTileAgg(fields) {
return fields.filter(supportsGeoTileAgg);
}
export function supportsGeoTileAgg(field) {
// TODO add geo_shape support with license check
return (
field &&
field.aggregatable &&
!indexPatterns.isNestedField(field) &&
field.type === ES_GEO_FIELD_TYPE.GEO_POINT
getAggregatableGeoFieldTypes().includes(field.type)
);
}

View file

@ -6,7 +6,12 @@
jest.mock('./kibana_services', () => ({}));
import { getSourceFields } from './index_pattern_util';
import {
getSourceFields,
getAggregatableGeoFieldTypes,
supportsGeoTileAgg,
} from './index_pattern_util';
import { ES_GEO_FIELD_TYPE } from '../common/constants';
describe('getSourceFields', () => {
test('Should remove multi fields from field list', () => {
@ -27,3 +32,77 @@ describe('getSourceFields', () => {
expect(sourceFields).toEqual([{ name: 'agent' }]);
});
});
describe('Gold+ licensing', () => {
const testStubs = [
{
field: {
type: 'geo_point',
aggregatable: true,
},
supportedInBasic: true,
supportedInGold: true,
},
{
field: {
type: 'geo_shape',
aggregatable: false,
},
supportedInBasic: false,
supportedInGold: false,
},
{
field: {
type: 'geo_shape',
aggregatable: true,
},
supportedInBasic: false,
supportedInGold: true,
},
];
describe('basic license', () => {
beforeEach(() => {
require('./kibana_services').getIsGoldPlus = () => false;
});
describe('getAggregatableGeoFieldTypes', () => {
test('Should only include geo_point fields ', () => {
const aggregatableGeoFieldTypes = getAggregatableGeoFieldTypes();
expect(aggregatableGeoFieldTypes).toEqual([ES_GEO_FIELD_TYPE.GEO_POINT]);
});
});
describe('supportsGeoTileAgg', () => {
testStubs.forEach((stub, index) => {
test(`stub: ${index}`, () => {
const supported = supportsGeoTileAgg(stub.field);
expect(supported).toEqual(stub.supportedInBasic);
});
});
});
});
describe('gold license', () => {
beforeEach(() => {
require('./kibana_services').getIsGoldPlus = () => true;
});
describe('getAggregatableGeoFieldTypes', () => {
test('Should add geo_shape field', () => {
const aggregatableGeoFieldTypes = getAggregatableGeoFieldTypes();
expect(aggregatableGeoFieldTypes).toEqual([
ES_GEO_FIELD_TYPE.GEO_POINT,
ES_GEO_FIELD_TYPE.GEO_SHAPE,
]);
});
});
describe('supportsGeoTileAgg', () => {
testStubs.forEach((stub, index) => {
test(`stub: ${index}`, () => {
const supported = supportsGeoTileAgg(stub.field);
expect(supported).toEqual(stub.supportedInGold);
});
});
});
});
});

View file

@ -48,6 +48,7 @@ export function getShowMapsInspectorAdapter(): boolean;
export function getPreserveDrawingBuffer(): boolean;
export function getEnableVectorTiles(): boolean;
export function getProxyElasticMapsServiceInMaps(): boolean;
export function getIsGoldPlus(): boolean;
export function setLicenseId(args: unknown): void;
export function setInspector(args: unknown): void;
@ -74,3 +75,4 @@ export function setSearchService(args: DataPublicPluginStart['search']): void;
export function setKibanaCommonConfig(config: MapsLegacyConfigType): void;
export function setMapAppConfig(config: MapsConfigType): void;
export function setKibanaVersion(version: string): void;
export function setIsGoldPlus(isGoldPlus: boolean): void;

View file

@ -166,3 +166,12 @@ export const getProxyElasticMapsServiceInMaps = () =>
getKibanaCommonConfig().proxyElasticMapsServiceInMaps;
export const getRegionmapLayers = () => _.get(getKibanaCommonConfig(), 'regionmap.layers', []);
export const getTilemap = () => _.get(getKibanaCommonConfig(), 'tilemap', []);
let isGoldPlus = false;
export const setIsGoldPlus = (igp) => {
isGoldPlus = igp;
};
export const getIsGoldPlus = () => {
return isGoldPlus;
};

View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from 'src/core/public';
import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/public';
import { Setup as InspectorSetupContract } from 'src/plugins/inspector/public';
// @ts-ignore
import { MapView } from './inspector/views/map_view';
@ -21,29 +21,31 @@ import {
setIndexPatternSelect,
setIndexPatternService,
setInspector,
setIsGoldPlus,
setKibanaCommonConfig,
setKibanaVersion,
setLicenseId,
setMapAppConfig,
setMapsCapabilities,
setNavigation,
setSavedObjectsClient,
setSearchService,
setTimeFilter,
setToasts,
setUiActions,
setUiSettings,
setVisualizations,
setSearchService,
setMapAppConfig,
setKibanaCommonConfig,
setKibanaVersion,
} from './kibana_services';
import { featureCatalogueEntry } from './feature_catalogue_entry';
// @ts-ignore
import { getMapsVisTypeAlias } from './maps_vis_type_alias';
import { HomePublicPluginSetup } from '../../../../src/plugins/home/public';
import { VisualizationsSetup } from '../../../../src/plugins/visualizations/public';
import { MAP_SAVED_OBJECT_TYPE } from '../common/constants';
import { APP_ID, MAP_SAVED_OBJECT_TYPE } from '../common/constants';
import { MapEmbeddableFactory } from './embeddable/map_embeddable_factory';
import { EmbeddableSetup } from '../../../../src/plugins/embeddable/public';
import { MapsXPackConfig, MapsConfigType } from '../config';
import { MapsConfigType, MapsXPackConfig } from '../config';
import { ILicense } from '../../licensing/common/types';
export interface MapsPluginSetupDependencies {
inspector: InspectorSetupContract;
@ -76,7 +78,14 @@ export const bindSetupCoreAndPlugins = (
};
export const bindStartCoreAndPlugins = (core: CoreStart, plugins: any) => {
const { fileUpload, data, inspector } = plugins;
const { fileUpload, data, inspector, licensing } = plugins;
if (licensing) {
licensing.license$.subscribe((license: ILicense) => {
const gold = license.check(APP_ID, 'gold');
setIsGoldPlus(gold.state === 'valid');
});
}
setInspector(inspector);
setFileUpload(fileUpload);
setIndexPatternSelect(data.ui.IndexPatternSelect);

View file

@ -241,5 +241,38 @@ export default function ({ getPageObjects, getService }) {
});
});
});
describe('vector grid with geo_shape', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('geo grid vector grid example with shape');
});
const LAYER_ID = 'g1xkv';
it('should get expected number of grid cells', async () => {
const mapboxStyle = await PageObjects.maps.getMapboxStyle();
expect(mapboxStyle.sources[LAYER_ID].data.features.length).to.equal(13);
});
describe('inspector', () => {
afterEach(async () => {
await inspector.close();
});
it('should contain geotile_grid aggregation elasticsearch request', async () => {
await inspector.open();
await inspector.openInspectorRequestsView();
const requestStats = await inspector.getTableData();
const totalHits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits (total)');
expect(totalHits).to.equal('4'); //4 geometries result in 13 cells due to way they overlap geotile_grid cells
const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits');
expect(hits).to.equal('0'); // aggregation requests do not return any documents
const indexPatternName = PageObjects.maps.getInspectorStatRowHit(
requestStats,
'Index pattern'
);
expect(indexPatternName).to.equal('geo_shapes*');
});
});
});
});
}

View file

@ -36,7 +36,7 @@
"index": ".kibana",
"source": {
"index-pattern": {
"fields" : "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"geometry\",\"type\":\"geo_shape\",\"esTypes\":[\"geo_shape\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"prop1\",\"type\":\"number\",\"esTypes\":[\"byte\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
"fields" : "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"geometry\",\"type\":\"geo_shape\",\"esTypes\":[\"geo_shape\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"prop1\",\"type\":\"number\",\"esTypes\":[\"byte\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
"title": "geo_shapes*"
},
"type": "index-pattern"
@ -553,6 +553,37 @@
}
}
{
"type": "doc",
"value": {
"id": "map:0c86d024-a767-11ea-bb37-0242ac130002",
"index": ".kibana",
"source": {
"map": {
"bounds": {
"coordinates": [
[
-160,
60
],
[
160,
-60
]
],
"type": "envelope"
},
"description": "",
"layerListJSON": "[{\"id\":\"g1xkv\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"resolution\": \"COARSE\",\"type\":\"ES_GEO_GRID\",\"id\":\"64ddd934-a767-11ea-bb37-0242ac130002\",\"indexPatternId\":\"561253e0-f731-11e8-8487-11b9dd924f96\",\"geoField\":\"geometry\",\"requestType\":\"grid\",\"metrics\":[{\"type\":\"count\"}]},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"color\":\"Blues\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#cccccc\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"Count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"minSize\":4,\"maxSize\":32}}},\"temporary\":true,\"previousStyle\":null},\"type\":\"VECTOR\"}]",
"mapStateJSON": "{\"zoom\":3,\"center\":{\"lon\":76,\"lat\":4},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-20T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000}}",
"title": "geo grid vector grid example with shape",
"uiStateJSON": "{\"isDarkMode\":false}"
},
"type": "map"
}
}
}
{
"type": "doc",
"value": {