[maps] fetch geometry from fields API (#122431)

* [maps] fetch geometry from fields API

* tslint, eslint

* fix elasticsearch_geo_utils unit test

* more clean up of unit test

* i18n

* clean up

* eslint

* update functional test expects

* eslint

* remove unused turfCircle import

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Nathan Reese 2022-01-19 11:50:25 -07:00 committed by GitHub
parent d7dbf15919
commit bdb3ce465f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 146 additions and 208 deletions

View file

@ -411,7 +411,6 @@
"venn.js": "0.2.20",
"vinyl": "^2.2.0",
"vt-pbf": "^3.1.1",
"wellknown": "^0.5.0",
"whatwg-fetch": "^3.0.0",
"xml2js": "^0.4.22",
"yauzl": "^2.10.0"

View file

@ -42,14 +42,24 @@ describe('hitsToGeoJson', () => {
_id: 'doc1',
_index: 'index1',
fields: {
[geoFieldName]: '20,100',
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
],
},
},
{
_id: 'doc2',
_index: 'index1',
_source: {
[geoFieldName]: '30,110',
fields: {
[geoFieldName]: [
{
type: 'Point',
coordinates: [110, 30],
},
],
},
},
];
@ -73,12 +83,17 @@ describe('hitsToGeoJson', () => {
it('Should handle documents where geoField is not populated', () => {
const hits = [
{
_source: {
[geoFieldName]: '20,100',
fields: {
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
],
},
},
{
_source: {},
fields: {},
},
];
const geojson = hitsToGeoJson(hits, flattenHitMock, geoFieldName, 'geo_point', []);
@ -90,10 +105,15 @@ describe('hitsToGeoJson', () => {
const hits = [
{
_source: {
[geoFieldName]: '20,100',
myField: 8,
},
fields: {
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
],
myScriptedField: 10,
},
},
@ -109,8 +129,17 @@ describe('hitsToGeoJson', () => {
{
_id: 'doc1',
_index: 'index1',
_source: {
[geoFieldName]: ['20,100', '30,110'],
fields: {
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
{
type: 'Point',
coordinates: [110, 30],
},
],
myField: 8,
},
},
@ -151,15 +180,15 @@ describe('hitsToGeoJson', () => {
{
_id: 'doc1',
_index: 'index1',
_source: {
fields: {
[geoFieldName]: {
type: 'GeometryCollection',
geometries: [
{
type: 'geometrycollection', //explicitly test coercion to proper GeoJson type value
type: 'geometrycollection',
geometries: [
{
type: 'point', //explicitly test coercion to proper GeoJson type value
type: 'Point',
coordinates: [0, 0],
},
],
@ -216,8 +245,11 @@ describe('hitsToGeoJson', () => {
{
_id: 'doc1',
_index: 'index1',
_source: {
[geoFieldName]: '20,100',
fields: {
[geoFieldName]: {
type: 'Point',
coordinates: [100, 20],
},
myDateField: '1587156257081',
},
},
@ -234,16 +266,21 @@ describe('hitsToGeoJson', () => {
const geoFieldName = 'my.location';
const indexPatternFlattenHit = (hit) => {
return {
[geoFieldName]: _.get(hit._source, geoFieldName),
[geoFieldName]: _.get(hit.fields, geoFieldName),
};
};
it('Should handle geoField being an object', () => {
const hits = [
{
_source: {
fields: {
my: {
location: '20,100',
location: [
{
type: 'Point',
coordinates: [100, 20],
},
],
},
},
},
@ -258,8 +295,13 @@ describe('hitsToGeoJson', () => {
it('Should handle geoField containing dot in the name', () => {
const hits = [
{
_source: {
['my.location']: '20,100',
fields: {
['my.location']: [
{
type: 'Point',
coordinates: [100, 20],
},
],
},
},
];
@ -273,15 +315,25 @@ describe('hitsToGeoJson', () => {
it('Should not modify results of flattenHit', () => {
const geoFieldName = 'location';
const cachedProperities = {
[geoFieldName]: '20,100',
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
],
};
const cachedFlattenHit = () => {
return cachedProperities;
};
const hits = [
{
_source: {
[geoFieldName]: '20,100',
fields: {
[geoFieldName]: [
{
type: 'Point',
coordinates: [100, 20],
},
],
},
},
];
@ -296,8 +348,11 @@ describe('geoPointToGeometry', () => {
const lat = 41.12;
const lon = -71.34;
it('Should convert single docvalue_field', () => {
const value = `${lat},${lon}`;
it('Should convert value', () => {
const value = {
type: 'Point',
coordinates: [lon, lat],
};
const points = [];
geoPointToGeometry(value, points);
expect(points.length).toBe(1);
@ -305,10 +360,19 @@ describe('geoPointToGeometry', () => {
expect(points[0].coordinates).toEqual([lon, lat]);
});
it('Should convert multiple docvalue_fields', () => {
it('Should convert array of values', () => {
const lat2 = 30;
const lon2 = -60;
const value = [`${lat},${lon}`, `${lat2},${lon2}`];
const value = [
{
type: 'Point',
coordinates: [lon, lat],
},
{
type: 'Point',
coordinates: [lon2, lat2],
},
];
const points = [];
geoPointToGeometry(value, points);
expect(points.length).toBe(2);
@ -318,13 +382,13 @@ describe('geoPointToGeometry', () => {
});
describe('geoShapeToGeometry', () => {
it('Should convert value stored as geojson', () => {
it('Should convert value', () => {
const coordinates = [
[-77.03653, 38.897676],
[-77.009051, 38.889939],
];
const value = {
type: 'linestring',
type: 'LineString',
coordinates: coordinates,
};
const shapes = [];
@ -340,7 +404,7 @@ describe('geoShapeToGeometry', () => {
[101.0, 0.0],
];
const value = {
type: 'envelope',
type: 'Envelope',
coordinates: coordinates,
};
const shapes = [];
@ -366,11 +430,11 @@ describe('geoShapeToGeometry', () => {
const pointCoordinates = [125.6, 10.1];
const value = [
{
type: 'linestring',
type: 'LineString',
coordinates: linestringCoordinates,
},
{
type: 'point',
type: 'Point',
coordinates: pointCoordinates,
},
];
@ -382,28 +446,6 @@ describe('geoShapeToGeometry', () => {
expect(shapes[1].type).toBe('Point');
expect(shapes[1].coordinates).toEqual(pointCoordinates);
});
it('Should convert wkt shapes to geojson', () => {
const pointWkt = 'POINT (32 40)';
const linestringWkt = 'LINESTRING (50 60, 70 80)';
const shapes = [];
geoShapeToGeometry(pointWkt, shapes);
geoShapeToGeometry(linestringWkt, shapes);
expect(shapes.length).toBe(2);
expect(shapes[0]).toEqual({
coordinates: [32, 40],
type: 'Point',
});
expect(shapes[1]).toEqual({
coordinates: [
[50, 60],
[70, 80],
],
type: 'LineString',
});
});
});
describe('roundCoordinates', () => {

View file

@ -7,10 +7,6 @@
import _ from 'lodash';
import { i18n } from '@kbn/i18n';
// @ts-expect-error
import { parse } from 'wellknown';
// @ts-expect-error
import turfCircle from '@turf/circle';
import { Feature, FeatureCollection, Geometry, Polygon, Point, Position } from 'geojson';
import { BBox } from '@turf/helpers';
import {
@ -89,12 +85,12 @@ export function hitsToGeoJson(
ensureGeoField(geoFieldType);
if (geoFieldType === ES_GEO_FIELD_TYPE.GEO_POINT) {
geoPointToGeometry(
properties[geoFieldName] as string | string[] | undefined,
properties[geoFieldName] as Point | Point[] | undefined,
tmpGeometriesAccumulator
);
} else {
geoShapeToGeometry(
properties[geoFieldName] as string | string[] | ESGeometry | ESGeometry[] | undefined,
properties[geoFieldName] as ESGeometry | ESGeometry[] | undefined,
tmpGeometriesAccumulator
);
}
@ -131,12 +127,9 @@ export function hitsToGeoJson(
};
}
// Parse geo_point docvalue_field
// Either
// 1) Array of latLon strings
// 2) latLon string
// Parse geo_point fields API response
export function geoPointToGeometry(
value: string[] | string | undefined,
value: Point[] | Point | undefined,
accumulator: Geometry[]
): void {
if (!value) {
@ -150,99 +143,12 @@ export function geoPointToGeometry(
return;
}
const commaSplit = value.split(',');
const lat = parseFloat(commaSplit[0]);
const lon = parseFloat(commaSplit[1]);
accumulator.push({
type: GEO_JSON_TYPE.POINT,
coordinates: [lon, lat],
} as Point);
}
export function convertESShapeToGeojsonGeometry(value: ESGeometry): Geometry {
const geoJson = {
type: value.type,
coordinates: value.coordinates,
};
// https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-shape.html#input-structure
// For some unknown compatibility nightmarish reason, Elasticsearch types are not capitalized the same as geojson types
// For example: 'LineString' geojson type is 'linestring' in elasticsearch
// Convert feature types to geojson spec values
// Sometimes, the type in ES is capitalized correctly. Sometimes it is not. It depends on how the doc was ingested
// The below is the correction in-place.
switch (value.type) {
case 'point':
geoJson.type = GEO_JSON_TYPE.POINT;
break;
case 'linestring':
geoJson.type = GEO_JSON_TYPE.LINE_STRING;
break;
case 'polygon':
geoJson.type = GEO_JSON_TYPE.POLYGON;
break;
case 'multipoint':
geoJson.type = GEO_JSON_TYPE.MULTI_POINT;
break;
case 'multilinestring':
geoJson.type = GEO_JSON_TYPE.MULTI_LINE_STRING;
break;
case 'multipolygon':
geoJson.type = GEO_JSON_TYPE.MULTI_POLYGON;
break;
case 'geometrycollection':
case GEO_JSON_TYPE.GEOMETRY_COLLECTION:
// PEBKAC - geometry-collections need to be unrolled to their individual geometries first.
const invalidGeometrycollectionError = i18n.translate(
'xpack.maps.es_geo_utils.convert.invalidGeometryCollectionErrorMessage',
{
defaultMessage: `Should not pass GeometryCollection to convertESShapeToGeojsonGeometry`,
}
);
throw new Error(invalidGeometrycollectionError);
case 'envelope':
const envelopeCoords = geoJson.coordinates as Position[];
// format defined here https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-shape.html#_envelope
const polygon = formatEnvelopeAsPolygon({
minLon: envelopeCoords[0][0],
maxLon: envelopeCoords[1][0],
minLat: envelopeCoords[1][1],
maxLat: envelopeCoords[0][1],
});
geoJson.type = polygon.type;
geoJson.coordinates = polygon.coordinates;
break;
case 'circle':
const errorMessage = i18n.translate(
'xpack.maps.es_geo_utils.convert.unsupportedGeometryTypeErrorMessage',
{
defaultMessage: `Unable to convert {geometryType} geometry to geojson, not supported`,
values: {
geometryType: geoJson.type,
},
}
);
throw new Error(errorMessage);
}
return geoJson as unknown as Geometry;
}
function convertWKTStringToGeojson(value: string): Geometry {
try {
return parse(value);
} catch (e) {
const errorMessage = i18n.translate('xpack.maps.es_geo_utils.wkt.invalidWKTErrorMessage', {
defaultMessage: `Unable to convert {wkt} to geojson. Valid WKT expected.`,
values: {
wkt: value,
},
});
throw new Error(errorMessage);
}
accumulator.push(value as Point);
}
// Parse geo_shape fields API response
export function geoShapeToGeometry(
value: string | ESGeometry | string[] | ESGeometry[] | undefined,
value: ESGeometry | ESGeometry[] | undefined,
accumulator: Geometry[]
): void {
if (!value) {
@ -257,21 +163,38 @@ export function geoShapeToGeometry(
return;
}
if (typeof value === 'string') {
const geoJson = convertWKTStringToGeojson(value);
accumulator.push(geoJson);
} else if (
// Needs to deal with possible inconsistencies in capitalization
value.type === GEO_JSON_TYPE.GEOMETRY_COLLECTION ||
value.type === 'geometrycollection'
) {
if (value.type.toLowerCase() === GEO_JSON_TYPE.GEOMETRY_COLLECTION.toLowerCase()) {
const geometryCollection = value as unknown as { geometries: ESGeometry[] };
for (let i = 0; i < geometryCollection.geometries.length; i++) {
geoShapeToGeometry(geometryCollection.geometries[i], accumulator);
}
return;
}
// fields API does not return true geojson yet, circle and envelope still exist which are not part of geojson spec
if (value.type.toLowerCase() === 'envelope') {
const envelopeCoords = value.coordinates as Position[];
// format defined here https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-shape.html#_envelope
const polygon = formatEnvelopeAsPolygon({
minLon: envelopeCoords[0][0],
maxLon: envelopeCoords[1][0],
minLat: envelopeCoords[1][1],
maxLat: envelopeCoords[0][1],
});
accumulator.push(polygon);
} else if (value.type.toLowerCase() === 'circle') {
const errorMessage = i18n.translate(
'xpack.maps.es_geo_utils.convert.unsupportedGeometryTypeErrorMessage',
{
defaultMessage: `Unable to convert {geometryType} geometry to geojson, not supported`,
values: {
geometryType: value.type,
},
}
);
throw new Error(errorMessage);
} else {
const geoJson = convertESShapeToGeojsonGeometry(value);
accumulator.push(geoJson);
accumulator.push(value as Geometry);
}
}

View file

@ -281,21 +281,26 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
const indexPattern: IndexPattern = await this.getIndexPattern();
const fieldNames = searchFilters.fieldNames.filter(
(fieldName) => fieldName !== this._descriptor.geoField
);
const { docValueFields, sourceOnlyFields, scriptFields } = getDocValueAndSourceFields(
indexPattern,
searchFilters.fieldNames,
fieldNames,
'epoch_millis'
);
const topHits: {
size: number;
script_fields: Record<string, { script: ScriptField }>;
docvalue_fields: Array<string | { format: string; field: string }>;
fields: string[];
_source?: boolean | { includes: string[] };
sort?: Array<Record<string, SortDirectionNumeric>>;
} = {
size: topHitsSize,
script_fields: scriptFields,
docvalue_fields: docValueFields,
fields: [this._descriptor.geoField],
};
if (this._hasSort()) {
@ -389,9 +394,12 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
) {
const indexPattern = await this.getIndexPattern();
const fieldNames = searchFilters.fieldNames.filter(
(fieldName) => fieldName !== this._descriptor.geoField
);
const { docValueFields, sourceOnlyFields } = getDocValueAndSourceFields(
indexPattern,
searchFilters.fieldNames,
fieldNames,
'epoch_millis'
);
@ -418,6 +426,7 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
} else {
searchSource.setField('source', sourceOnlyFields);
}
searchSource.setField('fields', [this._descriptor.geoField]);
if (this._hasSort()) {
searchSource.setField('sort', this._buildEsSort());
}
@ -800,9 +809,12 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
const indexPattern = await this.getIndexPattern();
const indexSettings = await loadIndexSettings(indexPattern.title);
const fieldNames = searchFilters.fieldNames.filter(
(fieldName) => fieldName !== this._descriptor.geoField
);
const { docValueFields, sourceOnlyFields } = getDocValueAndSourceFields(
indexPattern,
searchFilters.fieldNames,
fieldNames,
'epoch_millis'
);

View file

@ -14910,12 +14910,10 @@
"xpack.maps.embeddableDisplayName": "マップ",
"xpack.maps.emsFileSelect.selectPlaceholder": "EMSレイヤーを選択",
"xpack.maps.emsSource.tooltipsTitle": "ツールチップフィールド",
"xpack.maps.es_geo_utils.convert.invalidGeometryCollectionErrorMessage": "GeometryCollectionを convertESShapeToGeojsonGeometryに渡さないでください",
"xpack.maps.es_geo_utils.convert.unsupportedGeometryTypeErrorMessage": "{geometryType} ジオメトリから Geojson に変換できません。サポートされていません",
"xpack.maps.es_geo_utils.distanceFilterAlias": "{pointLabel}の{distanceKm} km以内",
"xpack.maps.es_geo_utils.unsupportedFieldTypeErrorMessage": "サポートされていないフィールドタイプ、期待値:{expectedTypes}、提供された値:{fieldType}",
"xpack.maps.es_geo_utils.unsupportedGeometryTypeErrorMessage": "サポートされていないジオメトリタイプ、期待値:{expectedTypes}、提供された値:{geometryType}",
"xpack.maps.es_geo_utils.wkt.invalidWKTErrorMessage": "{wkt} を Geojson に変換できません。有効な WKT が必要です。",
"xpack.maps.esGeoLine.areEntitiesTrimmedMsg": "結果は ~{totalEntities} 中最初の {entityCount} トラックに制限されます。",
"xpack.maps.esGeoLine.tracksCountMsg": "{entityCount} 個のトラックが見つかりました。",
"xpack.maps.esGeoLine.tracksTrimmedMsg": "{entityCount} 中 {numTrimmedTracks} 個のトラックが不完全です。",

View file

@ -15105,12 +15105,10 @@
"xpack.maps.embeddableDisplayName": "地图",
"xpack.maps.emsFileSelect.selectPlaceholder": "选择 EMS 图层",
"xpack.maps.emsSource.tooltipsTitle": "工具提示字段",
"xpack.maps.es_geo_utils.convert.invalidGeometryCollectionErrorMessage": "不应将 GeometryCollection 传递给 convertESShapeToGeojsonGeometry",
"xpack.maps.es_geo_utils.convert.unsupportedGeometryTypeErrorMessage": "无法将 {geometryType} 几何图形转换成 geojson不支持",
"xpack.maps.es_geo_utils.distanceFilterAlias": "{pointLabel} {distanceKm}km 内",
"xpack.maps.es_geo_utils.unsupportedFieldTypeErrorMessage": "字段类型不受支持,应为 {expectedTypes},而提供的是 {fieldType}",
"xpack.maps.es_geo_utils.unsupportedGeometryTypeErrorMessage": "几何类型不受支持,应为 {expectedTypes},而提供的是 {geometryType}",
"xpack.maps.es_geo_utils.wkt.invalidWKTErrorMessage": "无法将 {wkt} 转换成 geojson。需要有效的 WKT。",
"xpack.maps.esGeoLine.areEntitiesTrimmedMsg": "结果限制为 ~{totalEntities} 条轨迹中的前 {entityCount} 条。",
"xpack.maps.esGeoLine.tracksCountMsg": "找到 {entityCount} 条轨迹。",
"xpack.maps.esGeoLine.tracksTrimmedMsg": "{entityCount} 条轨迹中有 {numTrimmedTracks} 条不完整。",

View file

@ -52,7 +52,7 @@ export default function ({ getPageObjects, getService }) {
geometryFieldName: 'geometry',
index: 'geo_shapes*',
requestBody:
'(_source:!(geometry),docvalue_fields:!(prop1),query:(bool:(filter:!(),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10001,stored_fields:!(geometry,prop1))',
'(_source:!f,docvalue_fields:!(prop1),query:(bool:(filter:!(),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10001,stored_fields:!(geometry,prop1))',
});
});

View file

@ -10311,15 +10311,6 @@ concat-stream@1.6.2, concat-stream@^1.4.7, concat-stream@^1.5.0, concat-stream@^
readable-stream "^2.2.2"
typedarray "^0.0.6"
concat-stream@~1.5.0:
version "1.5.2"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.5.2.tgz#708978624d856af41a5a741defdd261da752c266"
integrity sha1-cIl4Yk2FavQaWnQd790mHadSwmY=
dependencies:
inherits "~2.0.1"
readable-stream "~2.0.0"
typedarray "~0.0.5"
concat-stream@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-2.0.0.tgz#414cf5af790a48c60ab9be4527d56d5e41133cb1"
@ -22651,11 +22642,6 @@ process-nextick-args@^2.0.0, process-nextick-args@~2.0.0:
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
process-nextick-args@~1.0.6:
version "1.0.7"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
integrity sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=
process-on-spawn@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/process-on-spawn/-/process-on-spawn-1.0.0.tgz#95b05a23073d30a17acfdc92a440efd2baefdc93"
@ -24032,18 +24018,6 @@ readable-stream@~1.1.9:
isarray "0.0.1"
string_decoder "~0.10.x"
readable-stream@~2.0.0:
version "2.0.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
integrity sha1-j5A0HmilPMySh4jaz80Rs265t44=
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "~1.0.0"
process-nextick-args "~1.0.6"
string_decoder "~0.10.x"
util-deprecate "~1.0.1"
readdir-glob@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
@ -27877,7 +27851,7 @@ typedarray-to-buffer@^3.1.5:
dependencies:
is-typedarray "^1.0.0"
typedarray@^0.0.6, typedarray@~0.0.5:
typedarray@^0.0.6:
version "0.0.6"
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
@ -29518,14 +29492,6 @@ websocket-extensions@>=0.1.1:
resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42"
integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==
wellknown@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/wellknown/-/wellknown-0.5.0.tgz#09ae9871fa826cf0a6ec1537ef00c379d78d7101"
integrity sha1-Ca6YcfqCbPCm7BU37wDDedeNcQE=
dependencies:
concat-stream "~1.5.0"
minimist "~1.2.0"
wgs84@0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/wgs84/-/wgs84-0.0.0.tgz#34fdc555917b6e57cf2a282ed043710c049cdc76"