From ee3510b766d4cc25c20beba92b7da7ce68ddaca9 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 20 Apr 2021 10:24:25 +0100 Subject: [PATCH] Add index-time scripts to geo_point field mapper (#71861) This commit adds the ability to define an index-time geo_point field with a script parameter, allowing you to calculate points from other values within the indexed document. --- .../mapping/types/geo-point.asciidoc | 29 ++- .../103_geo_point_calculated_at_index.yml | 206 ++++++++++++++++++ .../mapper/AbstractGeometryFieldMapper.java | 45 ++-- .../AbstractPointGeometryFieldMapper.java | 9 +- .../index/mapper/GeoPointFieldMapper.java | 101 +++++++-- .../script/GeoPointFieldScript.java | 14 ++ .../fielddata/AbstractFieldDataTestCase.java | 5 +- .../mapper/GeoPointFieldMapperTests.java | 32 ++- .../index/mapper/GeoPointFieldTypeTests.java | 5 +- .../mapper/GeoPointScriptMapperTests.java | 90 ++++++++ .../mapper/CartesianFieldMapperTests.java | 2 +- 11 files changed, 503 insertions(+), 35 deletions(-) create mode 100644 modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptMapperTests.java diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 3d5f76a28528..03d9c731d93b 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -123,20 +123,43 @@ The following parameters are accepted by `geo_point` fields: If `true`, malformed geo-points are ignored. If `false` (default), malformed geo-points throw an exception and reject the whole document. - A geo-point is considered malformed if its latitude is outside the range + A geo-point is considered malformed if its latitude is outside the range -90 <= latitude <= 90, or if its longitude is outside the range -180 <= longitude <= 180. + Note that this cannot be set if the `script` parameter is used. `ignore_z_value`:: If `true` (default) three dimension points will be accepted (stored in source) but only latitude and longitude values will be indexed; the third dimension is ignored. If `false`, geo-points containing any more than latitude and longitude - (two dimensions) values throw an exception and reject the whole document. + (two dimensions) values throw an exception and reject the whole document. Note + that this cannot be set if the `script` parameter is used. <>:: Accepts an geopoint value which is substituted for any explicit `null` values. - Defaults to `null`, which means the field is treated as missing. + Defaults to `null`, which means the field is treated as missing. Note that this + cannot be set if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>, and should emit points + as a pair of (lat, lon) double values. ==== Using geo-points in scripts diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml new file mode 100644 index 000000000000..d47c0200b9e2 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml @@ -0,0 +1,206 @@ +--- +setup: + - do: + indices.create: + index: locations + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + location_from_doc_value: + type: geo_point + script: + source: | + emit(doc["location"].lat, doc["location"].lon); + location_from_source: + type: geo_point + script: + source: | + emit(params._source.location.lat, params._source.location.lon); + timestamp: + type: date + location: + type: geo_point + - do: + bulk: + index: locations + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "location" : {"lat": 13.5, "lon" : 34.89}} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "location" : {"lat": -7.9, "lon" : 120.78}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "location" : {"lat": 45.78, "lon" : -173.45}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "location" : {"lat": 32.45, "lon" : 45.6}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "location" : {"lat": -63.24, "lon" : 31.0}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "location" : {"lat": 0.0, "lon" : 0.0}} + + +--- +"get mapping": + - do: + indices.get_mapping: + index: locations + - match: {locations.mappings.properties.location_from_source.type: geo_point } + - match: + locations.mappings.properties.location_from_source.script.source: | + emit(params._source.location.lat, params._source.location.lon); + - match: {locations.mappings.properties.location_from_source.script.lang: painless } + +--- +"fetch fields from source": + - do: + search: + index: locations + body: + sort: timestamp + fields: [location, location_from_doc_value, location_from_source] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.location.0.type: "Point" } + - match: {hits.hits.0.fields.location.0.coordinates: [34.89, 13.5] } + # calculated from scripts adds annoying extra precision + - match: { hits.hits.0.fields.location_from_doc_value.0.type: "Point" } + - match: { hits.hits.0.fields.location_from_doc_value.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] } + - match: { hits.hits.0.fields.location_from_source.0.type: "Point" } + - match: { hits.hits.0.fields.location_from_source.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] } + +--- +"exists query": + - do: + search: + index: locations + body: + query: + exists: + field: location_from_source + - match: {hits.total.value: 6} + +--- +"geo bounding box query": + - do: + search: + index: locations + body: + query: + geo_bounding_box: + location_from_source: + top_left: + lat: 10 + lon: -10 + bottom_right: + lat: -10 + lon: 10 + - match: {hits.total.value: 1} + +--- +"geo shape query": + - do: + search: + index: locations + body: + query: + geo_shape: + location_from_source: + shape: + type: "envelope" + coordinates: [ [ -10, 10 ], [ 10, -10 ] ] + - match: {hits.total.value: 1} + +--- +"geo distance query": + - do: + search: + index: locations + body: + query: + geo_distance: + distance: "2000km" + location_from_source: + lat: 0 + lon: 0 + - match: {hits.total.value: 1} + +--- +"bounds agg": + - do: + search: + index: locations + body: + aggs: + bounds: + geo_bounds: + field: "location" + wrap_longitude: false + bounds_from_doc_value: + geo_bounds: + field: "location_from_doc_value" + wrap_longitude: false + bounds_from_source: + geo_bounds: + field: "location_from_source" + wrap_longitude: false + - match: {hits.total.value: 6} + - match: {aggregations.bounds.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_source.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_source.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lon: 120.77999993227422 } + +--- +"geo_distance sort": + - do: + search: + index: locations + body: + sort: + _geo_distance: + location_from_source: + lat: 0.0 + lon: 0.0 + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.location.lat: 0.0 } + - match: {hits.hits.0._source.location.lon: 0.0 } + - match: {hits.hits.1._source.location.lat: 13.5 } + - match: {hits.hits.1._source.location.lon: 34.89 } + - match: {hits.hits.2._source.location.lat: 32.45 } + - match: {hits.hits.2._source.location.lon: 45.6 } + - match: {hits.hits.3._source.location.lat: -63.24 } + - match: {hits.hits.3._source.location.lon: 31.0 } + +--- +"distance_feature query": + - do: + search: + index: locations + body: + query: + bool: + should: + distance_feature: + field: "location" + pivot: "1000km" + origin: [0.0, 0.0] + + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.location.lat: 0.0 } + - match: {hits.hits.0._source.location.lon: 0.0 } + - match: {hits.hits.1._source.location.lat: 13.5 } + - match: {hits.hits.1._source.location.lon: 34.89 } + - match: {hits.hits.2._source.location.lat: 32.45 } + - match: {hits.hits.2._source.location.lon: 45.6 } + - match: {hits.hits.3._source.location.lat: -63.24 } + - match: {hits.hits.3._source.location.lon: 31.0 } + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index d30db2e06f97..160e29f002ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -89,7 +89,7 @@ public abstract class AbstractGeometryFieldMapper extends FieldMapper { } @Override - public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; if (parsesArrayValue) { @@ -116,13 +116,13 @@ public abstract class AbstractGeometryFieldMapper extends FieldMapper { private final Explicit ignoreMalformed; private final Explicit ignoreZValue; - private final Parser parser; + private final Parser parser; protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, Map indexAnalyzers, Explicit ignoreMalformed, Explicit ignoreZValue, MultiFields multiFields, CopyTo copyTo, - Parser parser) { + Parser parser) { super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null); this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; @@ -132,10 +132,24 @@ public abstract class AbstractGeometryFieldMapper extends FieldMapper { protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, Explicit ignoreMalformed, Explicit ignoreZValue, MultiFields multiFields, CopyTo copyTo, - Parser parser) { + Parser parser) { this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, parser); } + protected AbstractGeometryFieldMapper( + String simpleName, + MappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Parser parser, + String onScriptError + ) { + super(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo, true, onScriptError); + this.ignoreMalformed = new Explicit<>(false, true); + this.ignoreZValue = new Explicit<>(false, true); + this.parser = parser; + } + @Override public AbstractGeometryFieldType fieldType() { return (AbstractGeometryFieldType) mappedFieldType; @@ -155,16 +169,19 @@ public abstract class AbstractGeometryFieldMapper extends FieldMapper { @Override public final void parse(ParseContext context) throws IOException { - parser.parse(context.parser(), v -> index(context, v), e -> { - if (ignoreMalformed()) { - context.addIgnoredField(fieldType().name()); - } else { - throw new MapperParsingException( - "Failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]", - e - ); - } - }); + if (hasScript) { + throw new MapperParsingException("failed to parse field [" + fieldType().name() + "] of type + " + contentType() + "]", + new IllegalArgumentException("Cannot index data directly into a field with a [script] parameter")); + } + parser.parse(context.parser(), v -> index(context, v), e -> { + if (ignoreMalformed()) { + context.addIgnoredField(fieldType().name()); + } else { + throw new MapperParsingException( + "failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]", e + ); + } + }); } public boolean ignoreMalformed() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java index 8fc2f8752bc8..69332e7cd937 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -39,11 +39,18 @@ public abstract class AbstractPointGeometryFieldMapper extends AbstractGeomet protected AbstractPointGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, Explicit ignoreMalformed, Explicit ignoreZValue, ParsedPoint nullValue, CopyTo copyTo, - Parser parser) { + Parser parser) { super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, parser); this.nullValue = nullValue; } + protected AbstractPointGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, + Parser parser, String onScriptError) { + super(simpleName, mappedFieldType, multiFields, copyTo, parser, onScriptError); + this.nullValue = null; + } + @Override public final boolean parsesArrayValue() { return true; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 0f17fbb77d52..e5d8484e8eac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -11,30 +11,39 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.StoredField; import org.apache.lucene.geo.LatLonGeometry; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.geo.GeoJsonGeometryFormat; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoShapeUtils; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.GeometryFormat; +import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointIndexFieldData; -import org.elasticsearch.index.mapper.GeoPointFieldMapper.ParsedGeoPoint; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.GeoPointFieldScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; /** @@ -42,7 +51,7 @@ import java.util.function.Supplier; * * Uses lucene 6 LatLonPoint encoding */ -public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper { +public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper { public static final String CONTENT_TYPE = "geo_point"; @@ -58,20 +67,27 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); final Parameter hasDocValues = Parameter.docValuesParam(m -> builder(m).hasDocValues.get(), true); final Parameter stored = Parameter.storeParam(m -> builder(m).stored.get(), false); + private final Parameter