From 1469e18c9876f5d092d1d3887bd26ef768331ee4 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 12 Apr 2021 10:04:12 +0200 Subject: [PATCH] Add support for script parameter to boolean field mapper (#71454) Relates to #68984 --- docs/reference/mapping/types/boolean.asciidoc | 22 ++- .../mapper/SearchAsYouTypeFieldMapper.java | 2 +- .../63_boolean_calculated_at_index.yml | 134 ++++++++++++++++++ .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/BooleanFieldMapper.java | 70 +++++++-- .../index/mapper/DynamicFieldsBuilder.java | 7 +- .../index/mapper/FieldMapper.java | 113 ++++++++++++--- .../index/mapper/NumberFieldMapper.java | 74 ++-------- .../index/mapper/TextFieldMapper.java | 2 +- .../script/BooleanFieldScript.java | 11 +- .../elasticsearch/script/ScriptCompiler.java | 6 + .../search/lookup/FieldValues.java | 33 +++++ .../elasticsearch/index/codec/CodecTests.java | 3 +- .../fielddata/AbstractFieldDataTestCase.java | 13 +- .../fielddata/IndexFieldDataServiceTests.java | 15 +- .../AbstractScriptFieldTypeTestCase.java | 1 + .../index/mapper/BooleanFieldMapperTests.java | 12 ++ .../index/mapper/BooleanFieldTypeTests.java | 2 +- .../mapper/BooleanScriptFieldTypeTests.java | 5 +- .../mapper/BooleanScriptMapperTests.java | 90 ++++++++++++ .../index/mapper/DoubleScriptMapperTests.java | 53 +++---- .../index/mapper/ExternalMapper.java | 3 +- .../FieldAliasMapperValidationTests.java | 3 +- .../index/mapper/IndexTimeScriptTests.java | 5 - .../index/mapper/LongScriptMapperTests.java | 54 +++---- .../mapper/MultiFieldsSerializationTests.java | 5 +- .../index/mapper/NumberFieldMapperTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 8 +- .../query/SearchExecutionContextTests.java | 7 +- .../elasticsearch/index/MapperTestUtils.java | 3 +- .../index/mapper/MapperScriptTestCase.java | 64 +++++++-- .../index/mapper/MockFieldMapper.java | 2 +- .../aggregations/AggregatorTestCase.java | 3 +- .../test/AbstractBuilderTestCase.java | 3 +- .../AggregateDoubleMetricFieldMapper.java | 17 ++- .../job/RollupIndexerIndexingTests.java | 7 +- 36 files changed, 630 insertions(+), 226 deletions(-) create mode 100644 modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 832801b5d786..d43714d0ad01 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -180,7 +180,27 @@ The following parameters are accepted by `boolean` fields: Accepts any of the true or false values listed above. The value is substituted for any explicit `null` values. Defaults to `null`, which - means the field is treated as missing. + means the field is treated as missing. Note that this cannot be set + if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `reject` (default), which + will cause the entire document to be rejected, and `ignore`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>. <>:: diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index a7bc4b9de915..84c2afc9bf3c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -545,7 +545,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper { PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder) { - super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo, false, null); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml new file mode 100644 index 000000000000..922f602f343e --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml @@ -0,0 +1,134 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + over_v: + type: boolean + script: + source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + params: + min_v: 5.0 + # Test fetching from _source + over_v_from_source: + type: boolean + script: + source: | + emit(params._source.voltage >= 5.0); + # Test many booleans + big_vals: + type: boolean + script: + source: | + for (def v : doc['temperature']) { + emit(v >= 200); + } + for (def v : doc['voltage']) { + emit(v >= 5.0); + } + + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.over_v.type: boolean } + - match: + sensor.mappings.properties.over_v.script.source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + - match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} } + - match: {sensor.mappings.properties.over_v.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [over_v, over_v_from_source, big_vals] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.over_v: [false] } + - match: {hits.hits.0.fields.over_v_from_source: [false] } + - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [over_v, over_v_from_source, big_vals] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.over_v: [false] } + - match: {hits.hits.0.fields.over_v_from_source: [false] } + - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + over_v: + terms: + field: over_v + - match: {hits.total.value: 6} + - match: {aggregations.over_v.buckets.0.key_as_string: "true"} + - match: {aggregations.over_v.buckets.0.doc_count: 4} + - match: {aggregations.over_v.buckets.1.key_as_string: "false"} + - match: {aggregations.over_v.buckets.1.doc_count: 2} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + over_v: true + sort: + timestamp: asc + - match: {hits.total.value: 4} + - match: {hits.hits.0._source.voltage: 5.6} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index e324ed89a464..1f2015eec88b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -142,7 +142,7 @@ public abstract class AbstractGeometryFieldMapper extends Fie Explicit ignoreMalformed, Explicit ignoreZValue, MultiFields multiFields, CopyTo copyTo, Indexer indexer, Parser parser) { - super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null); this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; this.indexer = indexer; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 113ae8d7ba14..a83a2df0e3d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -13,6 +13,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; @@ -25,7 +26,11 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.BooleanFieldScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -33,6 +38,7 @@ import java.time.ZoneId; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; /** @@ -72,43 +78,69 @@ public class BooleanFieldMapper extends FieldMapper { (n, c, o) -> o == null ? null : XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue) .acceptsNull(); + private final Parameter