github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add support for script parameter to boolean field mapper (#71454)

Browse source 
Relates to #68984
This commit is contained in:
Luca Cavanna 2021-04-12 10:04:12 +02:00 • committed by GitHub
parent f9440789e5
commit 1469e18c98
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 630 additions and 226 deletions
Download patch file Download diff file Expand all files Collapse all files

22
docs/reference/mapping/types/boolean.asciidoc
View file

@ -180,7 +180,27 @@ The following parameters are accepted by `boolean` fields:
Accepts any of the true or false values listed above. The value is
substituted for any explicit `null` values. Defaults to `null`, which
means the field is treated as missing.
means the field is treated as missing. Note that this cannot be set
if the `script` parameter is used.
`on_script_error`::
Defines what to do if the script defined by the `script` parameter
throws an error at indexing time. Accepts `reject` (default), which
will cause the entire document to be rejected, and `ignore`, which
will register the field in the document's
<<mapping-ignored-field,`_ignored`>> metadata field and continue
indexing. This parameter can only be set if the `script` field is
also set.
`script`::
If this parameter is set, then the field will index values generated
by this script, rather than reading the values directly from the
source. If a value is set for this field on the input document, then
the document will be rejected with an error.
Scripts are in the same format as their
<<runtime-mapping-fields,runtime equivalent>>.
<<mapping-store,`store`>>::

2
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
View file

@ -545,7 +545,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
PrefixFieldMapper prefixField,
ShingleFieldMapper[] shingleFields,
Builder builder) {
super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo);
super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo, false, null);
this.prefixField = prefixField;
this.shingleFields = shingleFields;
this.maxShingleSize = builder.maxShingleSize.getValue();

134
modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml Normal file
View file

@ -0,0 +1,134 @@
---
setup:
- do:
indices.create:
index: sensor
body:
settings:
number_of_shards: 1
number_of_replicas: 0
mappings:
properties:
timestamp:
type: date
temperature:
type: long
voltage:
type: double
node:
type: keyword
over_v:
type: boolean
script:
source: |
for (def v : doc['voltage']) {
emit(v >= params.min_v);
}
params:
min_v: 5.0
# Test fetching from _source
over_v_from_source:
type: boolean
script:
source: |
emit(params._source.voltage >= 5.0);
# Test many booleans
big_vals:
type: boolean
script:
source: |
for (def v : doc['temperature']) {
emit(v >= 200);
}
for (def v : doc['voltage']) {
emit(v >= 5.0);
}
- do:
bulk:
index: sensor
refresh: true
body: |
{"index":{}}
{"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
{"index":{}}
{"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
{"index":{}}
{"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
{"index":{}}
{"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
{"index":{}}
{"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
{"index":{}}
{"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
---
"get mapping":
- do:
indices.get_mapping:
index: sensor
- match: {sensor.mappings.properties.over_v.type: boolean }
- match:
sensor.mappings.properties.over_v.script.source: |
for (def v : doc['voltage']) {
emit(v >= params.min_v);
}
- match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} }
- match: {sensor.mappings.properties.over_v.script.lang: painless }
---
"fetch fields":
- do:
search:
index: sensor
body:
sort: timestamp
fields: [over_v, over_v_from_source, big_vals]
- match: {hits.total.value: 6}
- match: {hits.hits.0.fields.over_v: [false] }
- match: {hits.hits.0.fields.over_v_from_source: [false] }
- match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
---
"docvalue_fields":
- do:
search:
index: sensor
body:
sort: timestamp
docvalue_fields: [over_v, over_v_from_source, big_vals]
- match: {hits.total.value: 6}
- match: {hits.hits.0.fields.over_v: [false] }
- match: {hits.hits.0.fields.over_v_from_source: [false] }
- match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
---
"terms agg":
- do:
search:
index: sensor
body:
aggs:
over_v:
terms:
field: over_v
- match: {hits.total.value: 6}
- match: {aggregations.over_v.buckets.0.key_as_string: "true"}
- match: {aggregations.over_v.buckets.0.doc_count: 4}
- match: {aggregations.over_v.buckets.1.key_as_string: "false"}
- match: {aggregations.over_v.buckets.1.doc_count: 2}
---
"term query":
- do:
search:
index: sensor
body:
query:
term:
over_v: true
sort:
timestamp: asc
- match: {hits.total.value: 4}
- match: {hits.hits.0._source.voltage: 5.6}

2
server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View file

@ -142,7 +142,7 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
MultiFields multiFields, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo);
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.indexer = indexer;

70
server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
View file

@ -13,6 +13,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
@ -25,7 +26,11 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.script.BooleanFieldScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.lookup.FieldValues;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
@ -33,6 +38,7 @@ import java.time.ZoneId;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;
/**
@ -72,43 +78,69 @@ public class BooleanFieldMapper extends FieldMapper {
(n, c, o) -> o == null ? null : XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue)
.acceptsNull();
private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
public Builder(String name) {
private final ScriptCompiler scriptCompiler;
public Builder(String name, ScriptCompiler scriptCompiler) {
super(name);
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
this.script.precludesParameters(nullValue);
this.script.setValidator(s -> {
if (s != null && indexed.get() == false && docValues.get() == false) {
throw new MapperParsingException("Cannot define script on field with index:false and doc_values:false");
}
});
}
@Override
protected List<Parameter<?>> getParameters() {
return List.of(meta, docValues, indexed, nullValue, stored);
return List.of(meta, docValues, indexed, nullValue, stored, script, onScriptError);
}
@Override
public BooleanFieldMapper build(ContentPath contentPath) {
MappedFieldType ft = new BooleanFieldType(buildFullName(contentPath), indexed.getValue(), stored.getValue(),
docValues.getValue(), nullValue.getValue(), meta.getValue());
docValues.getValue(), nullValue.getValue(), scriptValues(), meta.getValue());
return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, contentPath), copyTo.build(), this);
}
private FieldValues<Boolean> scriptValues() {
if (script.get() == null) {
return null;
}
BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT);
return scriptFactory == null ? null : (lookup, ctx, doc, consumer) -> scriptFactory
.newFactory(name, script.get().getParams(), lookup)
.newInstance(ctx)
.runForDoc(doc, consumer);
}
}
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler()));
public static final class BooleanFieldType extends TermBasedFieldType {
private final Boolean nullValue;
private final FieldValues<Boolean> scriptValues;
public BooleanFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues,
Boolean nullValue, Map<String, String> meta) {
Boolean nullValue, FieldValues<Boolean> scriptValues, Map<String, String> meta) {
super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.nullValue = nullValue;
this.scriptValues = scriptValues;
}
public BooleanFieldType(String name) {
this(name, true, false, true, false, Collections.emptyMap());
this(name, true, false, true, false, null, Collections.emptyMap());
}
public BooleanFieldType(String name, boolean searchable) {
this(name, searchable, false, true, false, Collections.emptyMap());
this(name, searchable, false, true, false, null, Collections.emptyMap());
}
@Override
@ -121,7 +153,9 @@ public class BooleanFieldMapper extends FieldMapper {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
if (this.scriptValues != null) {
return FieldValues.valueFetcher(this.scriptValues, context);
}
return new SourceValueFetcher(name(), context, nullValue) {
@Override
protected Boolean parseSourceValue(Object value) {
@ -208,14 +242,21 @@ public class BooleanFieldMapper extends FieldMapper {
private final boolean indexed;
private final boolean hasDocValues;
private final boolean stored;
private final Script script;
private final FieldValues<Boolean> scriptValues;
private final ScriptCompiler scriptCompiler;
protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType,
MultiFields multiFields, CopyTo copyTo, Builder builder) {
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo,
builder.script.get() != null, builder.onScriptError.getValue());
this.nullValue = builder.nullValue.getValue();
this.stored = builder.stored.getValue();
this.indexed = builder.indexed.getValue();
this.hasDocValues = builder.docValues.getValue();
this.script = builder.script.get();
this.scriptValues = builder.scriptValues();
this.scriptCompiler = builder.scriptCompiler;
}
@Override
@ -240,7 +281,10 @@ public class BooleanFieldMapper extends FieldMapper {
value = context.parser().booleanValue();
}
}
indexValue(context, value);
}
private void indexValue(ParseContext context, Boolean value) {
if (value == null) {
return;
}
@ -257,14 +301,18 @@ public class BooleanFieldMapper extends FieldMapper {
}
}
@Override
protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, value -> indexValue(parseContext, value));
}
@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName()).init(this);
return new Builder(simpleName(), scriptCompiler).init(this);
}
@Override
protected String contentType() {
return CONTENT_TYPE;
}
}

7
server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
View file

@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
import org.elasticsearch.script.ScriptCompiler;
import java.io.IOException;
import java.time.format.DateTimeParseException;
@ -274,7 +275,7 @@ final class DynamicFieldsBuilder {
new NumberFieldMapper.Builder(
name,
NumberFieldMapper.NumberType.LONG,
null,
ScriptCompiler.NONE,
context.indexSettings().getSettings()
), context);
}
@ -287,13 +288,13 @@ final class DynamicFieldsBuilder {
createDynamicField(new NumberFieldMapper.Builder(
name,
NumberFieldMapper.NumberType.FLOAT,
null,
ScriptCompiler.NONE,
context.indexSettings().getSettings()), context);
}
@Override
public void newDynamicBooleanField(ParseContext context, String name) throws IOException {
createDynamicField(new BooleanFieldMapper.Builder(name), context);
createDynamicField(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE), context);
}
@Override

113
server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View file

@ -59,6 +59,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
protected final Map<String, NamedAnalyzer> indexAnalyzers;
protected final MultiFields multiFields;
protected final CopyTo copyTo;
protected final boolean hasScript;
protected final String onScriptError;
/**
* Create a FieldMapper with no index analyzers
@ -69,9 +71,25 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
*/
protected FieldMapper(String simpleName, MappedFieldType mappedFieldType,
MultiFields multiFields, CopyTo copyTo) {
this(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo);
this(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo, false, null);
}
/**
* Create a FieldMapper with no index analyzers
* @param simpleName the leaf name of the mapper
* @param mappedFieldType the MappedFieldType associated with this mapper
* @param multiFields sub fields of this mapper
* @param copyTo copyTo fields of this mapper
* @param hasScript whether a script is defined for the field
* @param onScriptError the behaviour for when the defined script fails at runtime
*/
protected FieldMapper(String simpleName, MappedFieldType mappedFieldType,
MultiFields multiFields, CopyTo copyTo,
boolean hasScript, String onScriptError) {
this(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo, hasScript, onScriptError);
}
/**
* Create a FieldMapper with a single associated index analyzer
* @param simpleName the leaf name of the mapper
@ -83,7 +101,26 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
protected FieldMapper(String simpleName, MappedFieldType mappedFieldType,
NamedAnalyzer indexAnalyzer,
MultiFields multiFields, CopyTo copyTo) {
this(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), indexAnalyzer), multiFields, copyTo);
this(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), indexAnalyzer), multiFields, copyTo,
false, null);
}
/**
* Create a FieldMapper with a single associated index analyzer
* @param simpleName the leaf name of the mapper
* @param mappedFieldType the MappedFieldType associated with this mapper
* @param indexAnalyzer the index-time analyzer to use for this field
* @param multiFields sub fields of this mapper
* @param copyTo copyTo fields of this mapper
* @param hasScript whether a script is defined for the field
* @param onScriptError the behaviour for when the defined script fails at runtime
*/
protected FieldMapper(String simpleName, MappedFieldType mappedFieldType,
NamedAnalyzer indexAnalyzer,
MultiFields multiFields, CopyTo copyTo,
boolean hasScript, String onScriptError) {
this(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), indexAnalyzer), multiFields, copyTo,
hasScript, onScriptError);
}
/**
@ -94,10 +131,13 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
* the mapper will add
* @param multiFields sub fields of this mapper
* @param copyTo copyTo fields of this mapper
* @param hasScript whether a script is defined for the field
* @param onScriptError the behaviour for when the defined script fails at runtime
*/
protected FieldMapper(String simpleName, MappedFieldType mappedFieldType,
Map<String, NamedAnalyzer> indexAnalyzers,
MultiFields multiFields, CopyTo copyTo) {
MultiFields multiFields, CopyTo copyTo,
boolean hasScript, String onScriptError) {
super(simpleName);
if (mappedFieldType.name().isEmpty()) {
throw new IllegalArgumentException("name cannot be empty string");
@ -106,6 +146,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
this.indexAnalyzers = indexAnalyzers;
this.multiFields = multiFields;
this.copyTo = Objects.requireNonNull(copyTo);
this.hasScript = hasScript;
this.onScriptError = onScriptError;
}
@Override
@ -148,6 +190,9 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
*/
public void parse(ParseContext context) throws IOException {
try {
if (hasScript) {
throw new IllegalArgumentException("Cannot index data directly into a field with a [script] parameter");
}
parseCreateField(context);
} catch (Exception e) {
String valuePreview = "";
@ -172,11 +217,19 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
multiFields.parse(this, context);
}
/**
* Parse the field value and populate the fields on {@link ParseContext#doc()}.
*
* Implementations of this method should ensure that on failing to parse parser.currentToken() must be the
* current failing token
*/
protected abstract void parseCreateField(ParseContext context) throws IOException;
/**
* @return whether this field mapper uses a script to generate its values
*/
public boolean hasScript() {
return false;
public final boolean hasScript() {
return hasScript;
}
/**
@ -184,20 +237,34 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
*
* This method should only be called if {@link #hasScript()} has returned {@code true}
* @param searchLookup a SearchLookup to be passed the script
* @param ctx a LeafReaderContext exposing values from an incoming document
* @param pc the ParseContext over the incoming document
* @param readerContext a LeafReaderContext exposing values from an incoming document
* @param doc the id of the document to execute the script against
* @param parseContext the ParseContext over the incoming document
*/
public void executeScript(SearchLookup searchLookup, LeafReaderContext ctx, int doc, ParseContext pc) {
throw new UnsupportedOperationException("FieldMapper " + name() + " does not have an index-time script");
public final void executeScript(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
try {
indexScriptValues(searchLookup, readerContext, doc, parseContext);
} catch (Exception e) {
if ("ignore".equals(onScriptError)) {
parseContext.addIgnoredField(name());
} else {
throw new MapperParsingException("Error executing script on field [" + name() + "]", e);
}
}
}
/**
* Parse the field value and populate the fields on {@link ParseContext#doc()}.
* Run the script associated with the field and index the values that it emits
*
* Implementations of this method should ensure that on failing to parse parser.currentToken() must be the
* current failing token
* This method should only be called if {@link #hasScript()} has returned {@code true}
* @param searchLookup a SearchLookup to be passed the script
* @param readerContext a LeafReaderContext exposing values from an incoming document
* @param doc the id of the document to execute the script against
* @param parseContext the ParseContext over the incoming document
*/
protected abstract void parseCreateField(ParseContext context) throws IOException;
protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
throw new UnsupportedOperationException("FieldMapper " + name() + " does not support [script]");
}
protected final void createFieldNamesField(ParseContext context) {
assert fieldType().hasDocValues() == false : "_field_names should only be used when doc_values are turned off";
@ -532,8 +599,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
private MergeValidator<T> mergeValidator;
private T value;
private boolean isSet;
private List<Parameter<?>> requires = new ArrayList<>();
private List<Parameter<?>> precludes = new ArrayList<>();
private final List<Parameter<?>> requires = new ArrayList<>();
private final List<Parameter<?>> precludes = new ArrayList<>();
/**
* Creates a new Parameter
@ -875,7 +942,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
* @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges
* @return a script parameter
*/
public static FieldMapper.Parameter<Script> scriptParam(
public static Parameter<Script> scriptParam(
Function<FieldMapper, Script> initializer
) {
return new FieldMapper.Parameter<>(
@ -896,6 +963,20 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
).acceptsNull();
}
/**
* Defines an on_script_error parameter
* @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges
* @param dependentScriptParam the corresponding required script parameter
* @return a new on_error_script parameter
*/
public static Parameter<String> onScriptErrorParam(Function<FieldMapper, String> initializer,
Parameter<Script> dependentScriptParam) {
return Parameter.restrictedStringParam(
"on_script_error",
true,
initializer,
"reject", "ignore").requiresParameters(dependentScriptParam);
}
}
public static final class Conflicts {

72
server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View file

@ -44,7 +44,6 @@ import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.lookup.FieldValues;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException;
import java.time.ZoneId;
@ -81,24 +80,19 @@ public class NumberFieldMapper extends FieldMapper {
private final Parameter<Number> nullValue;
private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).builder.script.get());
private final Parameter<String> onScriptError = Parameter.restrictedStringParam(
"on_script_error",
true,
m -> toType(m).onScriptError,
"reject", "ignore"
);
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
private final ScriptCompiler scriptCompiler;
private final NumberType type;
private final ScriptCompiler compiler;
public Builder(String name, NumberType type, ScriptCompiler compiler, Settings settings) {
this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings));
}
public static Builder docValuesOnly(String name, NumberType type) {
Builder builder = new Builder(name, type, null, false, false);
Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false);
builder.indexed.setValue(false);
return builder;
}
@ -106,7 +100,7 @@ public class NumberFieldMapper extends FieldMapper {
public Builder(String name, NumberType type, ScriptCompiler compiler, boolean ignoreMalformedByDefault, boolean coerceByDefault) {
super(name);
this.type = type;
this.compiler = compiler;
this.scriptCompiler = Objects.requireNonNull(compiler);
this.ignoreMalformed
= Parameter.explicitBoolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
@ -115,7 +109,6 @@ public class NumberFieldMapper extends FieldMapper {
this.nullValue = new Parameter<>("null_value", false, () -> null,
(n, c, o) -> o == null ? null : type.parse(o, false), m -> toType(m).nullValue).acceptsNull();
this.onScriptError.requiresParameters(this.script);
this.script.precludesParameters(ignoreMalformed, coerce, nullValue);
this.script.setValidator(s -> {
if (s != null && indexed.get() == false && hasDocValues.get() == false) {
@ -138,8 +131,7 @@ public class NumberFieldMapper extends FieldMapper {
if (this.script.get() == null) {
return null;
}
assert compiler != null;
return type.compile(name, script.get(), compiler);
return type.compile(name, script.get(), scriptCompiler);
}
@Override
@ -1033,28 +1025,8 @@ public class NumberFieldMapper extends FieldMapper {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
if (this.scriptValues != null) {
return new ValueFetcher() {
LeafReaderContext ctx;
@Override
public void setNextReader(LeafReaderContext context) {
this.ctx = context;
return FieldValues.valueFetcher(this.scriptValues, context);
}
@Override
public List<Object> fetchValues(SourceLookup lookup) {
List<Object> values = new ArrayList<>();
try {
scriptValues.valuesForDoc(context.lookup(), ctx, lookup.docId(), values::add);
} catch (Exception e) {
// ignore errors - if they exist here then they existed at index time
// and so on_script_error must have been set to `ignore`
}
return values;
}
};
}
return new SourceValueFetcher(name(), context, nullValue) {
@Override
protected Object parseSourceValue(Object value) {
@ -1099,8 +1071,6 @@ public class NumberFieldMapper extends FieldMapper {
private final Explicit<Boolean> coerce;
private final Number nullValue;
private final FieldValues<Number> scriptValues;
private final String onScriptError;
private final boolean ignoreMalformedByDefault;
private final boolean coerceByDefault;
@ -1110,7 +1080,7 @@ public class NumberFieldMapper extends FieldMapper {
MultiFields multiFields,
CopyTo copyTo,
Builder builder) {
super(simpleName, mappedFieldType, multiFields, copyTo);
super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.getValue());
this.type = builder.type;
this.indexed = builder.indexed.getValue();
this.hasDocValues = builder.hasDocValues.getValue();
@ -1121,7 +1091,6 @@ public class NumberFieldMapper extends FieldMapper {
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
this.coerceByDefault = builder.coerce.getDefaultValue().value();
this.scriptValues = builder.scriptValues();
this.onScriptError = builder.onScriptError.get();
this.builder = builder;
}
@ -1133,10 +1102,6 @@ public class NumberFieldMapper extends FieldMapper {
return ignoreMalformed.value();
}
String onScriptError() {
return onScriptError;
}
@Override
public NumberFieldType fieldType() {
return (NumberFieldType) super.fieldType();
@ -1149,11 +1114,6 @@ public class NumberFieldMapper extends FieldMapper {
@Override
protected void parseCreateField(ParseContext context) throws IOException {
if (this.scriptValues != null) {
throw new IllegalArgumentException("Cannot index data directly into a field with a [script] parameter");
}
XContentParser parser = context.parser();
Object value;
Number numericValue = null;
@ -1204,26 +1164,12 @@ public class NumberFieldMapper extends FieldMapper {
}
@Override
public boolean hasScript() {
return this.scriptValues != null;
}
@Override
public void executeScript(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
assert this.scriptValues != null;
try {
protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, value -> indexValue(parseContext, value));
} catch (Exception e) {
if ("ignore".equals(onScriptError)) {
parseContext.addIgnoredField(name());
} else {
throw new MapperParsingException("Error executing script on field [" + name() + "]", e);
}
}
}
@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), type, builder.compiler, ignoreMalformedByDefault, coerceByDefault).init(this);
return new Builder(simpleName(), type, builder.scriptCompiler, ignoreMalformedByDefault, coerceByDefault).init(this);
}
}

2
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View file

@ -792,7 +792,7 @@ public class TextFieldMapper extends FieldMapper {
SubFieldInfo prefixFieldInfo,
SubFieldInfo phraseFieldInfo,
MultiFields multiFields, CopyTo copyTo, Builder builder) {
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo);
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
assert mappedFieldType.getTextSearchInfo().isTokenized();
assert mappedFieldType.hasDocValues() == false;
if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) {

11
server/src/main/java/org/elasticsearch/script/BooleanFieldScript.java
View file

@ -12,6 +12,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.search.lookup.SearchLookup;
import java.util.Map;
import java.util.function.Consumer;
public abstract class BooleanFieldScript extends AbstractFieldScript {
@ -45,6 +46,14 @@ public abstract class BooleanFieldScript extends AbstractFieldScript {
execute();
}
public final void runForDoc(int docId, Consumer<Boolean> consumer) {
runForDoc(docId);
int count = trues + falses;
for (int i = 0; i < count; i++) {
consumer.accept(i < falses ? false : true);
}
}
/**
* How many {@code true} values were returned for this document.
*/
@ -59,7 +68,7 @@ public abstract class BooleanFieldScript extends AbstractFieldScript {
return falses;
}
protected final void emit(boolean v) {
public final void emit(boolean v) {
if (v) {
trues++;
} else {

6
server/src/main/java/org/elasticsearch/script/ScriptCompiler.java
View file

@ -22,4 +22,10 @@ public interface ScriptCompiler {
*/
<T> T compile(Script script, ScriptContext<T> scriptContext);
ScriptCompiler NONE = new ScriptCompiler() {
@Override
public <T> T compile(Script script, ScriptContext<T> scriptContext) {
throw new UnsupportedOperationException();
}
};
}

33
server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java
View file

@ -9,7 +9,11 @@
package org.elasticsearch.search.lookup;
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.SearchExecutionContext;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
/**
@ -25,4 +29,33 @@ public interface FieldValues<T> {
* @param consumer called with each document value
*/
void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer<T> consumer);
/**
* Creates a {@link ValueFetcher} that fetches values from a {@link FieldValues} instance
* @param fieldValues the source of the values
* @param context the search execution context
* @return the value fetcher
*/
static ValueFetcher valueFetcher(FieldValues<?> fieldValues, SearchExecutionContext context) {
return new ValueFetcher() {
LeafReaderContext ctx;
@Override
public void setNextReader(LeafReaderContext context) {
this.ctx = context;
}
@Override
public List<Object> fetchValues(SourceLookup lookup) {
List<Object> values = new ArrayList<>();
try {
fieldValues.valuesForDoc(context.lookup(), ctx, lookup.docId(), values::add);
} catch (Exception e) {
// ignore errors - if they exist here then they existed at index time
// and so on_script_error must have been set to `ignore`
}
return values;
}
};
}
}

3
server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
View file

@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -108,7 +109,7 @@ public class CodecTests extends ESTestCase {
MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
MapperPlugin.NOOP_FIELD_FILTER);
MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry,
() -> null, () -> false, null);
() -> null, () -> false, ScriptCompiler.NONE);
return new CodecService(service, LogManager.getLogger("test"));
}

13
server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
View file

@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.junit.After;
@ -92,22 +93,22 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
.fielddata(true).build(contentPath).fieldType();
}
} else if (type.equals("float")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.FLOAT, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.FLOAT, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("double")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("long")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.LONG, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.LONG, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("int")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("short")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.SHORT, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.SHORT, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("byte")) {
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.BYTE, null, false, true)
fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.BYTE, ScriptCompiler.NONE, false, true)
.docValues(docValues).build(contentPath).fieldType();
} else if (type.equals("geo_point")) {
fieldType = new GeoPointFieldMapper.Builder(fieldName, false).docValues(docValues).build(contentPath).fieldType();

15
server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
View file

@ -36,6 +36,7 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -80,10 +81,10 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData);
for (MappedFieldType mapper : Arrays.asList(
new NumberFieldMapper.Builder("int", BYTE, null, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("int", SHORT, null, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("int", INTEGER, null, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("long", LONG, null, false, true).build(contentPath).fieldType()
new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true).build(contentPath).fieldType(),
new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true).build(contentPath).fieldType()
)) {
ifdService.clear();
fd = ifdService.getForField(mapper, "test", () -> {
@ -92,7 +93,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
assertTrue(fd instanceof SortedNumericIndexFieldData);
}
final MappedFieldType floatMapper = new NumberFieldMapper.Builder("float", NumberType.FLOAT, null, false, true)
final MappedFieldType floatMapper = new NumberFieldMapper.Builder("float", NumberType.FLOAT, ScriptCompiler.NONE, false, true)
.build(contentPath).fieldType();
ifdService.clear();
fd = ifdService.getForField(floatMapper, "test", () -> {
@ -101,7 +102,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
assertTrue(fd instanceof SortedNumericIndexFieldData);
final MappedFieldType doubleMapper
= new NumberFieldMapper.Builder("double", DOUBLE, null, false, true)
= new NumberFieldMapper.Builder("double", DOUBLE, ScriptCompiler.NONE, false, true)
.build(contentPath).fieldType();
ifdService.clear();
fd = ifdService.getForField(doubleMapper, "test", () -> {
@ -321,6 +322,6 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
public void testRequireDocValuesOnBools() {
doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field"));
doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, Collections.emptyMap()));
doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap()));
}
}

1
server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
View file

@ -310,6 +310,7 @@ public abstract class AbstractScriptFieldTypeTestCase extends MapperServiceTestC
}
@Override
@SuppressWarnings("unchecked")
protected <T> T compileScript(Script script, ScriptContext<T> context) {
if (context == BooleanFieldScript.CONTEXT) {
return (T) BooleanFieldScriptTests.DUMMY;

12
server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
View file

@ -21,6 +21,8 @@ import org.elasticsearch.index.mapper.ParseContext.Document;
import java.io.IOException;
import static org.hamcrest.Matchers.equalTo;
public class BooleanFieldMapperTests extends MapperTestCase {
@Override
@ -178,4 +180,14 @@ public class BooleanFieldMapperTests extends MapperTestCase {
throw new IllegalStateException();
}
}
public void testScriptAndPrecludedParameters() {
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
b.field("type", "boolean");
b.field("script", "test");
b.field("null_value", true);
})));
assertThat(e.getMessage(),
equalTo("Failed to parse mapping: Field [null_value] cannot be set in conjunction with field [script]"));
}
}

2
server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
View file

@ -50,7 +50,7 @@ public class BooleanFieldTypeTests extends FieldTypeTestCase {
assertEquals(List.of(false), fetchSourceValue(fieldType, ""));
MappedFieldType nullFieldType = new BooleanFieldMapper.BooleanFieldType(
"field", true, false, true, true, Collections.emptyMap()
"field", true, false, true, true, null, Collections.emptyMap()
);
assertEquals(List.of(true), fetchSourceValue(nullFieldType, null));
}

5
server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
View file

@ -32,12 +32,13 @@ import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
import org.elasticsearch.script.BooleanFieldScript;
import org.elasticsearch.script.ScoreScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.test.ESTestCase;
@ -302,7 +303,7 @@ public class BooleanScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeT
}
public void testDualingQueries() throws IOException {
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo").build(new ContentPath());
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE).build(new ContentPath());
try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
List<Boolean> values = randomList(0, 2, ESTestCase::randomBoolean);
String source = "{\"foo\": " + values + "}";

90
server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java Normal file
View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.script.BooleanFieldScript;
import org.elasticsearch.search.lookup.SearchLookup;
import java.util.Map;
import java.util.function.Consumer;
public class BooleanScriptMapperTests extends MapperScriptTestCase<BooleanFieldScript.Factory> {
private static BooleanFieldScript.Factory factory(Consumer<BooleanFieldScript> executor) {
return new BooleanFieldScript.Factory() {
@Override
public BooleanFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
return new BooleanFieldScript.LeafFactory() {
@Override
public BooleanFieldScript newInstance(LeafReaderContext ctx) {
return new BooleanFieldScript(fieldName, params, searchLookup, ctx) {
@Override
public void execute() {
executor.accept(this);
}
};
}
};
}
};
}
@Override
protected String type() {
return BooleanFieldMapper.CONTENT_TYPE;
}
@Override
protected BooleanFieldScript.Factory serializableScript() {
return factory(s -> {});
}
@Override
protected BooleanFieldScript.Factory errorThrowingScript() {
return factory(s -> {
throw new UnsupportedOperationException("Oops");
});
}
@Override
protected BooleanFieldScript.Factory singleValueScript() {
return factory(s -> s.emit(true));
}
@Override
protected BooleanFieldScript.Factory multipleValuesScript() {
return factory(s -> {
s.emit(true);
s.emit(false);
});
}
@Override
protected void assertMultipleValues(IndexableField[] fields) {
assertEquals(4, fields.length);
assertEquals("indexed,omitNorms,indexOptions=DOCS<field:F>", fields[0].toString());
assertEquals("docValuesType=SORTED_NUMERIC<field:0>", fields[1].toString());
assertEquals("indexed,omitNorms,indexOptions=DOCS<field:T>", fields[2].toString());
assertEquals("docValuesType=SORTED_NUMERIC<field:1>", fields[3].toString());
}
@Override
protected void assertDocValuesDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("indexed,omitNorms,indexOptions=DOCS<field:T>", fields[0].toString());
}
@Override
protected void assertIndexDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("docValuesType=SORTED_NUMERIC<field:1>", fields[0].toString());
}
}

41
server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptMapperTests.java
View file

@ -13,7 +13,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.script.DoubleFieldScript;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
import java.util.Map;
import java.util.function.Consumer;
@ -40,7 +39,7 @@ public class DoubleScriptMapperTests extends MapperScriptTestCase<DoubleFieldScr
@Override
protected String type() {
return "double";
return NumberFieldMapper.NumberType.DOUBLE.typeName();
}
@Override
@ -56,26 +55,20 @@ public class DoubleScriptMapperTests extends MapperScriptTestCase<DoubleFieldScr
}
@Override
protected DoubleFieldScript.Factory compileScript(String name) {
if ("single-valued".equals(name)) {
protected DoubleFieldScript.Factory singleValueScript() {
return factory(s -> s.emit(3.14));
}
if ("multi-valued".equals(name)) {
@Override
protected DoubleFieldScript.Factory multipleValuesScript() {
return factory(s -> {
s.emit(3.14);
s.emit(2.78);
});
}
return super.compileScript(name);
}
public void testMultipleValues() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "double");
b.field("script", "multi-valued");
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertMultipleValues(IndexableField[] fields) {
assertEquals(4, fields.length);
assertEquals("DoublePoint <field:3.14>", fields[0].toString());
assertEquals("docValuesType=SORTED_NUMERIC<field:4614253070214989087>", fields[1].toString());
@ -83,26 +76,14 @@ public class DoubleScriptMapperTests extends MapperScriptTestCase<DoubleFieldScr
assertEquals("docValuesType=SORTED_NUMERIC<field:4613442422282062397>", fields[3].toString());
}
public void testDocValuesDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "double");
b.field("script", "single-valued");
b.field("doc_values", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertDocValuesDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("DoublePoint <field:3.14>", fields[0].toString());
}
public void testIndexDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "double");
b.field("script", "single-valued");
b.field("index", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertIndexDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("docValuesType=SORTED_NUMERIC<field:4614253070214989087>", fields[0].toString());
}

3
server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
View file

@ -17,6 +17,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.script.ScriptCompiler;
import java.io.IOException;
import java.nio.charset.Charset;
@ -52,7 +53,7 @@ public class ExternalMapper extends FieldMapper {
public static class Builder extends FieldMapper.Builder {
private final BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN);
private final BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL);
private final BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL, ScriptCompiler.NONE);
private final GeoPointFieldMapper.Builder latLonPointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT, false);
private final GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE, false, true);
private final Mapper.Builder stringBuilder;

3
server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
View file

@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
@ -177,7 +178,7 @@ public class FieldAliasMapperValidationTests extends ESTestCase {
}
private static FieldMapper createFieldMapper(String parent, String name) {
return new BooleanFieldMapper.Builder(name).build(new ContentPath(parent));
return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE).build(new ContentPath(parent));
}
private static ObjectMapper createObjectMapper(String name) {

5
server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptTests.java
View file

@ -230,10 +230,5 @@ public class IndexTimeScriptTests extends MapperServiceTestCase {
public void emitValue(double v) {
super.emit(v);
}
public List<Object> extractValuesFromSource(String path) {
return super.extractFromSource(path);
}
}
}

42
server/src/test/java/org/elasticsearch/index/mapper/LongScriptMapperTests.java
View file

@ -13,7 +13,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.script.LongFieldScript;
import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
import java.util.Map;
import java.util.function.Consumer;
@ -40,7 +39,7 @@ public class LongScriptMapperTests extends MapperScriptTestCase<LongFieldScript.
@Override
protected String type() {
return "long";
return NumberFieldMapper.NumberType.LONG.typeName();
}
@Override
@ -56,26 +55,20 @@ public class LongScriptMapperTests extends MapperScriptTestCase<LongFieldScript.
}
@Override
protected LongFieldScript.Factory compileScript(String name) {
if ("single-valued".equals(name)) {
protected LongFieldScript.Factory singleValueScript() {
return factory(s -> s.emit(4));
}
if ("multi-valued".equals(name)) {
@Override
protected LongFieldScript.Factory multipleValuesScript() {
return factory(s -> {
s.emit(1);
s.emit(2);
});
}
return super.compileScript(name);
}
public void testMultipleValues() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "long");
b.field("script", "multi-valued");
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertMultipleValues(IndexableField[] fields) {
assertEquals(4, fields.length);
assertEquals("LongPoint <field:1>", fields[0].toString());
assertEquals("docValuesType=SORTED_NUMERIC<field:1>", fields[1].toString());
@ -83,28 +76,15 @@ public class LongScriptMapperTests extends MapperScriptTestCase<LongFieldScript.
assertEquals("docValuesType=SORTED_NUMERIC<field:2>", fields[3].toString());
}
public void testDocValuesDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "long");
b.field("script", "single-valued");
b.field("doc_values", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertDocValuesDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("LongPoint <field:4>", fields[0].toString());
}
public void testIndexDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "long");
b.field("script", "single-valued");
b.field("index", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
IndexableField[] fields = doc.rootDoc().getFields("field");
@Override
protected void assertIndexDisabled(IndexableField[] fields) {
assertEquals(1, fields.length);
assertEquals("docValuesType=SORTED_NUMERIC<field:4>", fields[0].toString());
}
}

5
server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.Strings;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.test.ESTestCase;
import java.util.ArrayList;
@ -35,10 +36,10 @@ public class MultiFieldsSerializationTests extends ESTestCase {
sortedNames.sort(Comparator.naturalOrder());
for (String name : names) {
builder.add(new BooleanFieldMapper.Builder(name));
builder.add(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE));
}
Mapper.Builder root = new BooleanFieldMapper.Builder("root");
Mapper.Builder root = new BooleanFieldMapper.Builder("root", ScriptCompiler.NONE);
FieldMapper.MultiFields multiFields = builder.build(root, new ContentPath());
String serialized = Strings.toString(multiFields);

2
server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
View file

@ -68,7 +68,7 @@ public abstract class NumberFieldMapperTests extends MapperTestCase {
b.field("script", "test");
b.field("on_script_error", "ignore");
},
m -> assertThat(((NumberFieldMapper)m).onScriptError(), equalTo("ignore")));
m -> assertThat((m).onScriptError, equalTo("ignore")));
}
}

8
server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
View file

@ -9,7 +9,6 @@
package org.elasticsearch.index.mapper;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
@ -42,6 +41,7 @@ import org.elasticsearch.index.mapper.MappedFieldType.Relation;
import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Before;
@ -634,14 +634,14 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
}
public void testFetchSourceValue() throws IOException {
MappedFieldType mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER, null, false, true)
MappedFieldType mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER, ScriptCompiler.NONE, false, true)
.build(new ContentPath())
.fieldType();
assertEquals(List.of(3), fetchSourceValue(mapper, 3.14));
assertEquals(List.of(42), fetchSourceValue(mapper, "42.9"));
assertEquals(List.of(3, 42), fetchSourceValues(mapper, 3.14, "foo", "42.9"));
MappedFieldType nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT, null, false, true)
MappedFieldType nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT, ScriptCompiler.NONE, false, true)
.nullValue(2.71f)
.build(new ContentPath())
.fieldType();
@ -650,7 +650,7 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
}
public void testFetchHalfFloatFromSource() throws IOException {
MappedFieldType mapper = new NumberFieldMapper.Builder("field", NumberType.HALF_FLOAT, null, false, true)
MappedFieldType mapper = new NumberFieldMapper.Builder("field", NumberType.HALF_FLOAT, ScriptCompiler.NONE, false, true)
.build(new ContentPath())
.fieldType();
/*

7
server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
View file

@ -44,6 +44,8 @@ import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.IndexFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.KeywordScriptFieldType;
import org.elasticsearch.index.mapper.LongScriptFieldType;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
@ -59,8 +61,7 @@ import org.elasticsearch.index.mapper.RuntimeField;
import org.elasticsearch.index.mapper.TestRuntimeField;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.index.mapper.KeywordScriptFieldType;
import org.elasticsearch.index.mapper.LongScriptFieldType;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@ -451,7 +452,7 @@ public class SearchExecutionContextTests extends ESTestCase {
indexSettings.getIndexVersionCreated(),
searchExecutionContextSupplier,
null,
null,
ScriptCompiler.NONE,
indexAnalyzers,
indexSettings,
() -> true

3
test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View file

@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.test.IndexSettingsModule;
import java.io.IOException;
@ -55,6 +56,6 @@ public class MapperTestUtils {
xContentRegistry,
similarityService,
mapperRegistry,
() -> null, () -> false, null);
() -> null, () -> false, ScriptCompiler.NONE);
}
}

64
test/framework/src/main/java/org/elasticsearch/index/mapper/MapperScriptTestCase.java
View file

@ -8,6 +8,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
@ -26,23 +27,29 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
protected abstract FactoryType errorThrowingScript();
protected FactoryType compileScript(String name) {
throw new UnsupportedOperationException("Unknown script " + name);
}
protected abstract FactoryType singleValueScript();
protected abstract FactoryType multipleValuesScript();
@Override
@SuppressWarnings("unchecked")
protected final <T> T compileScript(Script script, ScriptContext<T> context) {
protected <T> T compileScript(Script script, ScriptContext<T> context) {
if (script.getIdOrCode().equals("serializer_test")) {
return (T) serializableScript();
}
if (script.getIdOrCode().equals("throws")) {
return (T) errorThrowingScript();
}
return (T) compileScript(script.getIdOrCode());
if (script.getIdOrCode().equals("single-valued")) {
return (T) singleValueScript();
}
if (script.getIdOrCode().equals("multi-valued")) {
return (T) multipleValuesScript();
}
throw new UnsupportedOperationException("Unknown script " + script.getIdOrCode());
}
public void testSerialization() throws IOException {
public void testToXContent() throws IOException {
DocumentMapper mapper = createDocumentMapper(mapping(b -> {
b.startObject("scripted");
b.field("type", type());
@ -70,7 +77,7 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
assertEquals("Cannot index data directly into a field with a [script] parameter", e.getCause().getMessage());
}
public void testStoredScriptsNotPermitted() {
public final void testStoredScriptsNotPermitted() {
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.startObject("script").field("id", "foo").endObject();
@ -78,7 +85,7 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
assertThat(e.getMessage(), equalTo("Failed to parse mapping: stored scripts are not supported on field [field]"));
}
public void testIndexAndDocValuesFalseNotPermitted() {
public final void testIndexAndDocValuesFalseNotPermitted() {
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.field("index", false);
@ -88,7 +95,7 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
assertThat(e.getMessage(), containsString("Cannot define script on field with index:false and doc_values:false"));
}
public void testScriptErrorParameterRequiresScript() {
public final void testOnScriptErrorParameterRequiresScript() {
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.field("on_script_error", "ignore");
@ -97,7 +104,7 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
equalTo("Failed to parse mapping: Field [on_script_error] requires field [script] to be configured"));
}
public void testIgnoreScriptErrors() throws IOException {
public final void testIgnoreScriptErrors() throws IOException {
DocumentMapper mapper = createDocumentMapper(mapping(b -> {
b.startObject("message").field("type", "keyword").endObject();
b.startObject("message_error");
@ -114,7 +121,7 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
assertThat(doc.rootDoc().getField("_ignored").stringValue(), equalTo("message_error"));
}
public void testRejectScriptErrors() throws IOException {
public final void testRejectScriptErrors() throws IOException {
DocumentMapper mapper = createDocumentMapper(mapping(b -> {
b.startObject("message").field("type", "keyword").endObject();
b.startObject("message_error");
@ -128,4 +135,39 @@ public abstract class MapperScriptTestCase<FactoryType> extends MapperServiceTes
Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("message", "foo"))));
assertThat(e.getMessage(), equalTo("Error executing script on field [message_error]"));
}
public final void testMultipleValues() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.field("script", "multi-valued");
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
assertMultipleValues(doc.rootDoc().getFields("field"));
}
protected abstract void assertMultipleValues(IndexableField[] fields);
public final void testDocValuesDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.field("script", "single-valued");
b.field("doc_values", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
assertDocValuesDisabled(doc.rootDoc().getFields("field"));
}
protected abstract void assertDocValuesDisabled(IndexableField[] fields);
public final void testIndexDisabled() throws IOException {
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", type());
b.field("script", "single-valued");
b.field("index", false);
}));
ParsedDocument doc = mapper.parse(source(b -> {}));
assertIndexDisabled(doc.rootDoc().getFields("field"));
}
protected abstract void assertIndexDisabled(IndexableField[] fields);
}

2
test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
View file

@ -28,7 +28,7 @@ public class MockFieldMapper extends FieldMapper {
public MockFieldMapper(MappedFieldType fieldType, Map<String, NamedAnalyzer> indexAnalyzers) {
super(findSimpleName(fieldType.name()), fieldType, indexAnalyzers,
MultiFields.empty(), new CopyTo.Builder().build());
MultiFields.empty(), new CopyTo.Builder().build(), false, null);
}
public MockFieldMapper(String fullName,

3
test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
View file

@ -92,6 +92,7 @@ import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.plugins.SearchPlugin;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.NestedDocuments;
import org.elasticsearch.search.SearchModule;
@ -875,7 +876,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
private static class MockParserContext extends Mapper.TypeParser.ParserContext {
MockParserContext() {
super(null, null, null, null, null, null, null, null, null, null);
super(null, null, null, null, null, null, ScriptCompiler.NONE, null, null, null);
}
@Override

3
test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
View file

@ -57,6 +57,7 @@ import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.plugins.SearchPlugin;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.MockScriptService;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptEngine;
import org.elasticsearch.script.ScriptModule;
@ -344,7 +345,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap());
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
() -> createShardContext(null), () -> false, null);
() -> createShardContext(null), () -> false, ScriptCompiler.NONE);
IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
});
indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,

17
x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
View file

@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.lookup.SearchLookup;
@ -184,9 +185,21 @@ public class AggregateDoubleMetricFieldMapper extends FieldMapper {
if (m == Metric.value_count) {
// value_count metric can only be an integer and not a double
builder = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER, null, false, false);
builder = new NumberFieldMapper.Builder(
fieldName,
NumberFieldMapper.NumberType.INTEGER,
ScriptCompiler.NONE,
false,
false
);
} else {
builder = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE, null, false, true);
builder = new NumberFieldMapper.Builder(
fieldName,
NumberFieldMapper.NumberType.DOUBLE,
ScriptCompiler.NONE,
false,
true
);
}
NumberFieldMapper fieldMapper = builder.build(context);
metricMappers.put(m, fieldMapper);

7
x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
View file

@ -40,8 +40,9 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
@ -536,7 +537,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
if (job.getGroupConfig().getHistogram() != null) {
for (String field : job.getGroupConfig().getHistogram().getFields()) {
MappedFieldType ft = new NumberFieldMapper.Builder(field, NumberType.LONG, null, false, false)
MappedFieldType ft = new NumberFieldMapper.Builder(field, NumberType.LONG, ScriptCompiler.NONE, false, false)
.build(new ContentPath(0))
.fieldType();
fieldTypes.put(ft.name(), ft);
@ -554,7 +555,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
if (job.getMetricsConfig() != null) {
for (MetricConfig metric : job.getMetricsConfig()) {
MappedFieldType ft = new NumberFieldMapper.Builder(metric.getField(), NumberType.LONG, null, false, false)
MappedFieldType ft = new NumberFieldMapper.Builder(metric.getField(), NumberType.LONG, ScriptCompiler.NONE, false, false)
.build(new ContentPath(0))
.fieldType();
fieldTypes.put(ft.name(), ft);