github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-19 04:45:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

Store arrays offsets for scaled float fields natively with synthetic source (#125793)

Browse source 
This patch builds on the work in #113757, #122999, #124594, #125529, and 
#125709 to natively store array offsets for scaled float fields instead of
falling back to ignored source when synthetic_source_keep: arrays.
This commit is contained in:
Jordan Powers 2025-03-28 12:26:29 -07:00 • committed by GitHub
parent 848a6783f0
commit 71e74bdd66
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 298 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

126
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
View file

@ -19,6 +19,8 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
@ -32,6 +34,7 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
import org.elasticsearch.index.mapper.BlockDocValuesReader;
import org.elasticsearch.index.mapper.BlockLoader;
import org.elasticsearch.index.mapper.BlockSourceReader;
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
import org.elasticsearch.index.mapper.DocumentParserContext;
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
import org.elasticsearch.index.mapper.FieldMapper;
@ -40,6 +43,8 @@ import org.elasticsearch.index.mapper.MapperBuilderContext;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer;
import org.elasticsearch.index.mapper.SourceLoader;
import org.elasticsearch.index.mapper.SourceValueFetcher;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.mapper.TimeSeriesParams;
@ -67,6 +72,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
/** A {@link FieldMapper} for scaled floats. Values are internally multiplied
* by a scaling factor and rounded to the closest long. */
public class ScaledFloatFieldMapper extends FieldMapper {
@ -125,12 +132,34 @@ public class ScaledFloatFieldMapper extends FieldMapper {
private final Parameter<TimeSeriesParams.MetricType> metric;
private final IndexMode indexMode;
private final IndexVersion indexCreatedVersion;
private final SourceKeepMode indexSourceKeepMode;
public Builder(String name, Settings settings, IndexMode indexMode) {
this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode);
public Builder(
String name,
Settings settings,
IndexMode indexMode,
IndexVersion indexCreatedVersion,
SourceKeepMode indexSourceKeepMode
) {
this(
name,
IGNORE_MALFORMED_SETTING.get(settings),
COERCE_SETTING.get(settings),
indexMode,
indexCreatedVersion,
indexSourceKeepMode
);
}
public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) {
public Builder(
String name,
boolean ignoreMalformedByDefault,
boolean coerceByDefault,
IndexMode indexMode,
IndexVersion indexCreatedVersion,
SourceKeepMode indexSourceKeepMode
) {
super(name);
this.ignoreMalformed = Parameter.explicitBoolParam(
"ignore_malformed",
@ -159,6 +188,8 @@ public class ScaledFloatFieldMapper extends FieldMapper {
);
}
});
this.indexCreatedVersion = indexCreatedVersion;
this.indexSourceKeepMode = indexSourceKeepMode;
}
Builder scalingFactor(double scalingFactor) {
@ -200,11 +231,35 @@ public class ScaledFloatFieldMapper extends FieldMapper {
coerce.getValue().value(),
context.isSourceSynthetic()
);
return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
String offsetsFieldName = getOffsetsFieldName(
context,
indexSourceKeepMode,
hasDocValues.getValue(),
stored.getValue(),
this,
indexCreatedVersion,
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT
);
return new ScaledFloatFieldMapper(
leafName(),
type,
builderParams(this, context),
context.isSourceSynthetic(),
this,
offsetsFieldName
);
}
}
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
public static final TypeParser PARSER = new TypeParser(
(n, c) -> new Builder(
n,
c.getSettings(),
c.getIndexSettings().getMode(),
c.indexVersionCreated(),
c.getIndexSettings().sourceKeepMode()
)
);
public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
@ -533,12 +588,17 @@ public class ScaledFloatFieldMapper extends FieldMapper {
private final TimeSeriesParams.MetricType metricType;
private final IndexMode indexMode;
private final IndexVersion indexCreatedVersion;
private final String offsetsFieldName;
private final SourceKeepMode indexSourceKeepMode;
private ScaledFloatFieldMapper(
String simpleName,
ScaledFloatFieldType mappedFieldType,
BuilderParams builderParams,
boolean isSourceSynthetic,
Builder builder
Builder builder,
String offsetsFieldName
) {
super(simpleName, mappedFieldType, builderParams);
this.isSourceSynthetic = isSourceSynthetic;
@ -553,6 +613,9 @@ public class ScaledFloatFieldMapper extends FieldMapper {
this.coerceByDefault = builder.coerce.getDefaultValue().value();
this.metricType = builder.metric.getValue();
this.indexMode = builder.indexMode;
this.indexCreatedVersion = builder.indexCreatedVersion;
this.offsetsFieldName = offsetsFieldName;
this.indexSourceKeepMode = builder.indexSourceKeepMode;
}
boolean coerce() {
@ -564,6 +627,11 @@ public class ScaledFloatFieldMapper extends FieldMapper {
return ignoreMalformed.value();
}
@Override
public String getOffsetFieldName() {
return offsetsFieldName;
}
@Override
public ScaledFloatFieldType fieldType() {
return (ScaledFloatFieldType) super.fieldType();
@ -576,7 +644,9 @@ public class ScaledFloatFieldMapper extends FieldMapper {
@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this);
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode)
.metric(metricType)
.init(this);
}
@Override
@ -606,11 +676,16 @@ public class ScaledFloatFieldMapper extends FieldMapper {
value = numericValue;
}
boolean shouldStoreOffsets = offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField();
if (value == null) {
value = nullValue;
}
if (value == null) {
if (shouldStoreOffsets) {
context.getOffSetContext().recordNull(offsetsFieldName);
}
return;
}
@ -636,6 +711,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored);
if (shouldStoreOffsets) {
context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue);
}
if (hasDocValues == false && (indexed || stored)) {
context.addToFieldNames(fieldType().name());
}
@ -777,17 +856,34 @@ public class ScaledFloatFieldMapper extends FieldMapper {
}
}
private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
if (offsetsFieldName != null) {
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
layers.add(
new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
fullPath(),
offsetsFieldName,
(b, value) -> b.value(decodeForSyntheticSource(value, scalingFactor))
)
);
if (ignoreMalformed.value()) {
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
}
return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
} else {
return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(decodeForSyntheticSource(value, scalingFactor));
}
};
}
}
@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
if (hasDocValues) {
return new SyntheticSourceSupport.Native(
() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(decodeForSyntheticSource(value, scalingFactor));
}
}
);
return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
}
return super.syntheticSourceSupport();

19
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
View file

@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentParsingException;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.NumberFieldMapperTests;
@ -365,6 +366,24 @@ public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests {
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed);
}
@Override
protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed) {
@Override
public SyntheticSourceExample example(int maxVals) {
var example = super.example(maxVals);
// Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
// uses the inputValue as both the input and expected.
return new SyntheticSourceExample(
example.expectedForSyntheticSource(),
example.expectedForSyntheticSource(),
example.expectedForBlockLoader(),
example.mapping()
);
}
};
}
private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
private final boolean ignoreMalformedEnabled;
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);

4
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
View file

@ -222,14 +222,14 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase {
}
public void testFetchSourceValue() throws IOException {
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
.build(MapperBuilderContext.root(false, false))
.fieldType();
assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
assertEquals(List.of(), fetchSourceValue(mapper, ""));
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
.nullValue(2.71)
.build(MapperBuilderContext.root(false, false))
.fieldType();

49
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.index.mapper.extras;
import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Collection;
import static java.util.Collections.singletonList;
public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
private static final double TEST_SCALING_FACTOR = 10.0;
@Override
protected Collection<? extends Plugin> getPlugins() {
return singletonList(new MapperExtrasPlugin());
}
@Override
protected void minimalMapping(XContentBuilder b) throws IOException {
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
}
public void testOffsetArray() throws Exception {
verifyOffsets("{\"field\":[1.0,10.0,100.0,0.1,10.0,1.0,0.1,100.0]}");
verifyOffsets("{\"field\":[10.0,null,1.0,null,5.0,null,null,6.3,1.5]}");
}
@Override
protected String getFieldTypeName() {
fail("Should not be called because minimalMapping is overridden");
return null;
}
@Override
protected Double randomValue() {
return randomLong() / TEST_SCALING_FACTOR;
}
}

51
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.index.mapper.extras;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Collection;
import static java.util.Collections.singletonList;
public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
private static final double TEST_SCALING_FACTOR = 10.0;
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return singletonList(MapperExtrasPlugin.class);
}
@Override
protected void minimalMapping(XContentBuilder b) throws IOException {
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
}
@Override
protected String getFieldTypeName() {
fail("Should not be called because minimalMapping is overridden");
return null;
}
@Override
protected Object getRandomValue() {
return randomLong() / TEST_SCALING_FACTOR;
}
@Override
protected Object getMalformedValue() {
return randomBoolean() ? RandomStrings.randomAsciiOfLength(random(), 8) : Double.POSITIVE_INFINITY;
}
}

1
server/src/main/java/org/elasticsearch/index/IndexVersions.java
View file

@ -157,6 +157,7 @@ public class IndexVersions {
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS = def(9_018_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG = def(9_019_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0);
/*
* STOP! READ THIS FIRST! No, really,
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

77
test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java
View file

@ -35,6 +35,7 @@ import java.util.Set;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase {
@ -123,14 +124,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
inputDocuments.add(inputDocument);
}
var mapping = jsonBuilder().startObject()
.startObject("properties")
.startObject("field")
.field("type", getFieldTypeName())
.field("ignore_malformed", true)
.endObject()
.endObject()
.endObject();
var mapping = jsonBuilder().startObject().startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.field("ignore_malformed", true).endObject().endObject().endObject();
var indexService = createIndex(
"test-index",
Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(),
@ -177,13 +173,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
.startObject("parent")
.field("type", "nested")
.startObject("properties")
.startObject("field")
.field("type", getFieldTypeName())
.endObject()
.endObject()
.endObject()
.endObject()
.endObject();
.startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject().endObject().endObject();
var indexService = createIndex(
"test-index",
@ -241,6 +233,12 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
}
}
protected void minimalMapping(XContentBuilder b) throws IOException {
String fieldTypeName = getFieldTypeName();
assertThat(fieldTypeName, notNullValue());
b.field("type", fieldTypeName);
}
protected abstract String getFieldTypeName();
protected abstract Object getRandomValue();
@ -248,13 +246,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
protected abstract Object getMalformedValue();
protected void verifySyntheticArray(Object[][] arrays) throws IOException {
var mapping = jsonBuilder().startObject()
.startObject("properties")
.startObject("field")
.field("type", getFieldTypeName())
.endObject()
.endObject()
.endObject();
var mapping = jsonBuilder().startObject().startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject();
verifySyntheticArray(arrays, mapping, "_id");
}
@ -325,20 +319,17 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
}
protected void verifySyntheticObjectArray(List<List<Object[]>> documents) throws IOException {
var mapping = jsonBuilder().startObject()
.startObject("properties")
.startObject("object")
.startObject("properties")
.startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject().endObject().endObject();
var indexService = createIndex(
"test-index",
Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(),
jsonBuilder().startObject()
.startObject("properties")
.startObject("object")
.startObject("properties")
.startObject("field")
.field("type", getFieldTypeName())
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
mapping
);
for (int i = 0; i < documents.size(); i++) {
var document = documents.get(i);
@ -393,20 +384,18 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
}
protected void verifySyntheticArrayInObject(List<Object[]> documents) throws IOException {
var mapping = jsonBuilder().startObject()
.startObject("properties")
.startObject("object")
.startObject("properties")
.startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject().endObject().endObject();
var indexService = createIndex(
"test-index",
Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(),
jsonBuilder().startObject()
.startObject("properties")
.startObject("object")
.startObject("properties")
.startObject("field")
.field("type", getFieldTypeName())
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
mapping
);
for (int i = 0; i < documents.size(); i++) {
var arrayValue = documents.get(i);

124
test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java
View file

@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.HashSet;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCase {
@ -33,18 +34,10 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
}
public void testOffsetArrayNoDocValues() throws Exception {
String mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}",
"doc_values": false
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.field("doc_values", false);
mapping.endObject().endObject().endObject().endObject();
try (var mapperService = createMapperService(mapping)) {
var fieldMapper = mapperService.mappingLookup().getMapper("field");
assertThat(fieldMapper.getOffsetFieldName(), nullValue());
@ -52,19 +45,10 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
}
public void testOffsetArrayStored() throws Exception {
String mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}",
"store": true
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
;
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.field("store", true);
mapping.endObject().endObject().endObject().endObject();
try (var mapperService = createMapperService(mapping)) {
var fieldMapper = mapperService.mappingLookup().getMapper("field");
assertThat(fieldMapper.getOffsetFieldName(), nullValue());
@ -72,22 +56,10 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
}
public void testOffsetMultiFields() throws Exception {
String mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}",
"fields": {
"sub": {
"type": "text"
}
}
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.startObject("fields").startObject("sub").field("type", "text").endObject().endObject();
mapping.endObject().endObject().endObject().endObject();
try (var mapperService = createMapperService(mapping)) {
var fieldMapper = mapperService.mappingLookup().getMapper("field");
assertThat(fieldMapper.getOffsetFieldName(), nullValue());
@ -95,17 +67,9 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
}
public void testOffsetArrayNoSyntheticSource() throws Exception {
String mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}"
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject().endObject();
try (var mapperService = createMapperService(Settings.EMPTY, mapping)) {
var fieldMapper = mapperService.mappingLookup().getMapper("field");
assertThat(fieldMapper.getOffsetFieldName(), nullValue());
@ -114,36 +78,14 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
public void testOffsetArrayNoSourceArrayKeep() throws Exception {
var settingsBuilder = Settings.builder().put("index.mapping.source.mode", "synthetic");
String mapping;
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
if (randomBoolean()) {
mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}",
"synthetic_source_keep": "{{synthetic_source_keep}}"
}
}
}
}
""".replace("{{synthetic_source_keep}}", randomBoolean() ? "none" : "all").replace("{{type}}", getFieldTypeName());
} else {
mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}"
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
if (randomBoolean()) {
settingsBuilder.put("index.mapping.synthetic_source_keep", "none");
}
mapping.field("synthetic_source_keep", randomBoolean() ? "none" : "all");
} else if (randomBoolean()) {
settingsBuilder.put("index.mapping.synthetic_source_keep", "none");
}
mapping.endObject().endObject().endObject().endObject();
try (var mapperService = createMapperService(settingsBuilder.build(), mapping)) {
var fieldMapper = mapperService.mappingLookup().getMapper("field");
assertThat(fieldMapper.getOffsetFieldName(), nullValue());
@ -183,6 +125,12 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
verifyOffsets("{\"field\":" + values + "}");
}
protected void minimalMapping(XContentBuilder b) throws IOException {
String fieldTypeName = getFieldTypeName();
assertThat(fieldTypeName, notNullValue());
b.field("type", fieldTypeName);
}
protected abstract String getFieldTypeName();
protected abstract Object randomValue();
@ -192,21 +140,13 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
}
protected void verifyOffsets(String source, String expectedSource) throws IOException {
String mapping = """
{
"_doc": {
"properties": {
"field": {
"type": "{{type}}"
}
}
}
}
""".replace("{{type}}", getFieldTypeName());
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field");
minimalMapping(mapping);
mapping.endObject().endObject().endObject().endObject();
verifyOffsets(mapping, source, expectedSource);
}
private void verifyOffsets(String mapping, String source, String expectedSource) throws IOException {
private void verifyOffsets(XContentBuilder mapping, String source, String expectedSource) throws IOException {
try (var mapperService = createMapperService(mapping)) {
var mapper = mapperService.documentMapper();