From 0c1b3acee23c06a8aa3d08075672b13ef857037f Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 30 Apr 2025 09:33:35 -0700 Subject: [PATCH] Properly handle multi fields in block loaders with synthetic source enabled (#127483) --- .../mapper/extras/ScaledFloatFieldMapper.java | 3 +- .../index/mapper/BooleanFieldMapper.java | 3 +- .../index/mapper/DateFieldMapper.java | 3 +- .../index/mapper/GeoPointFieldMapper.java | 12 +- .../index/mapper/IpFieldMapper.java | 3 +- .../index/mapper/KeywordFieldMapper.java | 3 +- .../index/mapper/NumberFieldMapper.java | 3 +- .../GeoPointFieldBlockLoaderTests.java | 58 ++-- .../TextFieldWithParentBlockLoaderTests.java | 108 +++++-- .../datageneration/MappingGenerator.java | 1 + .../datasource/DataSourceRequest.java | 1 + .../DefaultMappingParametersHandler.java | 156 ++++----- .../index/mapper/BlockLoaderTestCase.java | 299 ++++++++---------- .../index/mapper/BlockLoaderTestRunner.java | 148 +++++++++ ...gateMetricDoubleFieldBlockLoaderTests.java | 6 + .../ConstantKeywordFieldBlockLoaderTests.java | 6 + .../unsignedlong/UnsignedLongFieldMapper.java | 3 +- .../GeoShapeWithDocValuesFieldMapper.java | 3 +- .../index/mapper/PointFieldMapper.java | 3 +- .../index/mapper/ShapeFieldMapper.java | 3 +- .../mapper/GeoShapeFieldBlockLoaderTests.java | 6 + .../mapper/PointFieldBlockLoaderTests.java | 6 + .../mapper/ShapeFieldBlockLoaderTests.java | 6 + 23 files changed, 538 insertions(+), 305 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 9e7940201c4a..5bf5295e7add 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -378,7 +378,8 @@ public class ScaledFloatFieldMapper extends FieldMapper { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l / scalingFactor); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 601395025592..2843900c564c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -350,7 +350,8 @@ public class BooleanFieldMapper extends FieldMapper { return new BlockDocValuesReader.BooleansBlockLoader(name()); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7a1fbd538347..3511c8dc1932 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -948,7 +948,8 @@ public final class DateFieldMapper extends FieldMapper { return new BlockDocValuesReader.LongsBlockLoader(name()); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index dce49367bd56..764f0c4cc503 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -316,7 +316,7 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper convert(s, null); + case String s -> convert(s, null, false); case null -> null; default -> throw new IllegalStateException("Unexpected null_value format"); }; if (params.preference() == MappedFieldType.FieldExtractPreference.DOC_VALUES && hasDocValues(fieldMapping, true)) { if (values instanceof List == false) { - var point = convert(values, nullValue); + var point = convert(values, nullValue, testContext.isMultifield()); return point != null ? point.getEncoded() : null; } var resultList = ((List) values).stream() - .map(v -> convert(v, nullValue)) + .map(v -> convert(v, nullValue, testContext.isMultifield())) .filter(Objects::nonNull) .map(GeoPoint::getEncoded) .sorted() @@ -55,8 +55,9 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { return maybeFoldList(resultList); } + // stored source is used if (params.syntheticSource() == false) { - return exactValuesFromSource(values, nullValue); + return exactValuesFromSource(values, nullValue, false); } // Usually implementation of block loader from source adjusts values read from source @@ -67,25 +68,25 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { // That is unless "synthetic_source_keep" forces fallback synthetic source again. if (testContext.forceFallbackSyntheticSource()) { - return exactValuesFromSource(values, nullValue); + return exactValuesFromSource(values, nullValue, false); } String syntheticSourceKeep = (String) fieldMapping.getOrDefault("synthetic_source_keep", "none"); if (syntheticSourceKeep.equals("all")) { - return exactValuesFromSource(values, nullValue); + return exactValuesFromSource(values, nullValue, false); } if (syntheticSourceKeep.equals("arrays") && extractedFieldValues.documentHasObjectArrays()) { - return exactValuesFromSource(values, nullValue); + return exactValuesFromSource(values, nullValue, false); } // synthetic source and doc_values are present if (hasDocValues(fieldMapping, true)) { if (values instanceof List == false) { - return toWKB(normalize(convert(values, nullValue))); + return toWKB(normalize(convert(values, nullValue, false))); } var resultList = ((List) values).stream() - .map(v -> convert(v, nullValue)) + .map(v -> convert(v, nullValue, false)) .filter(Objects::nonNull) .sorted(Comparator.comparingLong(GeoPoint::getEncoded)) .map(p -> toWKB(normalize(p))) @@ -94,16 +95,20 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { } // synthetic source but no doc_values so using fallback synthetic source - return exactValuesFromSource(values, nullValue); + return exactValuesFromSource(values, nullValue, false); } @SuppressWarnings("unchecked") - private Object exactValuesFromSource(Object value, GeoPoint nullValue) { + private Object exactValuesFromSource(Object value, GeoPoint nullValue, boolean needsMultifieldAdjustment) { if (value instanceof List == false) { - return toWKB(convert(value, nullValue)); + return toWKB(convert(value, nullValue, needsMultifieldAdjustment)); } - var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).map(this::toWKB).toList(); + var resultList = ((List) value).stream() + .map(v -> convert(v, nullValue, needsMultifieldAdjustment)) + .filter(Objects::nonNull) + .map(this::toWKB) + .toList(); return maybeFoldList(resultList); } @@ -163,14 +168,17 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { } @SuppressWarnings("unchecked") - private GeoPoint convert(Object value, GeoPoint nullValue) { + private GeoPoint convert(Object value, GeoPoint nullValue, boolean needsMultifieldAdjustment) { if (value == null) { - return nullValue; + if (nullValue == null) { + return null; + } + return possiblyAdjustMultifieldValue(nullValue, needsMultifieldAdjustment); } if (value instanceof String s) { try { - return new GeoPoint(s); + return possiblyAdjustMultifieldValue(new GeoPoint(s), needsMultifieldAdjustment); } catch (Exception e) { return null; } @@ -180,9 +188,9 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { if (m.get("type") != null) { var coordinates = (List) m.get("coordinates"); // Order is GeoJSON is lon,lat - return new GeoPoint(coordinates.get(1), coordinates.get(0)); + return possiblyAdjustMultifieldValue(new GeoPoint(coordinates.get(1), coordinates.get(0)), needsMultifieldAdjustment); } else { - return new GeoPoint((Double) m.get("lat"), (Double) m.get("lon")); + return possiblyAdjustMultifieldValue(new GeoPoint((Double) m.get("lat"), (Double) m.get("lon")), needsMultifieldAdjustment); } } @@ -190,6 +198,20 @@ public class GeoPointFieldBlockLoaderTests extends BlockLoaderTestCase { return null; } + private GeoPoint possiblyAdjustMultifieldValue(GeoPoint point, boolean isMultifield) { + // geo_point multifields are parsed from a geohash representation of the original point (GeoPointFieldMapper#index) + // and it's not exact. + // So if this is a multifield we need another adjustment here. + // Note that this does not apply to block loader from source because in this case we parse raw original values. + // Same thing happens with synthetic source since it is generated from the parent field data that didn't go through multi field + // parsing logic. + if (isMultifield) { + return point.resetFromString(point.geohash()); + } + + return point; + } + private GeoPoint normalize(GeoPoint point) { if (point == null) { return null; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java index 1f154fa7581a..6343aeea2d9d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java @@ -9,51 +9,108 @@ package org.elasticsearch.index.mapper.blockloader; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.datageneration.DocumentGenerator; import org.elasticsearch.datageneration.FieldType; +import org.elasticsearch.datageneration.MappingGenerator; +import org.elasticsearch.datageneration.Template; import org.elasticsearch.datageneration.datasource.DataSourceHandler; import org.elasticsearch.datageneration.datasource.DataSourceRequest; import org.elasticsearch.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.datageneration.datasource.DefaultMappingParametersHandler; import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.index.mapper.BlockLoaderTestRunner; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; -import java.util.HashMap; +import java.io.IOException; import java.util.List; import java.util.Map; -public class TextFieldWithParentBlockLoaderTests extends BlockLoaderTestCase { - public TextFieldWithParentBlockLoaderTests(Params params) { - // keyword because we need a keyword parent field - super(FieldType.KEYWORD.toString(), List.of(new DataSourceHandler() { +import static org.elasticsearch.index.mapper.BlockLoaderTestCase.buildSpecification; +import static org.elasticsearch.index.mapper.BlockLoaderTestCase.hasDocValues; + +public class TextFieldWithParentBlockLoaderTests extends MapperServiceTestCase { + private final BlockLoaderTestCase.Params params; + private final BlockLoaderTestRunner runner; + + @ParametersFactory(argumentFormatting = "preference=%s") + public static List args() { + return BlockLoaderTestCase.args(); + } + + public TextFieldWithParentBlockLoaderTests(BlockLoaderTestCase.Params params) { + this.params = params; + this.runner = new BlockLoaderTestRunner(params); + } + + // This is similar to BlockLoaderTestCase#testBlockLoaderOfMultiField but has customizations required to properly test the case + // of text multi field in a keyword field. + public void testBlockLoaderOfParentField() throws IOException { + var template = new Template(Map.of("parent", new Template.Leaf("parent", FieldType.KEYWORD.toString()))); + var specification = buildSpecification(List.of(new DataSourceHandler() { @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { - assert request.fieldType().equals(FieldType.KEYWORD.toString()); + // This is a bit tricky meta-logic. + // We want to customize mapping but to do this we need the mapping for the same field type + // so we use name to untangle this. + if (request.fieldName().equals("parent") == false) { + return null; + } - // We need to force multi field generation return new DataSourceResponse.LeafMappingParametersGenerator(() -> { - var defaultSupplier = DefaultMappingParametersHandler.keywordMapping( - request, - DefaultMappingParametersHandler.commonMappingParameters() - ); - var mapping = defaultSupplier.get(); - // we don't need this here - mapping.remove("copy_to"); + var dataSource = request.dataSource(); + + var keywordParentMapping = dataSource.get( + new DataSourceRequest.LeafMappingParametersGenerator( + dataSource, + "_field", + FieldType.KEYWORD.toString(), + request.eligibleCopyToFields(), + request.dynamicMapping() + ) + ).mappingGenerator().get(); + + var textMultiFieldMapping = dataSource.get( + new DataSourceRequest.LeafMappingParametersGenerator( + dataSource, + "_field", + FieldType.TEXT.toString(), + request.eligibleCopyToFields(), + request.dynamicMapping() + ) + ).mappingGenerator().get(); + + // we don't need this here + keywordParentMapping.remove("copy_to"); - var textMultiFieldMappingSupplier = DefaultMappingParametersHandler.textMapping(request, new HashMap<>()); - var textMultiFieldMapping = textMultiFieldMappingSupplier.get(); textMultiFieldMapping.put("type", "text"); textMultiFieldMapping.remove("fields"); - mapping.put("fields", Map.of("txt", textMultiFieldMapping)); + keywordParentMapping.put("fields", Map.of("mf", textMultiFieldMapping)); - return mapping; + return keywordParentMapping; }); } - }), params); + })); + var mapping = new MappingGenerator(specification).generate(template); + var fieldMapping = mapping.lookup().get("parent"); + + var document = new DocumentGenerator(specification).generate(template, mapping); + var fieldValue = document.get("parent"); + + Object expected = expected(fieldMapping, fieldValue, new BlockLoaderTestCase.TestContext(false, true)); + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); + var mapperService = params.syntheticSource() + ? createSytheticSourceMapperService(mappingXContent) + : createMapperService(mappingXContent); + + runner.runTest(mapperService, document, expected, "parent.mf"); } - @Override @SuppressWarnings("unchecked") - protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + private Object expected(Map fieldMapping, Object value, BlockLoaderTestCase.TestContext testContext) { assert fieldMapping.containsKey("fields"); Object normalizer = fieldMapping.get("normalizer"); @@ -66,12 +123,7 @@ public class TextFieldWithParentBlockLoaderTests extends BlockLoaderTestCase { } // we are using block loader of the text field itself - var textFieldMapping = (Map) ((Map) fieldMapping.get("fields")).get("txt"); + var textFieldMapping = (Map) ((Map) fieldMapping.get("fields")).get("mf"); return TextFieldBlockLoaderTests.expectedValue(textFieldMapping, value, params, testContext); } - - @Override - protected String blockLoaderFieldName(String originalName) { - return originalName + ".txt"; - } } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/MappingGenerator.java index 4c8e835f6170..795302e0972c 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/MappingGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/MappingGenerator.java @@ -104,6 +104,7 @@ public class MappingGenerator { var mappingParametersGenerator = specification.dataSource() .get( new DataSourceRequest.LeafMappingParametersGenerator( + specification.dataSource(), fieldName, leaf.type(), context.eligibleCopyToDestinations(), diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DataSourceRequest.java index e5e845eb4858..1323fa23d226 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DataSourceRequest.java @@ -199,6 +199,7 @@ public interface DataSourceRequest { } record LeafMappingParametersGenerator( + DataSource dataSource, String fieldName, String fieldType, Set eligibleCopyToFields, diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java index de3e49e8457e..2e234f8aec41 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java @@ -36,34 +36,27 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { return null; } - var map = commonMappingParameters(); - if (ESTestCase.randomBoolean()) { - map.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); - } - return new DataSourceResponse.LeafMappingParametersGenerator(switch (fieldType) { - case KEYWORD -> keywordMapping(request, map); - case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, fieldType); - case SCALED_FLOAT -> scaledFloatMapping(map); - case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); - case BOOLEAN -> booleanMapping(map); - case DATE -> dateMapping(map); - case GEO_POINT -> geoPointMapping(map); - case TEXT -> textMapping(request, new HashMap<>()); - case IP -> ipMapping(map); - case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>()); - case WILDCARD -> wildcardMapping(new HashMap<>()); + case KEYWORD -> keywordMapping(request); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(fieldType); + case SCALED_FLOAT -> scaledFloatMapping(); + case COUNTED_KEYWORD -> countedKeywordMapping(); + case BOOLEAN -> booleanMapping(); + case DATE -> dateMapping(); + case GEO_POINT -> geoPointMapping(); + case TEXT -> textMapping(request); + case IP -> ipMapping(); + case CONSTANT_KEYWORD -> constantKeywordMapping(); + case WILDCARD -> wildcardMapping(); }); } - private Supplier> plain(Map injected) { - return () -> injected; - } - - private Supplier> numberMapping(Map injected, FieldType fieldType) { + private Supplier> numberMapping(FieldType fieldType) { return () -> { + var mapping = commonMappingParameters(); + if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } if (ESTestCase.randomDouble() <= 0.2) { Number value = switch (fieldType) { @@ -77,18 +70,17 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { default -> throw new IllegalStateException("Unexpected field type"); }; - injected.put("null_value", value); + mapping.put("null_value", value); } - return injected; + return mapping; }; } - public static Supplier> keywordMapping( - DataSourceRequest.LeafMappingParametersGenerator request, - Map injected - ) { + private Supplier> keywordMapping(DataSourceRequest.LeafMappingParametersGenerator request) { return () -> { + var mapping = commonMappingParameters(); + // Inject copy_to sometimes but reflect that it is not widely used in reality. // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields. // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail. @@ -100,69 +92,79 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { .collect(Collectors.toSet()); if (options.isEmpty() == false) { - injected.put("copy_to", ESTestCase.randomFrom(options)); + mapping.put("copy_to", ESTestCase.randomFrom(options)); } } if (ESTestCase.randomDouble() <= 0.2) { - injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); + mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); } if (ESTestCase.randomDouble() <= 0.2) { - injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); + mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); } - return injected; + return mapping; }; } - private Supplier> scaledFloatMapping(Map injected) { + private Supplier> scaledFloatMapping() { return () -> { - injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5)); + var mapping = commonMappingParameters(); + + mapping.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5)); if (ESTestCase.randomDouble() <= 0.2) { - injected.put("null_value", ESTestCase.randomDouble()); + mapping.put("null_value", ESTestCase.randomDouble()); } if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } - return injected; + return mapping; }; } - private Supplier> booleanMapping(Map injected) { + private Supplier> countedKeywordMapping() { + return () -> Map.of("index", ESTestCase.randomBoolean()); + } + + private Supplier> booleanMapping() { return () -> { + var mapping = commonMappingParameters(); + if (ESTestCase.randomDouble() <= 0.2) { - injected.put("null_value", ESTestCase.randomFrom(true, false, "true", "false")); + mapping.put("null_value", ESTestCase.randomFrom(true, false, "true", "false")); } if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } - return injected; + return mapping; }; } // just a custom format, specific format does not matter private static final String FORMAT = "yyyy_MM_dd_HH_mm_ss_n"; - private Supplier> dateMapping(Map injected) { + private Supplier> dateMapping() { return () -> { + var mapping = commonMappingParameters(); + String format = null; if (ESTestCase.randomBoolean()) { format = FORMAT; - injected.put("format", format); + mapping.put("format", format); } if (ESTestCase.randomDouble() <= 0.2) { var instant = ESTestCase.randomInstantBetween(Instant.parse("2300-01-01T00:00:00Z"), Instant.parse("2350-01-01T00:00:00Z")); if (format == null) { - injected.put("null_value", instant.toEpochMilli()); + mapping.put("null_value", instant.toEpochMilli()); } else { - injected.put( + mapping.put( "null_value", DateTimeFormatter.ofPattern(format, Locale.ROOT).withZone(ZoneId.from(ZoneOffset.UTC)).format(instant) ); @@ -170,82 +172,89 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { } if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } - return injected; + return mapping; }; } - private Supplier> geoPointMapping(Map injected) { + private Supplier> geoPointMapping() { return () -> { + var mapping = commonMappingParameters(); + if (ESTestCase.randomDouble() <= 0.2) { var point = GeometryTestUtils.randomPoint(false); - injected.put("null_value", WellKnownText.toWKT(point)); + mapping.put("null_value", WellKnownText.toWKT(point)); } if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } - return injected; + return mapping; }; } - public static Supplier> textMapping( - DataSourceRequest.LeafMappingParametersGenerator request, - Map injected - ) { + private Supplier> textMapping(DataSourceRequest.LeafMappingParametersGenerator request) { return () -> { - injected.put("store", ESTestCase.randomBoolean()); - injected.put("index", ESTestCase.randomBoolean()); + var mapping = new HashMap(); + + mapping.put("store", ESTestCase.randomBoolean()); + mapping.put("index", ESTestCase.randomBoolean()); if (ESTestCase.randomDouble() <= 0.1) { - var keywordMultiFieldMapping = keywordMapping(request, commonMappingParameters()).get(); + var keywordMultiFieldMapping = keywordMapping(request).get(); keywordMultiFieldMapping.put("type", "keyword"); keywordMultiFieldMapping.remove("copy_to"); - injected.put("fields", Map.of("kwd", keywordMultiFieldMapping)); + mapping.put("fields", Map.of("kwd", keywordMultiFieldMapping)); } - return injected; + return mapping; }; } - private Supplier> ipMapping(Map injected) { + private Supplier> ipMapping() { return () -> { + var mapping = commonMappingParameters(); + if (ESTestCase.randomDouble() <= 0.2) { - injected.put("null_value", NetworkAddress.format(ESTestCase.randomIp(ESTestCase.randomBoolean()))); + mapping.put("null_value", NetworkAddress.format(ESTestCase.randomIp(ESTestCase.randomBoolean()))); } if (ESTestCase.randomBoolean()) { - injected.put("ignore_malformed", ESTestCase.randomBoolean()); + mapping.put("ignore_malformed", ESTestCase.randomBoolean()); } - return injected; + return mapping; }; } - private Supplier> constantKeywordMapping(Map injected) { + private Supplier> constantKeywordMapping() { return () -> { + var mapping = new HashMap(); + // value is optional and can be set from the first document // we don't cover this case here - injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); + mapping.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); - return injected; + return mapping; }; } - private Supplier> wildcardMapping(Map injected) { + private Supplier> wildcardMapping() { return () -> { + var mapping = new HashMap(); + if (ESTestCase.randomDouble() <= 0.2) { - injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); + mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); } if (ESTestCase.randomDouble() <= 0.2) { - injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); + mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); } - return injected; + return mapping; }; } @@ -254,6 +263,11 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { map.put("store", ESTestCase.randomBoolean()); map.put("index", ESTestCase.randomBoolean()); map.put("doc_values", ESTestCase.randomBoolean()); + + if (ESTestCase.randomBoolean()) { + map.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); + } + return map; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index 965c6dd4dd92..3b6c4ec5e012 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -11,24 +11,13 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.RandomIndexWriter; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.datageneration.DataGeneratorSpecification; import org.elasticsearch.datageneration.DocumentGenerator; -import org.elasticsearch.datageneration.Mapping; import org.elasticsearch.datageneration.MappingGenerator; import org.elasticsearch.datageneration.Template; import org.elasticsearch.datageneration.datasource.DataSourceHandler; import org.elasticsearch.datageneration.datasource.DataSourceRequest; import org.elasticsearch.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; -import org.elasticsearch.search.fetch.StoredFieldsSpec; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -38,7 +27,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Stream; public abstract class BlockLoaderTestCase extends MapperServiceTestCase { @@ -60,46 +48,26 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { public record Params(boolean syntheticSource, MappedFieldType.FieldExtractPreference preference) {} - public record TestContext(boolean forceFallbackSyntheticSource) {} + public record TestContext(boolean forceFallbackSyntheticSource, boolean isMultifield) {} private final String fieldType; protected final Params params; + private final Collection customDataSourceHandlers; + private final BlockLoaderTestRunner runner; private final String fieldName; - private final MappingGenerator mappingGenerator; - private final DocumentGenerator documentGenerator; protected BlockLoaderTestCase(String fieldType, Params params) { this(fieldType, List.of(), params); } - protected BlockLoaderTestCase(String fieldType, Collection customHandlers, Params params) { + protected BlockLoaderTestCase(String fieldType, Collection customDataSourceHandlers, Params params) { this.fieldType = fieldType; this.params = params; + this.customDataSourceHandlers = customDataSourceHandlers; + this.runner = new BlockLoaderTestRunner(params); this.fieldName = randomAlphaOfLengthBetween(5, 10); - - var specification = DataGeneratorSpecification.builder() - .withFullyDynamicMapping(false) - // Disable dynamic mapping and disabled objects - .withDataSourceHandlers(List.of(new DataSourceHandler() { - @Override - public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) { - return new DataSourceResponse.DynamicMappingGenerator(isObject -> false); - } - - @Override - public DataSourceResponse.ObjectMappingParametersGenerator handle( - DataSourceRequest.ObjectMappingParametersGenerator request - ) { - return new DataSourceResponse.ObjectMappingParametersGenerator(HashMap::new); // just defaults - } - })) - .withDataSourceHandlers(customHandlers) - .build(); - - this.mappingGenerator = new MappingGenerator(specification); - this.documentGenerator = new DocumentGenerator(specification); } @Override @@ -114,9 +82,19 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { public void testBlockLoader() throws IOException { var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType))); - var mapping = mappingGenerator.generate(template); + var specification = buildSpecification(customDataSourceHandlers); - runTest(template, mapping, fieldName, new TestContext(false)); + var mapping = new MappingGenerator(specification).generate(template); + var document = new DocumentGenerator(specification).generate(template, mapping); + + Object expected = expected(mapping.lookup().get(fieldName), getFieldValue(document, fieldName), new TestContext(false, false)); + + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); + var mapperService = params.syntheticSource + ? createSytheticSourceMapperService(mappingXContent) + : createMapperService(mappingXContent); + + runner.runTest(mapperService, document, expected, fieldName); } @SuppressWarnings("unchecked") @@ -140,9 +118,11 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { currentLevel.put(fieldName, new Template.Leaf(fieldName, fieldType)); var template = new Template(top); - var mapping = mappingGenerator.generate(template); + var specification = buildSpecification(customDataSourceHandlers); + var mapping = new MappingGenerator(specification).generate(template); + var document = new DocumentGenerator(specification).generate(template, mapping); - TestContext testContext = new TestContext(false); + TestContext testContext = new TestContext(false, false); if (params.syntheticSource && randomBoolean()) { // force fallback synthetic source in the hierarchy @@ -150,29 +130,119 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { var topLevelMapping = (Map) ((Map) docMapping.get("properties")).get("top"); topLevelMapping.put("synthetic_source_keep", "all"); - testContext = new TestContext(true); + testContext = new TestContext(true, false); } - runTest(template, mapping, fullFieldName.toString(), testContext); - } - - private void runTest(Template template, Mapping mapping, String fieldName, TestContext testContext) throws IOException { var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); - var mapperService = params.syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); - var document = documentGenerator.generate(template, mapping); - var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(document); + Object expected = expected( + mapping.lookup().get(fullFieldName.toString()), + getFieldValue(document, fullFieldName.toString()), + testContext + ); - Object expected = expected(mapping.lookup().get(fieldName), getFieldValue(document, fieldName), testContext); - Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, documentXContent, blockLoaderFieldName(fieldName)); - assertEquals(expected, blockLoaderResult); + runner.runTest(mapperService, document, expected, fullFieldName.toString()); + } + + @SuppressWarnings("unchecked") + public void testBlockLoaderOfMultiField() throws IOException { + // We are going to have a parent field and a multi field of the same type in order to be sure we can index data. + // Then we'll test block loader of the multi field. + var template = new Template(Map.of("parent", new Template.Leaf("parent", fieldType))); + + var customHandlers = new ArrayList(); + customHandlers.add(new DataSourceHandler() { + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + // This is a bit tricky meta-logic. + // We want to customize mapping but to do this we need the mapping for the same field type + // so we use name to untangle this. + if (request.fieldName().equals("parent") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var dataSource = request.dataSource(); + + // We need parent field to have the same mapping as multi field due to different behavior caused f.e. by + // ignore_malformed. + // The name here should be different from "parent". + var mapping = dataSource.get( + new DataSourceRequest.LeafMappingParametersGenerator( + dataSource, + "_field", + request.fieldType(), + request.eligibleCopyToFields(), + request.dynamicMapping() + ) + ).mappingGenerator().get(); + + var parentMapping = new HashMap<>(mapping); + var multiFieldMapping = new HashMap<>(mapping); + + multiFieldMapping.put("type", fieldType); + multiFieldMapping.remove("fields"); + + parentMapping.put("fields", Map.of("mf", multiFieldMapping)); + + return parentMapping; + }); + } + }); + customHandlers.addAll(customDataSourceHandlers); + var specification = buildSpecification(customHandlers); + var mapping = new MappingGenerator(specification).generate(template); + var fieldMapping = (Map) ((Map) mapping.lookup().get("parent").get("fields")).get("mf"); + + var document = new DocumentGenerator(specification).generate(template, mapping); + + Object expected = expected(fieldMapping, getFieldValue(document, "parent"), new TestContext(false, true)); + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); + var mapperService = params.syntheticSource + ? createSytheticSourceMapperService(mappingXContent) + : createMapperService(mappingXContent); + + runner.runTest(mapperService, document, expected, "parent.mf"); + } + + public static DataGeneratorSpecification buildSpecification(Collection customHandlers) { + return DataGeneratorSpecification.builder() + .withFullyDynamicMapping(false) + // Disable dynamic mapping and disabled objects + .withDataSourceHandlers(List.of(new DataSourceHandler() { + @Override + public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) { + return new DataSourceResponse.DynamicMappingGenerator(isObject -> false); + } + + @Override + public DataSourceResponse.ObjectMappingParametersGenerator handle( + DataSourceRequest.ObjectMappingParametersGenerator request + ) { + return new DataSourceResponse.ObjectMappingParametersGenerator(HashMap::new); // just defaults + } + })) + .withDataSourceHandlers(customHandlers) + .build(); } protected abstract Object expected(Map fieldMapping, Object value, TestContext testContext); + protected static Object maybeFoldList(List list) { + if (list.isEmpty()) { + return null; + } + + if (list.size() == 1) { + return list.get(0); + } + + return list; + } + protected Object getFieldValue(Map document, String fieldName) { var rawValues = new ArrayList<>(); processLevel(document, fieldName, rawValues); @@ -204,128 +274,7 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { } } - protected static Object maybeFoldList(List list) { - if (list.isEmpty()) { - return null; - } - - if (list.size() == 1) { - return list.get(0); - } - - return list; - } - - /** - Allows to change the field name used to obtain a block loader. - Useful f.e. to test block loaders of multi fields. - */ - protected String blockLoaderFieldName(String originalName) { - return originalName; - } - - private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException { - try (Directory directory = newDirectory()) { - RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - - var source = new SourceToParse( - "1", - BytesReference.bytes(document), - XContentType.JSON, - null, - Map.of(), - true, - XContentMeteringParserDecorator.NOOP - ); - LuceneDocument doc = mapperService.documentMapper().parse(source).rootDoc(); - - iw.addDocument(doc); - iw.close(); - - try (DirectoryReader reader = DirectoryReader.open(directory)) { - LeafReaderContext context = reader.leaves().get(0); - return load(createBlockLoader(mapperService, fieldName), context, mapperService); - } - } - } - - private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperService mapperService) throws IOException { - // `columnAtATimeReader` is tried first, we mimic `ValuesSourceReaderOperator` - var columnAtATimeReader = blockLoader.columnAtATimeReader(context); - if (columnAtATimeReader != null) { - var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(context.reader().numDocs()), TestBlock.docs(0)); - if (block.size() == 0) { - return null; - } - return block.get(0); - } - - StoredFieldsSpec storedFieldsSpec = blockLoader.rowStrideStoredFieldSpec(); - SourceLoader.Leaf leafSourceLoader = null; - if (storedFieldsSpec.requiresSource()) { - var sourceLoader = mapperService.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); - leafSourceLoader = sourceLoader.leaf(context.reader(), null); - storedFieldsSpec = storedFieldsSpec.merge( - new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields()) - ); - } - BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(context, null), - leafSourceLoader - ); - storedFieldsLoader.advanceTo(0); - - BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(context.reader().numDocs()), 1); - blockLoader.rowStrideReader(context).read(0, storedFieldsLoader, builder); - var block = (TestBlock) builder.build(); - if (block.size() == 0) { - return null; - } - return block.get(0); - } - - private BlockLoader createBlockLoader(MapperService mapperService, String fieldName) { - SearchLookup searchLookup = new SearchLookup(mapperService.mappingLookup().fieldTypesLookup()::get, null, null); - - return mapperService.fieldType(fieldName).blockLoader(new MappedFieldType.BlockLoaderContext() { - @Override - public String indexName() { - return mapperService.getIndexSettings().getIndex().getName(); - } - - @Override - public IndexSettings indexSettings() { - return mapperService.getIndexSettings(); - } - - @Override - public MappedFieldType.FieldExtractPreference fieldExtractPreference() { - return params.preference; - } - - @Override - public SearchLookup lookup() { - return searchLookup; - } - - @Override - public Set sourcePaths(String name) { - return mapperService.mappingLookup().sourcePaths(name); - } - - @Override - public String parentField(String field) { - return mapperService.mappingLookup().parentField(field); - } - - @Override - public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { - return (FieldNamesFieldMapper.FieldNamesFieldType) mapperService.fieldType(FieldNamesFieldMapper.NAME); - } - }); - } - - protected static boolean hasDocValues(Map fieldMapping, boolean defaultValue) { + public static boolean hasDocValues(Map fieldMapping, boolean defaultValue) { return (boolean) fieldMapping.getOrDefault("doc_values", defaultValue); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java new file mode 100644 index 000000000000..c558cd9ddfa0 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Assert; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import static org.apache.lucene.tests.util.LuceneTestCase.newDirectory; +import static org.apache.lucene.tests.util.LuceneTestCase.random; + +public class BlockLoaderTestRunner { + private final BlockLoaderTestCase.Params params; + + public BlockLoaderTestRunner(BlockLoaderTestCase.Params params) { + this.params = params; + } + + public void runTest(MapperService mapperService, Map document, Object expected, String blockLoaderFieldName) + throws IOException { + var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(document); + + Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, documentXContent, blockLoaderFieldName); + Assert.assertEquals(expected, blockLoaderResult); + } + + private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException { + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + + var source = new SourceToParse( + "1", + BytesReference.bytes(document), + XContentType.JSON, + null, + Map.of(), + true, + XContentMeteringParserDecorator.NOOP + ); + LuceneDocument doc = mapperService.documentMapper().parse(source).rootDoc(); + + iw.addDocument(doc); + iw.close(); + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + LeafReaderContext context = reader.leaves().get(0); + return load(createBlockLoader(mapperService, fieldName), context, mapperService); + } + } + } + + private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperService mapperService) throws IOException { + // `columnAtATimeReader` is tried first, we mimic `ValuesSourceReaderOperator` + var columnAtATimeReader = blockLoader.columnAtATimeReader(context); + if (columnAtATimeReader != null) { + var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(context.reader().numDocs()), TestBlock.docs(0)); + if (block.size() == 0) { + return null; + } + return block.get(0); + } + + StoredFieldsSpec storedFieldsSpec = blockLoader.rowStrideStoredFieldSpec(); + SourceLoader.Leaf leafSourceLoader = null; + if (storedFieldsSpec.requiresSource()) { + var sourceLoader = mapperService.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); + leafSourceLoader = sourceLoader.leaf(context.reader(), null); + storedFieldsSpec = storedFieldsSpec.merge( + new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields()) + ); + } + BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(context, null), + leafSourceLoader + ); + storedFieldsLoader.advanceTo(0); + + BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(context.reader().numDocs()), 1); + blockLoader.rowStrideReader(context).read(0, storedFieldsLoader, builder); + var block = (TestBlock) builder.build(); + if (block.size() == 0) { + return null; + } + return block.get(0); + } + + private BlockLoader createBlockLoader(MapperService mapperService, String fieldName) { + SearchLookup searchLookup = new SearchLookup(mapperService.mappingLookup().fieldTypesLookup()::get, null, null); + + return mapperService.fieldType(fieldName).blockLoader(new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return mapperService.getIndexSettings().getIndex().getName(); + } + + @Override + public IndexSettings indexSettings() { + return mapperService.getIndexSettings(); + } + + @Override + public MappedFieldType.FieldExtractPreference fieldExtractPreference() { + return params.preference(); + } + + @Override + public SearchLookup lookup() { + return searchLookup; + } + + @Override + public Set sourcePaths(String name) { + return mapperService.mappingLookup().sourcePaths(name); + } + + @Override + public String parentField(String field) { + return mapperService.mappingLookup().parentField(field); + } + + @Override + public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { + return (FieldNamesFieldMapper.FieldNamesFieldType) mapperService.fieldType(FieldNamesFieldMapper.NAME); + } + }); + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldBlockLoaderTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldBlockLoaderTests.java index 283feffa5dfc..9460df7a16a5 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldBlockLoaderTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldBlockLoaderTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; import org.elasticsearch.xpack.aggregatemetric.mapper.datageneration.AggregateMetricDoubleDataSourceHandler; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -33,6 +34,11 @@ public class AggregateMetricDoubleFieldBlockLoaderTests extends BlockLoaderTestC }), params); } + @Override + public void testBlockLoaderOfMultiField() throws IOException { + // Multi fields are noop for aggregate_metric_double. + } + @Override protected Object expected(Map fieldMapping, Object value, TestContext testContext) { if (value instanceof Map map) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java index 471a6a9a69fc..53725cc088e5 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldBlockLoaderTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.mapper.BlockLoaderTestCase; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; +import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; @@ -22,6 +23,11 @@ public class ConstantKeywordFieldBlockLoaderTests extends BlockLoaderTestCase { super(FieldType.CONSTANT_KEYWORD.toString(), params); } + @Override + public void testBlockLoaderOfMultiField() throws IOException { + // Multi fields are noop for constant_keyword. + } + @Override protected Object expected(Map fieldMapping, Object value, TestContext testContext) { return new BytesRef((String) fieldMapping.get("value")); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index fda37c534356..5061c8e30351 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -379,7 +379,8 @@ public class UnsignedLongFieldMapper extends FieldMapper { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.LongsBlockLoader(name()); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 872d174a763b..bde057e95ffc 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -313,7 +313,8 @@ public class GeoShapeWithDocValuesFieldMapper extends AbstractShapeGeometryField if (blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) { return new GeoBoundsBlockLoader(name()); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 7381503e8d98..a338432cf495 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -245,7 +245,8 @@ public class PointFieldMapper extends AbstractPointGeometryFieldMapper return new CartesianBoundsBlockLoader(name()); } - if (isSyntheticSource) { + // Multi fields don't have fallback synthetic source. + if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java index 1c88429d0451..394ce2da7d16 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.xcontent.support.MapXContentParser; import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.elasticsearch.xpack.spatial.datageneration.GeoShapeDataSourceHandler; +import java.io.IOException; import java.nio.ByteOrder; import java.util.Collection; import java.util.Collections; @@ -36,6 +37,11 @@ public class GeoShapeFieldBlockLoaderTests extends BlockLoaderTestCase { super("geo_shape", List.of(new GeoShapeDataSourceHandler()), params); } + @Override + public void testBlockLoaderOfMultiField() throws IOException { + // Multi fields are noop for geo_shape. + } + @Override @SuppressWarnings("unchecked") protected Object expected(Map fieldMapping, Object value, TestContext testContext) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldBlockLoaderTests.java index 922adebaef87..bd4033b0a58d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldBlockLoaderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldBlockLoaderTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.elasticsearch.xpack.spatial.common.CartesianPoint; import org.elasticsearch.xpack.spatial.datageneration.PointDataSourceHandler; +import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collection; @@ -32,6 +33,11 @@ public class PointFieldBlockLoaderTests extends BlockLoaderTestCase { super("point", List.of(new PointDataSourceHandler()), params); } + @Override + public void testBlockLoaderOfMultiField() throws IOException { + // Multi fields are noop for point. + } + @Override @SuppressWarnings("unchecked") protected Object expected(Map fieldMapping, Object value, TestContext testContext) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java index 2a4a011d08db..9fd0b05c249b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xcontent.support.MapXContentParser; import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.elasticsearch.xpack.spatial.datageneration.ShapeDataSourceHandler; +import java.io.IOException; import java.nio.ByteOrder; import java.util.Collection; import java.util.Collections; @@ -34,6 +35,11 @@ public class ShapeFieldBlockLoaderTests extends BlockLoaderTestCase { super("shape", List.of(new ShapeDataSourceHandler()), params); } + @Override + public void testBlockLoaderOfMultiField() throws IOException { + // Multi fields are noop for shape. + } + @Override @SuppressWarnings("unchecked") protected Object expected(Map fieldMapping, Object value, TestContext testContext) {