From 26e22611328674611694758650b76ebb2a91c282 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 25 Apr 2025 10:26:27 -0700 Subject: [PATCH] Remove legacy block loader test infrastructure (#127273) --- .../extras/MatchOnlyTextFieldMapperTests.java | 7 - .../extras/ScaledFloatFieldMapperTests.java | 8 +- .../extras/TokenCountFieldMapperTests.java | 15 +- .../AnnotatedTextFieldMapperTests.java | 11 - .../index/mapper/TextFieldMapper.java | 10 +- .../index/mapper/BooleanFieldMapperTests.java | 8 - .../index/mapper/DateFieldMapperTests.java | 32 +-- .../index/mapper/FloatFieldMapperTests.java | 10 - .../mapper/GeoPointFieldMapperTests.java | 60 +---- .../mapper/HalfFloatFieldMapperTests.java | 12 - .../index/mapper/IpFieldMapperTests.java | 6 - .../index/mapper/KeywordFieldMapperTests.java | 12 - .../index/mapper/LongFieldMapperTests.java | 12 - .../index/mapper/TextFieldMapperTests.java | 43 ---- .../TextFieldBlockLoaderTests.java | 8 +- .../TextFieldWithParentBlockLoaderTests.java | 77 ++++++ .../DefaultMappingParametersHandler.java | 15 +- .../index/mapper/BlockLoaderTestCase.java | 10 +- .../KeywordFieldSyntheticSourceSupport.java | 24 +- .../index/mapper/MapperTestCase.java | 238 +----------------- .../index/mapper/NumberFieldMapperTests.java | 55 +--- ...xtFieldFamilySyntheticSourceTestSetup.java | 55 +--- .../mapper/OffsetSourceFieldMapperTests.java | 2 +- ...AggregateMetricDoubleFieldMapperTests.java | 12 - .../ConstantKeywordFieldMapperTests.java | 7 - .../UnsignedLongFieldMapperTests.java | 21 +- .../VersionStringFieldMapperTests.java | 7 - ...GeoShapeWithDocValuesFieldMapperTests.java | 23 -- .../GeometricShapeSyntheticSourceSupport.java | 27 +- .../index/mapper/PointFieldMapperTests.java | 68 +---- .../index/mapper/ShapeFieldMapperTests.java | 23 -- .../mapper/WildcardFieldMapperTests.java | 16 +- 32 files changed, 156 insertions(+), 778 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index 4ad4c7fe3bef..b913edb1f279 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -21,7 +21,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.DocumentMapper; @@ -44,7 +43,6 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; @@ -257,9 +255,4 @@ public class MatchOnlyTextFieldMapperTests extends MapperTestCase { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } - - @Override - protected Function loadBlockExpected() { - return v -> ((BytesRef) v).utf8ToString(); - } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 15be6b08fadd..7efc449aa856 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -377,7 +377,6 @@ public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests { return new SyntheticSourceExample( example.expectedForSyntheticSource(), example.expectedForSyntheticSource(), - example.expectedForBlockLoader(), example.mapping() ); } @@ -400,7 +399,7 @@ public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests { if (v.malformedOutput == null) { return new SyntheticSourceExample(v.input, v.output, this::mapping); } - return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); + return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping); } List values = randomList(1, maxValues, this::generateValue); List in = values.stream().map(Value::input).toList(); @@ -479,11 +478,6 @@ public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests { } } - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - assumeTrue("Disabled, tested by ScaledFloatFieldBlockLoaderTests instead", false); - return null; - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index 1a4619e00f82..8105eee707cc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -36,8 +36,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -213,17 +211,13 @@ public class TokenCountFieldMapperTests extends MapperTestCase { public SyntheticSourceExample example(int maxValues) { if (randomBoolean()) { var value = generateValue(); - return new SyntheticSourceExample(value.text, value.text, value.tokenCount, this::mapping); + return new SyntheticSourceExample(value.text, value.text, this::mapping); } var values = randomList(1, 5, this::generateValue); - var textArray = values.stream().map(Value::text).toList(); - var blockExpectedList = values.stream().map(Value::tokenCount).filter(Objects::nonNull).sorted().toList(); - var blockExpected = blockExpectedList.size() == 1 ? blockExpectedList.get(0) : blockExpectedList; - - return new SyntheticSourceExample(textArray, textArray, blockExpected, this::mapping); + return new SyntheticSourceExample(textArray, textArray, this::mapping); } private record Value(String text, Integer tokenCount) {} @@ -258,11 +252,6 @@ public class TokenCountFieldMapperTests extends MapperTestCase { }; } - protected Function loadBlockExpected() { - // we can get either a number from doc values or null - return v -> v != null ? (Number) v : null; - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 6c7718608964..377fa62fd5b4 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -61,7 +61,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -666,16 +665,6 @@ public class AnnotatedTextFieldMapperTests extends MapperTestCase { return TextFieldFamilySyntheticSourceTestSetup.syntheticSourceSupport("annotated_text", false); } - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - return TextFieldFamilySyntheticSourceTestSetup.getSupportedReaders(mapper, loaderFieldName); - } - - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - return TextFieldFamilySyntheticSourceTestSetup.loadBlockExpected(blockReaderSupport, columnReader); - } - @Override protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) { TextFieldFamilySyntheticSourceTestSetup.validateRoundTripReader(syntheticSource, reader, roundTripReader); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index c12e9c22af79..d77817915f38 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1032,9 +1032,13 @@ public final class TextFieldMapper extends FieldMapper { return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name()); } - // _ignored_source field will only be present if text field is not stored - // and there is no syntheticSourceDelegate - if (isSyntheticSource && syntheticSourceDelegate == null) { + // _ignored_source field will contain entries for this field if it is not stored + // and there is no syntheticSourceDelegate. + // See #syntheticSourceSupport(). + // But if a text field is a multi field it won't have an entry in _ignored_source. + // The parent might, but we don't have enough context here to figure this out. + // So we bail. + if (isSyntheticSource && syntheticSourceDelegate == null && parentField == null) { return fallbackSyntheticSourceBlockLoader(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 80c862c02aa7..47161239a363 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.time.Instant; import java.util.List; -import java.util.function.Function; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -371,19 +370,12 @@ public class BooleanFieldMapperTests extends MapperTestCase { return new SyntheticSourceExample( example.expectedForSyntheticSource(), example.expectedForSyntheticSource(), - example.expectedForBlockLoader(), example.mapping() ); } }; } - @Override - protected Function loadBlockExpected() { - // Just assert that we expect a boolean. Otherwise no munging. - return v -> (Boolean) v; - } - protected IngestScriptSupport ingestScriptSupport() { return new IngestScriptSupport() { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 1d49d2cc93cc..fdf2503036d8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -33,7 +33,6 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Comparator; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; @@ -595,15 +594,10 @@ public class DateFieldMapperTests extends MapperTestCase { if (randomBoolean()) { Value v = generateValue(); if (v.malformedOutput != null) { - return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); + return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping); } - return new SyntheticSourceExample( - v.input, - v.output, - resolution.convert(Instant.from(formatter.parse(v.output))), - this::mapping - ); + return new SyntheticSourceExample(v.input, v.output, this::mapping); } List values = randomList(1, maxValues, this::generateValue); @@ -625,11 +619,7 @@ public class DateFieldMapperTests extends MapperTestCase { List outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; - List outBlockList = outputFromDocValues.stream() - .map(v -> resolution.convert(Instant.from(formatter.parse(v)))) - .toList(); - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(in, out, outBlock, this::mapping); + return new SyntheticSourceExample(in, out, this::mapping); } private record Value(Object input, String output, Object malformedOutput) {} @@ -727,22 +717,6 @@ public class DateFieldMapperTests extends MapperTestCase { }; } - @Override - protected Function loadBlockExpected() { - return v -> asJacksonNumberOutput(((Number) v).longValue()); - } - - protected static Object asJacksonNumberOutput(long l) { - // If a long value fits in int, Jackson will write it as int in NumberOutput.outputLong() - // and we hit this during serialization of expected values. - // Code below mimics that behaviour in order for matching to work. - if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) { - return (int) l; - } else { - return l; - } - } - public void testLegacyField() throws Exception { // check that unknown date formats are treated leniently on old indices MapperService service = createMapperService(IndexVersion.fromId(5000099), Settings.EMPTY, () -> false, mapping(b -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index 902dfb99e710..ccae6c44ebc0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -14,7 +14,6 @@ import org.junit.AssumptionViolatedException; import java.io.IOException; import java.util.List; -import java.util.function.Function; public class FloatFieldMapperTests extends NumberFieldMapperTests { @@ -61,15 +60,6 @@ public class FloatFieldMapperTests extends NumberFieldMapperTests { } - @Override - protected Function loadBlockExpected() { - return v -> { - // The test converts the float into a string so we do do - Number n = (Number) v; - return Double.parseDouble(Float.toString(n.floatValue())); - }; - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 3a53440403c9..fb73af2102ae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -18,10 +18,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.WellKnownBinary; import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -37,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; @@ -611,13 +607,13 @@ public class GeoPointFieldMapperTests extends MapperTestCase { if (randomBoolean()) { Value v = generateValue(); if (v.malformedOutput != null) { - return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); + return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping); } if (columnReader) { - return new SyntheticSourceExample(v.input, decode(encode(v.output)), encode(v.output), this::mapping); + return new SyntheticSourceExample(v.input, decode(encode(v.output)), this::mapping); } - return new SyntheticSourceExample(v.input, v.output, v.output.toWKT(), this::mapping); + return new SyntheticSourceExample(v.input, v.output, this::mapping); } List values = randomList(1, maxVals, this::generateValue); @@ -635,18 +631,7 @@ public class GeoPointFieldMapperTests extends MapperTestCase { List outList = Stream.concat(outputFromDocValues.stream(), malformedValues).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; - if (columnReader) { - // When reading doc-values, the block is a list of encoded longs - List outBlockList = outputFromDocValues.stream().map(this::encode).toList(); - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(in, out, outBlock, this::mapping); - } else { - // When reading row-stride, the block is a list of WKT encoded BytesRefs. - // Values are ordered in order of input. - List outBlockList = values.stream().filter(v -> v.malformedOutput == null).map(v -> v.output.toWKT()).toList(); - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(in, out, outBlock, this::mapping); - } + return new SyntheticSourceExample(in, out, this::mapping); } private record Value(Object input, GeoPoint output, Object malformedOutput) {} @@ -736,41 +721,4 @@ public class GeoPointFieldMapperTests extends MapperTestCase { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } - - @Override - protected Function loadBlockExpected() { - throw new IllegalStateException("Should never reach here, call loadBlockExpected(BlockReaderSupport, boolean) instead"); - } - - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - if (columnReader) { - // When using column reader, we expect the output to be doc-values (which means encoded longs) - return v -> asJacksonNumberOutput(((Number) v).longValue()); - } else { - // When using row-stride reader, we expect the output to be WKT encoded BytesRef - return v -> asWKT((BytesRef) v); - } - } - - protected static Object asJacksonNumberOutput(long l) { - // Cast to int to mimic jackson-core behaviour in NumberOutput.outputLong() - if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) { - return (int) l; - } else { - return l; - } - } - - protected static Object asWKT(BytesRef value) { - // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability - Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes); - return WellKnownText.toWKT(geometry); - } - - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - return new BlockReaderSupport(ft.hasDocValues(), false, mapper, loaderFieldName); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index 0361df4385a3..905f26601001 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -16,7 +16,6 @@ import org.junit.AssumptionViolatedException; import java.io.IOException; import java.util.List; -import java.util.function.Function; public class HalfFloatFieldMapperTests extends NumberFieldMapperTests { @@ -65,17 +64,6 @@ public class HalfFloatFieldMapperTests extends NumberFieldMapperTests { } - @Override - protected Function loadBlockExpected() { - return v -> { - // The test converts the float into a string so we do do - Number n = (Number) v; - return Double.parseDouble( - Float.toString(HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(n.floatValue()))) - ); - }; - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 17ecc2c22db2..2ca2a7f8eb62 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -31,7 +31,6 @@ import java.net.InetAddress; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -435,11 +434,6 @@ public class IpFieldMapperTests extends MapperTestCase { }; } - @Override - protected Function loadBlockExpected() { - return v -> InetAddresses.toAddrString(InetAddressPoint.decode(BytesRef.deepCopyOf((BytesRef) v).bytes)); - } - @Override protected String randomSyntheticSourceKeep() { return "all"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index af9fb63d77f2..f6a7148c7109 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -52,7 +52,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.function.Function; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; @@ -676,17 +675,6 @@ public class KeywordFieldMapperTests extends MapperTestCase { return false; } - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - return new BlockReaderSupport(ft.hasDocValues(), ft.hasDocValues() || ft.isStored(), mapper, loaderFieldName); - } - - @Override - protected Function loadBlockExpected() { - return v -> ((BytesRef) v).utf8ToString(); - } - @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { assertFalse("keyword doesn't support ignore_malformed", ignoreMalformed); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java index 6782abb4a5b4..9a0f4baa8f21 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.math.BigInteger; import java.util.List; -import java.util.function.Function; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -121,17 +120,6 @@ public class LongFieldMapperTests extends WholeNumberFieldMapperTests { assertFetch(randomFetchTestMapper(), "field", 3.783147882954537E18, randomFetchTestFormat()); } - @Override - protected Function loadBlockExpected() { - return n -> { - Number number = ((Number) n); - if (Integer.MIN_VALUE <= number.longValue() && number.longValue() <= Integer.MAX_VALUE) { - return number.intValue(); - } - return number.longValue(); - }; - } - protected IngestScriptSupport ingestScriptSupport() { return new IngestScriptSupport() { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 32cbcfc2441a..f74af767e941 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -48,7 +48,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -85,7 +84,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -1181,11 +1179,6 @@ public class TextFieldMapperTests extends MapperTestCase { return TextFieldFamilySyntheticSourceTestSetup.syntheticSourceSupport("text", true); } - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - return TextFieldFamilySyntheticSourceTestSetup.loadBlockExpected(blockReaderSupport, columnReader); - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); @@ -1321,40 +1314,4 @@ public class TextFieldMapperTests extends MapperTestCase { assertFalse(dv.advanceExact(3)); }); } - - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - return TextFieldFamilySyntheticSourceTestSetup.getSupportedReaders(mapper, loaderFieldName); - } - - public void testBlockLoaderFromParentColumnReader() throws IOException { - testBlockLoaderFromParent(true, randomBoolean()); - } - - public void testBlockLoaderParentFromRowStrideReader() throws IOException { - testBlockLoaderFromParent(false, randomBoolean()); - } - - private void testBlockLoaderFromParent(boolean columnReader, boolean syntheticSource) throws IOException { - boolean storeParent = randomBoolean(); - KeywordFieldSyntheticSourceSupport kwdSupport = new KeywordFieldSyntheticSourceSupport(null, storeParent, null, false); - SyntheticSourceExample example = kwdSupport.example(5); - CheckedConsumer buildFields = b -> { - b.startObject("field"); - { - example.mapping().accept(b); - b.startObject("fields").startObject("sub"); - { - b.field("type", "text"); - } - b.endObject().endObject(); - } - b.endObject(); - }; - XContentBuilder mapping = mapping(buildFields); - MapperService mapper = syntheticSource ? createSytheticSourceMapperService(mapping) : createMapperService(mapping); - BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field.sub"); - var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); - testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java index 2f8f83a65a2e..5c9acaf18a45 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java @@ -25,9 +25,13 @@ public class TextFieldBlockLoaderTests extends BlockLoaderTestCase { super(FieldType.TEXT.toString(), params); } - @SuppressWarnings("unchecked") @Override protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + return expectedValue(fieldMapping, value, params, testContext); + } + + @SuppressWarnings("unchecked") + public static Object expectedValue(Map fieldMapping, Object value, Params params, TestContext testContext) { if (fieldMapping.getOrDefault("store", false).equals(true)) { return valuesInSourceOrder(value); } @@ -116,7 +120,7 @@ public class TextFieldBlockLoaderTests extends BlockLoaderTestCase { } @SuppressWarnings("unchecked") - private Object valuesInSourceOrder(Object value) { + private static Object valuesInSourceOrder(Object value) { if (value == null) { return null; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java new file mode 100644 index 000000000000..1f154fa7581a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.datageneration.FieldType; +import org.elasticsearch.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.datageneration.datasource.DefaultMappingParametersHandler; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TextFieldWithParentBlockLoaderTests extends BlockLoaderTestCase { + public TextFieldWithParentBlockLoaderTests(Params params) { + // keyword because we need a keyword parent field + super(FieldType.KEYWORD.toString(), List.of(new DataSourceHandler() { + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + assert request.fieldType().equals(FieldType.KEYWORD.toString()); + + // We need to force multi field generation + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var defaultSupplier = DefaultMappingParametersHandler.keywordMapping( + request, + DefaultMappingParametersHandler.commonMappingParameters() + ); + var mapping = defaultSupplier.get(); + // we don't need this here + mapping.remove("copy_to"); + + var textMultiFieldMappingSupplier = DefaultMappingParametersHandler.textMapping(request, new HashMap<>()); + var textMultiFieldMapping = textMultiFieldMappingSupplier.get(); + textMultiFieldMapping.put("type", "text"); + textMultiFieldMapping.remove("fields"); + + mapping.put("fields", Map.of("txt", textMultiFieldMapping)); + + return mapping; + }); + } + }), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, TestContext testContext) { + assert fieldMapping.containsKey("fields"); + + Object normalizer = fieldMapping.get("normalizer"); + boolean docValues = hasDocValues(fieldMapping, true); + boolean store = fieldMapping.getOrDefault("store", false).equals(true); + + if (normalizer == null && (docValues || store)) { + // we are using block loader of the parent field + return KeywordFieldBlockLoaderTests.expectedValue(fieldMapping, value, params, testContext); + } + + // we are using block loader of the text field itself + var textFieldMapping = (Map) ((Map) fieldMapping.get("fields")).get("txt"); + return TextFieldBlockLoaderTests.expectedValue(textFieldMapping, value, params, testContext); + } + + @Override + protected String blockLoaderFieldName(String originalName) { + return originalName + ".txt"; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java index 8cc052a379cf..115ffacfc98e 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java @@ -84,7 +84,7 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { }; } - private Supplier> keywordMapping( + public static Supplier> keywordMapping( DataSourceRequest.LeafMappingParametersGenerator request, Map injected ) { @@ -111,6 +111,14 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); } + if (ESTestCase.randomDouble() <= 0.1) { + var textMultiFieldMapping = textMapping(request, new HashMap<>()).get(); + textMultiFieldMapping.put("type", "text"); + textMultiFieldMapping.remove("fields"); + + injected.put("fields", Map.of("txt", textMultiFieldMapping)); + } + return injected; }; } @@ -192,7 +200,7 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { }; } - private Supplier> textMapping( + public static Supplier> textMapping( DataSourceRequest.LeafMappingParametersGenerator request, Map injected ) { @@ -206,7 +214,6 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { keywordMultiFieldMapping.remove("copy_to"); injected.put("fields", Map.of("kwd", keywordMultiFieldMapping)); - } return injected; @@ -250,7 +257,7 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { }; } - private static HashMap commonMappingParameters() { + public static HashMap commonMappingParameters() { var map = new HashMap(); map.put("store", ESTestCase.randomBoolean()); map.put("index", ESTestCase.randomBoolean()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index e2f35454fa1d..965c6dd4dd92 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -166,8 +166,8 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { var document = documentGenerator.generate(template, mapping); var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(document); - Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, documentXContent, fieldName); Object expected = expected(mapping.lookup().get(fieldName), getFieldValue(document, fieldName), testContext); + Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, documentXContent, blockLoaderFieldName(fieldName)); assertEquals(expected, blockLoaderResult); } @@ -216,6 +216,14 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase { return list; } + /** + Allows to change the field name used to obtain a block loader. + Useful f.e. to test block loaders of multi fields. + */ + protected String blockLoaderFieldName(String originalName) { + return originalName; + } + private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java index 502ffdde62e5..d87831b85df1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java @@ -18,8 +18,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import java.util.stream.Stream; public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.SyntheticSourceSupport { @@ -58,11 +56,7 @@ public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.Synthe if (ESTestCase.randomBoolean()) { Tuple v = generateValue(); Object sourceValue = preservesExactSource() ? v.v1() : v.v2(); - Object loadBlock = v.v2(); - if (loadBlockFromSource == false && ignoreAbove != null && v.v2().length() > ignoreAbove) { - loadBlock = null; - } - return new MapperTestCase.SyntheticSourceExample(v.v1(), sourceValue, loadBlock, this::mapping); + return new MapperTestCase.SyntheticSourceExample(v.v1(), sourceValue, this::mapping); } List> values = ESTestCase.randomList(1, maxValues, this::generateValue); List in = values.stream().map(Tuple::v1).toList(); @@ -76,7 +70,7 @@ public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.Synthe validValues.add(v); } }); - List outputFromDocValues = new HashSet<>(validValues).stream().sorted().collect(Collectors.toList()); + List outputFromDocValues = new HashSet<>(validValues).stream().sorted().toList(); Object out; if (preservesExactSource()) { @@ -87,19 +81,7 @@ public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.Synthe out = syntheticSourceOutputList.size() == 1 ? syntheticSourceOutputList.get(0) : syntheticSourceOutputList; } - List loadBlock; - if (loadBlockFromSource) { - // The block loader infrastructure will never return nulls. Just zap them all. - loadBlock = in.stream().filter(Objects::nonNull).toList(); - } else if (docValues) { - loadBlock = List.copyOf(outputFromDocValues); - } else { - // Meaning loading from terms. - loadBlock = List.copyOf(validValues); - } - - Object loadBlockResult = loadBlock.size() == 1 ? loadBlock.get(0) : loadBlock; - return new MapperTestCase.SyntheticSourceExample(in, out, loadBlockResult, this::mapping); + return new MapperTestCase.SyntheticSourceExample(in, out, this::mapping); } private Tuple generateValue() { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 00750414574c..52bf6484de80 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.index.translog.Translog; @@ -61,13 +60,11 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.LeafStoredFieldsLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; -import org.elasticsearch.test.ListMatcher; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -88,7 +85,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.test.MapMatcher.assertMap; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -1055,24 +1051,10 @@ public abstract class MapperTestCase extends MapperServiceTestCase { public record SyntheticSourceExample( CheckedConsumer inputValue, CheckedConsumer expectedForSyntheticSource, - CheckedConsumer expectedForBlockLoader, CheckedConsumer mapping ) { public SyntheticSourceExample(Object inputValue, Object result, CheckedConsumer mapping) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(result), mapping); - } - - /** - * Create an example that returns different results from doc values - * than from synthetic source. - */ - public SyntheticSourceExample( - Object inputValue, - Object result, - Object blockLoaderResults, - CheckedConsumer mapping - ) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(blockLoaderResults), mapping); + this(b -> b.value(inputValue), b -> b.value(result), mapping); } public void buildInput(XContentBuilder b) throws IOException { @@ -1093,13 +1075,6 @@ public abstract class MapperTestCase extends MapperServiceTestCase { expectedForSyntheticSource.accept(b); return Strings.toString(b.endObject()); } - - private Object expectedParsedForBlockLoader() throws IOException { - XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); - expectedForBlockLoader.accept(b); - String str = Strings.toString(b.endObject()); - return XContentHelper.convertToMap(JsonXContent.jsonXContent, str, false).get("field"); - } } public record SyntheticSourceInvalidExample(Matcher error, CheckedConsumer mapping) {} @@ -1156,7 +1131,7 @@ public abstract class MapperTestCase extends MapperServiceTestCase { v.mapping.accept(b); b.field("ignore_malformed", true); }; - assertSyntheticSource(new SyntheticSourceExample(v.value, v.value, v.value, mapping)); + assertSyntheticSource(new SyntheticSourceExample(v.value, v.value, mapping)); } } @@ -1377,215 +1352,6 @@ public abstract class MapperTestCase extends MapperServiceTestCase { assertNoDocValueLoader(b -> b.startArray("field").endArray()); } - public final void testBlockLoaderFromColumnReader() throws IOException { - testBlockLoader(false, true); - } - - public final void testBlockLoaderFromRowStrideReader() throws IOException { - testBlockLoader(false, false); - } - - public final void testBlockLoaderFromColumnReaderWithSyntheticSource() throws IOException { - testBlockLoader(true, true); - } - - public final void testBlockLoaderFromRowStrideReaderWithSyntheticSource() throws IOException { - testBlockLoader(true, false); - } - - /** - * Get the configuration for testing block loaders with this field. In particular, not all fields can be loaded from doc-values. - * For most ESQL types the preference is to read from doc-values if they exist, so that is the default behaviour here. - * However, for spatial types, the doc-values involve precision loss, and therefor it is preferable to read from source. - * And for text fields, doc values are not easily convertable to original values either, so special cases exist. - */ - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - return new BlockReaderSupport(ft.hasDocValues(), true, mapper, loaderFieldName); - } - - /** - * This record encapsulates the test configuration for testing block loaders (used in ES|QL). - * - * @param columnAtATimeReader true if the field supports column at a time readers (doc-values) - * @param syntheticSource true if the field supports synthetic source - * @param mapper the mapper service to use for testing - * @param loaderFieldName the field name to use for loading the field - */ - public record BlockReaderSupport(boolean columnAtATimeReader, boolean syntheticSource, MapperService mapper, String loaderFieldName) { - public BlockReaderSupport(boolean columnAtATimeReader, MapperService mapper, String loaderFieldName) { - this(columnAtATimeReader, true, mapper, loaderFieldName); - } - - private BlockLoader getBlockLoader(FieldExtractPreference fieldExtractPreference) { - SearchLookup searchLookup = new SearchLookup(mapper.mappingLookup().fieldTypesLookup()::get, null, null); - return mapper.fieldType(loaderFieldName).blockLoader(new MappedFieldType.BlockLoaderContext() { - @Override - public String indexName() { - return mapper.getIndexSettings().getIndex().getName(); - } - - @Override - public IndexSettings indexSettings() { - return mapper.getIndexSettings(); - } - - @Override - public FieldExtractPreference fieldExtractPreference() { - return fieldExtractPreference; - } - - @Override - public SearchLookup lookup() { - return searchLookup; - } - - @Override - public Set sourcePaths(String name) { - return mapper.mappingLookup().sourcePaths(name); - } - - @Override - public String parentField(String field) { - return mapper.mappingLookup().parentField(field); - } - - @Override - public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { - return (FieldNamesFieldMapper.FieldNamesFieldType) mapper.fieldType(FieldNamesFieldMapper.NAME); - } - }); - } - } - - private void testBlockLoader(boolean syntheticSource, boolean columnReader) throws IOException { - // TODO if we're not using synthetic source use a different sort of example. Or something. - var syntheticSourceSupport = syntheticSourceSupport(false, columnReader); - SyntheticSourceExample example = syntheticSourceSupport.example(5); - if (syntheticSource && columnReader == false) { - // The synthetic source testing support can't always handle now the difference between stored and synthetic source mode. - // In case of ignore above, the ignored values are always appended after the valid values - // (both if field has doc values or stored field). While stored source just reads original values (from _source) and there - // is no notion of values that are ignored. - // TODO: fix this by improving block loader support: https://github.com/elastic/elasticsearch/issues/115257 - assumeTrue("inconsistent synthetic source testing support with ignore above", syntheticSourceSupport.ignoreAbove() == false); - } - XContentBuilder mapping = fieldMapping(example.mapping); - MapperService mapper = syntheticSource ? createSytheticSourceMapperService(mapping) : createMapperService(mapping); - BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field"); - if (syntheticSource) { - // geo_point and point do not yet support synthetic source - assumeTrue( - "Synthetic source not completely supported for " + this.getClass().getSimpleName(), - blockReaderSupport.syntheticSource - ); - } - var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); - testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader); - } - - protected final void testBlockLoader( - boolean columnReader, - SyntheticSourceExample example, - BlockReaderSupport blockReaderSupport, - SourceLoader sourceLoader - ) throws IOException { - // EXTRACT_SPATIAL_BOUNDS is not currently supported in this test path. - var fieldExtractPreference = columnReader ? FieldExtractPreference.DOC_VALUES : FieldExtractPreference.NONE; - BlockLoader loader = blockReaderSupport.getBlockLoader(fieldExtractPreference); - Function valuesConvert = loadBlockExpected(blockReaderSupport, columnReader); - if (valuesConvert == null) { - assertNull(loader); - return; - } - try (Directory directory = newDirectory()) { - RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = blockReaderSupport.mapper.documentMapper().parse(source(b -> { - b.field("field"); - example.inputValue.accept(b); - })).rootDoc(); - iw.addDocument(doc); - iw.close(); - try (DirectoryReader reader = DirectoryReader.open(directory)) { - LeafReaderContext ctx = reader.leaves().get(0); - TestBlock block; - if (columnReader) { - if (blockReaderSupport.columnAtATimeReader) { - block = (TestBlock) loader.columnAtATimeReader(ctx) - .read(TestBlock.factory(ctx.reader().numDocs()), TestBlock.docs(0)); - } else { - assertNull(loader.columnAtATimeReader(ctx)); - return; - } - } else { - StoredFieldsSpec storedFieldsSpec = loader.rowStrideStoredFieldSpec(); - if (storedFieldsSpec.requiresSource()) { - storedFieldsSpec = storedFieldsSpec.merge( - new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields()) - ); - } - BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null), - storedFieldsSpec.requiresSource() ? sourceLoader.leaf(ctx.reader(), null) : null - ); - storedFieldsLoader.advanceTo(0); - BlockLoader.Builder builder = loader.builder(TestBlock.factory(ctx.reader().numDocs()), 1); - loader.rowStrideReader(ctx).read(0, storedFieldsLoader, builder); - block = (TestBlock) builder.build(); - } - Object inBlock = block.get(0); - if (inBlock != null) { - if (inBlock instanceof List l) { - inBlock = l.stream().map(valuesConvert).toList(); - } else { - inBlock = valuesConvert.apply(inBlock); - } - } - Object expected = example.expectedParsedForBlockLoader(); - if (List.of().equals(expected)) { - assertThat(inBlock, nullValue()); - return; - } - if (expected instanceof List l) { - ListMatcher m = ListMatcher.matchesList(); - for (Object v : l) { - m = m.item(blockItemMatcher(v)); - } - assertMap((List) inBlock, m); - return; - } - @SuppressWarnings("unchecked") - Matcher e = (Matcher) blockItemMatcher(expected); - assertThat(inBlock, e); - } - } - } - - /** - * Matcher for {@link #testBlockLoaderFromColumnReader} and {@link #testBlockLoaderFromRowStrideReader}. - */ - protected Matcher blockItemMatcher(Object expected) { - return equalTo(expected); - } - - /** - * How {@link MappedFieldType#blockLoader} should load values or {@code null} - * if that method isn't supported by field being tested. - */ - protected Function loadBlockExpected() { - return null; - } - - /** - * How {@link MappedFieldType#blockLoader} should load values or {@code null} - * if that method isn't supported by field being tested. - * This method should be overridden by fields that support different Block types - * when loading from doc values vs source. - */ - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - return loadBlockExpected(); - } - public final void testEmptyDocumentNoDocValueLoader() throws IOException { assumeFalse("Field will add values even if no fields are supplied", addsValueWhenNotSupplied()); assertNoDocValueLoader(b -> {}); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index da646358a090..58234da1be39 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; -import org.hamcrest.Matcher; import java.io.IOException; import java.util.ArrayList; @@ -37,7 +36,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notANumber; public abstract class NumberFieldMapperTests extends MapperTestCase { @@ -376,24 +374,6 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { assertThat(e.getCause().getMessage(), containsString("Only one field can be stored per key")); } - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - // Block loader can either use doc values or source. - // So with synthetic source it only works when doc values are enabled. - return new BlockReaderSupport(ft.hasDocValues(), ft.hasDocValues(), mapper, loaderFieldName); - } - - @Override - protected Function loadBlockExpected() { - return n -> ((Number) n); // Just assert it's a number - } - - @Override - protected Matcher blockItemMatcher(Object expected) { - return "NaN".equals(expected) ? notANumber() : equalTo(expected); - } - protected abstract Number randomNumber(); protected final class NumberSyntheticSourceSupportForKeepTests extends NumberSyntheticSourceSupport { @@ -419,7 +399,6 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { return new SyntheticSourceExample( example.expectedForSyntheticSource(), example.expectedForSyntheticSource(), - example.expectedForBlockLoader(), example.mapping() ); } @@ -451,33 +430,20 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { Tuple v = generateValue(); if (preservesExactSource()) { var rawInput = v.v1(); - - // This code actually runs with synthetic source disabled - // to test block loader loading from source. - // That's why we need to set expected block loader value here. - var blockLoaderResult = v.v2() instanceof Number n ? round.apply(n) : null; - return new SyntheticSourceExample(rawInput, rawInput, blockLoaderResult, this::mapping); + return new SyntheticSourceExample(rawInput, rawInput, this::mapping); } if (v.v2() instanceof Number n) { Number result = round.apply(n); - return new SyntheticSourceExample(v.v1(), result, result, this::mapping); + return new SyntheticSourceExample(v.v1(), result, this::mapping); } // ignore_malformed value - return new SyntheticSourceExample(v.v1(), v.v2(), List.of(), this::mapping); + return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping); } List> values = randomList(1, maxVals, this::generateValue); List in = values.stream().map(Tuple::v1).toList(); - Object out; - List outBlockList; + if (preservesExactSource()) { - // This code actually runs with synthetic source disabled - // to test block loader loading from source. - // That's why we need to set expected block loader value here. - out = in; - outBlockList = values.stream() - .filter(v -> v.v2() instanceof Number) - .map(t -> round.apply((Number) t.v2())) - .collect(Collectors.toCollection(ArrayList::new)); + return new SyntheticSourceExample(in, in, this::mapping); } else { List outList = values.stream() .filter(v -> v.v2() instanceof Number) @@ -485,17 +451,10 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { .sorted() .collect(Collectors.toCollection(ArrayList::new)); values.stream().filter(v -> false == v.v2() instanceof Number).map(Tuple::v2).forEach(outList::add); - out = outList.size() == 1 ? outList.get(0) : outList; + var out = outList.size() == 1 ? outList.get(0) : outList; - outBlockList = values.stream() - .filter(v -> v.v2() instanceof Number) - .map(t -> round.apply((Number) t.v2())) - .sorted() - .collect(Collectors.toCollection(ArrayList::new)); + return new SyntheticSourceExample(in, out, this::mapping); } - - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(in, out, outBlock, this::mapping); } private Tuple generateValue() { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java index 475bf9212e1c..3ee53de501b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java @@ -10,14 +10,12 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; @@ -32,28 +30,6 @@ public final class TextFieldFamilySyntheticSourceTestSetup { return new TextFieldFamilySyntheticSourceSupport(fieldType, supportsCustomIndexConfiguration); } - public static MapperTestCase.BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - String parentName = mapper.mappingLookup().parentField(ft.name()); - if (parentName == null) { - TextFieldMapper.TextFieldType text = (TextFieldMapper.TextFieldType) ft; - boolean supportsColumnAtATimeReader = text.syntheticSourceDelegate() != null - && text.syntheticSourceDelegate().hasDocValues() - && text.canUseSyntheticSourceDelegateForLoading(); - return new MapperTestCase.BlockReaderSupport(supportsColumnAtATimeReader, mapper, loaderFieldName); - } - MappedFieldType parent = mapper.fieldType(parentName); - if (false == parent.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) { - throw new UnsupportedOperationException(); - } - KeywordFieldMapper.KeywordFieldType kwd = (KeywordFieldMapper.KeywordFieldType) parent; - return new MapperTestCase.BlockReaderSupport(kwd.hasDocValues(), mapper, loaderFieldName); - } - - public static Function loadBlockExpected(MapperTestCase.BlockReaderSupport blockReaderSupport, boolean columnReader) { - return v -> ((BytesRef) v).utf8ToString(); - } - public static void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) { // `reader` here is reader of original document and `roundTripReader` reads document // created from synthetic source. @@ -107,24 +83,19 @@ public final class TextFieldFamilySyntheticSourceTestSetup { boolean loadingFromSource = ignoreAbove != null; MapperTestCase.SyntheticSourceExample delegate = keywordMultiFieldSyntheticSourceSupport.example(maxValues, loadingFromSource); - return new MapperTestCase.SyntheticSourceExample( - delegate.inputValue(), - delegate.expectedForSyntheticSource(), - delegate.expectedForBlockLoader(), - b -> { - b.field("type", fieldType); - if (index == false) { - b.field("index", false); - } - b.startObject("fields"); - { - b.startObject(randomAlphaOfLength(4)); - delegate.mapping().accept(b); - b.endObject(); - } + return new MapperTestCase.SyntheticSourceExample(delegate.inputValue(), delegate.expectedForSyntheticSource(), b -> { + b.field("type", fieldType); + if (index == false) { + b.field("index", false); + } + b.startObject("fields"); + { + b.startObject(randomAlphaOfLength(4)); + delegate.mapping().accept(b); b.endObject(); } - ); + b.endObject(); + }); } private MapperTestCase.SyntheticSourceExample storedFieldExample( @@ -133,13 +104,13 @@ public final class TextFieldFamilySyntheticSourceTestSetup { ) { if (randomBoolean()) { var randomString = randomString(); - return new MapperTestCase.SyntheticSourceExample(randomString, randomString, randomString, mapping); + return new MapperTestCase.SyntheticSourceExample(randomString, randomString, mapping); } var list = ESTestCase.randomList(1, maxValues, this::randomString); var output = list.size() == 1 ? list.get(0) : list; - return new MapperTestCase.SyntheticSourceExample(list, output, output, mapping); + return new MapperTestCase.SyntheticSourceExample(list, output, mapping); } private String randomString() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index 40140d6da5eb..47c4c8b23f42 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -106,7 +106,7 @@ public class OffsetSourceFieldMapperTests extends MapperTestCase { return new SyntheticSourceSupport() { @Override public SyntheticSourceExample example(int maxValues) { - return new SyntheticSourceExample(getSampleValueForDocument(), getSampleValueForDocument(), null, b -> minimalMapping(b)); + return new SyntheticSourceExample(getSampleValueForDocument(), getSampleValueForDocument(), b -> minimalMapping(b)); } @Override diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index 3674043a7276..874ced5436f6 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -36,7 +36,6 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Names.IGNORE_MALFORMED; import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.Names.METRICS; @@ -619,15 +618,4 @@ public class AggregateMetricDoubleFieldMapperTests extends MapperTestCase { protected boolean supportsCopyTo() { return false; } - - @Override - protected Function loadBlockExpected() { - return n -> ((Number) n); - } - - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - assumeTrue("Not supporting", false); - return null; - } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 2b9170afdfd7..c0c2db53b97e 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -40,7 +39,6 @@ import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Set; -import java.util.function.Function; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; import static org.hamcrest.Matchers.equalTo; @@ -319,11 +317,6 @@ public class ConstantKeywordFieldMapperTests extends MapperTestCase { throw new AssumptionViolatedException("not supported"); } - @Override - protected Function loadBlockExpected() { - return v -> ((BytesRef) v).utf8ToString(); - } - public void testNullValueSyntheticSource() throws IOException { DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> { b.startObject("field"); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 8c920b3293ba..d2aabf8b8240 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -34,7 +34,6 @@ import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; @@ -394,7 +393,6 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { return new SyntheticSourceExample( example.expectedForSyntheticSource(), example.expectedForSyntheticSource(), - example.expectedForBlockLoader(), example.mapping() ); } @@ -437,19 +435,6 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { return randomDoubleBetween(0L, Long.MAX_VALUE, true); } - protected Function loadBlockExpected() { - return v -> { - // Numbers are in the block as a long but the test needs to compare them to their BigInteger value parsed from xcontent. - if (v instanceof BigInteger ul) { - if (ul.bitLength() < Long.SIZE) { - return ul.longValue() ^ Long.MIN_VALUE; - } - return ul.subtract(BigInteger.ONE.shiftLeft(Long.SIZE - 1)).longValue(); - } - return ((Long) v).longValue() ^ Long.MIN_VALUE; - }; - } - class NumberSyntheticSourceSupport implements SyntheticSourceSupport { private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong()); private final boolean ignoreMalformedEnabled; @@ -465,7 +450,7 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { if (v.malformedOutput == null) { return new SyntheticSourceExample(v.input, v.output, this::mapping); } - return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); + return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping); } List values = randomList(1, maxVals, this::generateValue); List in = values.stream().map(Value::input).toList(); @@ -481,9 +466,7 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { List outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; - Object outBlock = outputFromDocValues.size() == 1 ? outputFromDocValues.get(0) : outputFromDocValues; - - return new SyntheticSourceExample(in, out, outBlock, this::mapping); + return new SyntheticSourceExample(in, out, this::mapping); } private record Value(Object input, BigInteger output, Object malformedOutput) {} diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 1b2245ba3a33..710b6652ed05 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Tuple; @@ -32,7 +31,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -167,11 +165,6 @@ public class VersionStringFieldMapperTests extends MapperTestCase { return new VersionStringSyntheticSourceSupport(); } - @Override - protected Function loadBlockExpected() { - return v -> new Version((BytesRef) v).toString(); - } - static class VersionStringSyntheticSourceSupport implements SyntheticSourceSupport { @Override public SyntheticSourceExample example(int maxValues) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index a34f0ba2eae3..9482f45063b6 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.geo.Orientation; @@ -20,12 +19,8 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Circle; -import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.WellKnownBinary; -import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper; @@ -54,7 +49,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.function.Function; import static org.elasticsearch.legacygeo.mapper.LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS; import static org.hamcrest.Matchers.containsString; @@ -555,23 +549,6 @@ public class GeoShapeWithDocValuesFieldMapperTests extends GeoFieldMapperTests { return new GeometricShapeSyntheticSourceSupport(FieldType.GEO_SHAPE, ignoreMalformed); } - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - return v -> asWKT((BytesRef) v); - } - - protected static Object asWKT(BytesRef value) { - // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability - Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes); - return WellKnownText.toWKT(geometry); - } - - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - // Synthetic source is currently not supported. - return new BlockReaderSupport(false, false, mapper, loaderFieldName); - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java index e1a91bb592ce..800473d43a68 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java @@ -8,8 +8,6 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.elasticsearch.common.geo.GeoJson; -import org.elasticsearch.common.geo.GeometryNormalizer; -import org.elasticsearch.common.geo.Orientation; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.ShapeType; @@ -50,9 +48,6 @@ public class GeometricShapeSyntheticSourceSupport implements MapperTestCase.Synt public MapperTestCase.SyntheticSourceExample example(int maxValues) throws IOException { if (randomBoolean()) { Value v = generateValue(); - if (v.blockLoaderOutput != null) { - return new MapperTestCase.SyntheticSourceExample(v.input, v.output, v.blockLoaderOutput, this::mapping); - } return new MapperTestCase.SyntheticSourceExample(v.input, v.output, this::mapping); } @@ -60,21 +55,10 @@ public class GeometricShapeSyntheticSourceSupport implements MapperTestCase.Synt List in = values.stream().map(Value::input).toList(); List out = values.stream().map(Value::output).toList(); - // Block loader infrastructure will never return nulls - List outBlockList = values.stream() - .filter(v -> v.input != null) - .map(v -> v.blockLoaderOutput != null ? v.blockLoaderOutput : v.output) - .toList(); - var outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - - return new MapperTestCase.SyntheticSourceExample(in, out, outBlock, this::mapping); + return new MapperTestCase.SyntheticSourceExample(in, out, this::mapping); } - private record Value(Object input, Object output, String blockLoaderOutput) { - Value(Object input, Object output) { - this(input, output, null); - } - } + private record Value(Object input, Object output) {} private Value generateValue() { if (ignoreMalformed && randomBoolean()) { @@ -130,16 +114,13 @@ public class GeometricShapeSyntheticSourceSupport implements MapperTestCase.Synt private Value value(Geometry geometry, boolean isGeoJson) { var wktString = WellKnownText.toWKT(geometry); - var normalizedWktString = fieldType == FieldType.GEO_SHAPE && GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry) - ? WellKnownText.toWKT(GeometryNormalizer.apply(Orientation.RIGHT, geometry)) - : wktString; if (isGeoJson) { var map = GeoJson.toMap(geometry); - return new Value(map, map, normalizedWktString); + return new Value(map, map); } - return new Value(wktString, wktString, normalizedWktString); + return new Value(wktString, wktString); } private void mapping(XContentBuilder b) throws IOException { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 8f28b462afca..ac9d6a0f5f87 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -11,11 +11,7 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.WellKnownBinary; -import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; @@ -32,8 +28,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; import static org.hamcrest.Matchers.containsString; @@ -448,37 +442,12 @@ public class PointFieldMapperTests extends CartesianFieldMapperTests { public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { Value v = generateValue(); - - if (v.point == null) { - return new SyntheticSourceExample(v.representation(), v.representation(), null, this::mapping); - } else if (columnReader) { - return new SyntheticSourceExample(v.representation(), v.representation(), encode(v.point()), this::mapping); - } - return new SyntheticSourceExample(v.representation(), v.representation(), v.point().toWKT(), this::mapping); + return new SyntheticSourceExample(v.representation(), v.representation(), this::mapping); } List values = randomList(1, maxVals, this::generateValue); var representations = values.stream().map(Value::representation).toList(); - if (columnReader) { - // When reading doc-values, the block is a list of encoded longs - List outBlockList = values.stream() - .map(Value::point) - .filter(Objects::nonNull) - .map(this::encode) - .sorted() - .toList(); - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(representations, representations, outBlock, this::mapping); - } else { - // When reading row-stride, the block is a list of WKT encoded BytesRefs - List outBlockList = values.stream() - .map(Value::point) - .filter(Objects::nonNull) - .map(CartesianPoint::toWKT) - .toList(); - Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; - return new SyntheticSourceExample(representations, representations, outBlock, this::mapping); - } + return new SyntheticSourceExample(representations, representations, this::mapping); } private record Value(CartesianPoint point, Object representation) {} @@ -567,37 +536,4 @@ public class PointFieldMapperTests extends CartesianFieldMapperTests { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } - - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - if (columnReader) { - // When using column reader, we expect the output to be doc-values (which means encoded longs) - return v -> asJacksonNumberOutput(((Number) v).longValue()); - } else { - // When using row-stride reader, we expect the output to be WKT encoded BytesRef - return v -> asWKT((BytesRef) v); - } - } - - protected static Object asJacksonNumberOutput(long l) { - // Cast to int to mimic jackson-core behaviour in NumberOutput.outputLong() - // that is called when deserializing expected value in SyntheticSourceExample. - if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) { - return (int) l; - } else { - return l; - } - } - - protected static Object asWKT(BytesRef value) { - // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability - Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes); - return WellKnownText.toWKT(geometry); - } - - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - MappedFieldType ft = mapper.fieldType(loaderFieldName); - return new BlockReaderSupport(ft.hasDocValues(), false, mapper, loaderFieldName); - } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 61491b88a8a0..ee6f6bc8b70c 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -8,13 +8,8 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.Orientation; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.utils.GeometryValidator; -import org.elasticsearch.geometry.utils.WellKnownBinary; -import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper; @@ -34,7 +29,6 @@ import org.junit.AssumptionViolatedException; import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; import static org.hamcrest.Matchers.containsString; @@ -374,23 +368,6 @@ public class ShapeFieldMapperTests extends CartesianFieldMapperTests { return new GeometricShapeSyntheticSourceSupport(GeometricShapeSyntheticSourceSupport.FieldType.SHAPE, ignoreMalformed); } - @Override - protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { - return v -> asWKT((BytesRef) v); - } - - protected static Object asWKT(BytesRef value) { - // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability - Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes); - return WellKnownText.toWKT(geometry); - } - - @Override - protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) { - // Synthetic source is currently not supported. - return new BlockReaderSupport(false, false, mapper, loaderFieldName); - } - @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index b7a9b8af057e..88c13ead9a59 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -85,7 +85,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.function.BiFunction; -import java.util.function.Function; import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; @@ -1223,11 +1222,6 @@ public class WildcardFieldMapperTests extends MapperTestCase { return new WildcardSyntheticSourceSupport(); } - @Override - protected Function loadBlockExpected() { - return v -> ((BytesRef) v).utf8ToString(); - } - static class WildcardSyntheticSourceSupport implements SyntheticSourceSupport { private final Integer ignoreAbove = randomBoolean() ? null : between(10, 100); private final boolean allIgnored = ignoreAbove != null && rarely(); @@ -1237,11 +1231,7 @@ public class WildcardFieldMapperTests extends MapperTestCase { public SyntheticSourceExample example(int maxValues) { if (randomBoolean()) { Tuple v = generateValue(); - Object loadBlock = v.v2(); - if (ignoreAbove != null && v.v2().length() > ignoreAbove) { - loadBlock = null; - } - return new SyntheticSourceExample(v.v1(), v.v2(), loadBlock, this::mapping); + return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping); } List> values = randomList(1, maxValues, this::generateValue); List in = values.stream().map(Tuple::v1).toList(); @@ -1256,11 +1246,9 @@ public class WildcardFieldMapperTests extends MapperTestCase { }); List outList = new ArrayList<>(new HashSet<>(docValuesValues)); Collections.sort(outList); - List outBlockList = List.copyOf(outList); - Object outBlockResult = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; outList.addAll(outExtraValues); Object out = outList.size() == 1 ? outList.get(0) : outList; - return new SyntheticSourceExample(in, out, outBlockResult, this::mapping); + return new SyntheticSourceExample(in, out, this::mapping); } private Tuple generateValue() {