From db5ddb321fc5c56b3152a83157ac4d9beab4c20d Mon Sep 17 00:00:00 2001 From: tmgordeeva Date: Fri, 26 Aug 2022 11:32:53 -0700 Subject: [PATCH] Synthetic source error on script loads (#88334) * Source Lookup refactor with error on script synthetic source load Refactors SourceLookup into a static source lookup used for most cases where we access the source again after the fetch phase, and a re-loading lookup used by scripts. The re-loading lookup now also fails with an error when we are using synthetic source preventing silent failures or non-sensical behavior from scripts. --- .../script/ScriptScoreBenchmark.java | 4 +- .../subphase/FetchSourcePhaseBenchmark.java | 3 +- docs/changelog/88334.yaml | 5 + .../ExpressionFieldScriptTests.java | 7 +- .../ExpressionNumberSortScriptTests.java | 7 +- .../ExpressionTermsSetQueryTests.java | 7 +- .../PercolatorHighlightSubFetchPhase.java | 3 +- .../search/fields/SearchFieldsIT.java | 3 +- .../search/stats/SearchStatsIT.java | 3 +- .../action/update/UpdateHelper.java | 3 +- .../index/mapper/DocumentParser.java | 4 +- .../index/mapper/MappingLookup.java | 15 + .../index/mapper/NestedValueFetcher.java | 3 +- .../index/mapper/SourceLoader.java | 2 +- .../index/query/SearchExecutionContext.java | 7 +- .../index/termvectors/TermVectorsService.java | 3 +- .../search/fetch/FetchPhase.java | 8 +- .../search/fetch/FetchSubPhase.java | 2 +- .../search/fetch/subphase/FieldFetcher.java | 20 +- .../search/lookup/SearchLookup.java | 5 +- .../search/lookup/SourceLookup.java | 384 +++++++++++++----- .../fielddata/IndexFieldDataServiceTests.java | 3 +- .../AbstractScriptFieldTypeTestCase.java | 12 +- .../index/mapper/BooleanFieldScriptTests.java | 3 +- .../mapper/BooleanScriptFieldTypeTests.java | 4 +- .../mapper/CompositeRuntimeFieldTests.java | 4 +- .../index/mapper/DateFieldScriptTests.java | 5 +- .../mapper/DateScriptFieldTypeTests.java | 4 +- .../index/mapper/DoubleFieldScriptTests.java | 5 +- .../mapper/DoubleScriptFieldTypeTests.java | 4 +- .../mapper/GeoPointFieldScriptTests.java | 3 +- .../mapper/GeoPointScriptFieldTypeTests.java | 2 +- .../index/mapper/IgnoredFieldMapperTests.java | 4 +- .../index/mapper/IndexFieldMapperTests.java | 7 +- .../index/mapper/IpFieldScriptTests.java | 5 +- .../index/mapper/IpScriptFieldTypeTests.java | 4 +- .../mapper/KeywordScriptFieldTypeTests.java | 22 +- .../index/mapper/LongFieldScriptTests.java | 5 +- .../mapper/LongScriptFieldTypeTests.java | 6 +- .../mapper/LookupRuntimeFieldTypeTests.java | 6 +- .../mapper/PlaceHolderFieldMapperTests.java | 7 +- .../mapper/ProvidedIdFieldMapperTests.java | 4 +- .../index/mapper/RoutingFieldMapperTests.java | 4 +- .../index/mapper/StringFieldScriptTests.java | 9 +- .../index/mapper/VersionFieldMapperTests.java | 7 +- .../KeyedFlattenedFieldTypeTests.java | 5 +- .../script/CompositeFieldScriptTests.java | 7 +- ...dNumericDocValuesLongFieldScriptTests.java | 3 +- ...tedSetDocValuesStringFieldScriptTests.java | 3 +- .../bucket/range/RangeAggregatorTests.java | 3 +- .../bucket/terms/TermsAggregatorTests.java | 3 +- .../support/ScriptValuesTests.java | 3 +- .../fetch/subphase/FetchSourcePhaseTests.java | 6 +- .../fetch/subphase/FieldFetcherTests.java | 14 +- .../search/lookup/SourceLookupTests.java | 15 +- .../search/query/ScriptScoreQueryTests.java | 3 +- ...tScriptFieldDistanceFeatureQueryTests.java | 5 +- ...gScriptFieldDistanceFeatureQueryTests.java | 5 +- .../index/mapper/FieldTypeTestCase.java | 6 +- .../index/mapper/MapperTestCase.java | 11 +- .../mapper/DataTierFieldTypeTests.java | 2 +- .../AggregateDoubleMetricFieldTypeTests.java | 4 +- .../mapper/ConstantKeywordFieldTypeTests.java | 5 +- 63 files changed, 520 insertions(+), 220 deletions(-) create mode 100644 docs/changelog/88334.yaml diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 23d342079cdb..47f1b4f41353 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -40,6 +40,7 @@ import org.elasticsearch.script.DocValuesDocReader; import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -88,7 +89,8 @@ public class ScriptScoreBenchmark { private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private final SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService) + (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService), + new SourceLookup.ReaderSourceProvider() ); @Param({ "expression", "metal", "painless_cast", "painless_def" }) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java index a2eba3ac6833..29947be4b877 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -79,8 +79,7 @@ public class FetchSourcePhaseBenchmark { @Benchmark public BytesReference filterObjects() throws IOException { - SourceLookup lookup = new SourceLookup(); - lookup.setSource(sourceBytes); + SourceLookup lookup = new SourceLookup(new SourceLookup.BytesSourceProvider(sourceBytes)); Object value = lookup.filter(fetchContext); return FetchSourcePhase.objectToBytes(value, XContentType.JSON, Math.min(1024, lookup.internalSourceRef().length())); } diff --git a/docs/changelog/88334.yaml b/docs/changelog/88334.yaml new file mode 100644 index 000000000000..b464c468115b --- /dev/null +++ b/docs/changelog/88334.yaml @@ -0,0 +1,5 @@ +pr: 88334 +summary: Synthetic source error on script loads +area: Geo +type: bug +issues: [] diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index 68c58d8ee7de..6d843e7b8065 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.script.FieldScript; import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -49,7 +50,11 @@ public class ExpressionFieldScriptTests extends ESTestCase { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup, fdt) -> fieldData, + new SourceLookup.ReaderSourceProvider() + ); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 01aec0e0e3b2..30e94ed37947 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.script.DocValuesDocReader; import org.elasticsearch.script.NumberSortScript; import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -50,7 +51,11 @@ public class ExpressionNumberSortScriptTests extends ESTestCase { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup, fdt) -> fieldData, + new SourceLookup.ReaderSourceProvider() + ); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index 60a1d6e64551..ad547c656f4b 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.TermsSetQueryScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -49,7 +50,11 @@ public class ExpressionTermsSetQueryTests extends ESTestCase { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup, fdt) -> fieldData, + new SourceLookup.ReaderSourceProvider() + ); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index 4a8e087074d4..f0c3bde40241 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -86,7 +87,7 @@ final class PercolatorHighlightSubFetchPhase implements FetchSubPhase { percolatorLeafReaderContext, slot ); - subContext.sourceLookup().setSource(document); + subContext.sourceLookup().setSourceProvider(new SourceLookup.BytesSourceProvider(document)); // force source because MemoryIndex does not store fields SearchHighlightContext highlight = new SearchHighlightContext(fetchContext.highlight().fields(), true); FetchSubPhaseProcessor processor = highlightPhase.getProcessor(fetchContext, highlight, query); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 317a9e5304f1..ecc161f3de23 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.lookup.FieldLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -142,7 +143,7 @@ public class SearchFieldsIT extends ESIntegTestCase { static Object sourceScript(Map vars, String path) { @SuppressWarnings("unchecked") - Map source = (Map) vars.get("_source"); + Map source = ((SourceLookup) vars.get("_source")).source(); return XContentMapValues.extractValue(path, source); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java index cd5fddf3613f..2273d46caf82 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESIntegTestCase; import java.util.Collection; @@ -61,7 +62,7 @@ public class SearchStatsIT extends ESIntegTestCase { @Override protected Map, Object>> pluginScripts() { return Collections.singletonMap("_source.field", vars -> { - Map src = (Map) vars.get("_source"); + Map src = ((SourceLookup) vars.get("_source")).source(); return src.get("field"); }); } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index daaa69f5786a..16e6a430e92b 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -332,8 +332,7 @@ public class UpdateHelper { BytesReference sourceFilteredAsBytes = sourceAsBytes; if (request.fetchSource().includes().length > 0 || request.fetchSource().excludes().length > 0) { - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(source); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.MapSourceProvider(source)); Object value = sourceLookup.filter(request.fetchSource()); try { final int initialCapacity = sourceAsBytes != null ? Math.min(1024, sourceAsBytes.length()) : 1024; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index d79f07d9cac2..cdfd37f4bf26 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -144,7 +145,8 @@ public final class DocumentParser { context.mappingLookup().indexTimeLookup()::get, (ft, lookup, fto) -> ft.fielddataBuilder( new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto) - ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + new SourceLookup.ReaderSourceProvider() ); // field scripts can be called both by the loop at the end of this method and via // the document reader, so to ensure that we don't run them multiple times we diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index a45fa7ff0e24..ee15f5e273fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.search.lookup.SourceLookup; import java.util.ArrayList; import java.util.Collection; @@ -470,4 +471,18 @@ public final class MappingLookup { throw new MapperParsingException("Field [" + name + "] attempted to shadow a time_series_metric"); } } + + /** + * Returns a SourceProvider describing how to read the source. If using synthetic source, returns the null source provider + * expecting the source to either be provided later by a fetch phase or not be accessed at all (as in scripts). + * @return + */ + public SourceLookup.SourceProvider getSourceProvider() { + SourceFieldMapper sourceMapper = (SourceFieldMapper) getMapper("_source"); + if (sourceMapper == null || sourceMapper.isSynthetic() == false) { + return new SourceLookup.ReaderSourceProvider(); + } else { + return new SourceLookup.NullSourceProvider(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java index f12669ded80e..1947fa48fb9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java @@ -50,8 +50,7 @@ public class NestedValueFetcher implements ValueFetcher { for (Object entry : nestedValues) { // add this one entry only to the stub and use this as source lookup stub.put(nestedFieldName, entry); - SourceLookup nestedSourceLookup = new SourceLookup(); - nestedSourceLookup.setSource(filteredSource); + SourceLookup nestedSourceLookup = new SourceLookup(new SourceLookup.MapSourceProvider(filteredSource)); Map fetchResult = nestedFieldFetcher.fetch(nestedSourceLookup); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index e7d4f54abd1b..a55e4befa41f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -83,7 +83,7 @@ public interface SourceLoader { }; /** - * Load {@code _source} from doc vales. + * Load {@code _source} from doc values. */ class Synthetic implements SourceLoader { private final SyntheticFieldLoader loader; diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index f1514a959a31..ae42e2ba366e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -59,6 +59,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -481,12 +482,16 @@ public class SearchExecutionContext extends QueryRewriteContext { */ public SearchLookup lookup() { if (this.lookup == null) { + SourceLookup.SourceProvider sourceProvider = mappingLookup == null + ? new SourceLookup.ReaderSourceProvider() + : mappingLookup.getSourceProvider(); this.lookup = new SearchLookup( this::getFieldType, (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply( fieldType, new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation) - ) + ), + sourceProvider ); } return this.lookup; diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 3852e40e800f..7b162f4f9d2a 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -272,8 +272,7 @@ public class TermVectorsService { } if (source != null) { MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(source); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.MapSourceProvider(source)); for (String field : fields) { if (values.containsKey(field) == false) { SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field)); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 6b2fddd7fa0e..a74c48b9499c 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -383,11 +383,12 @@ public class FetchPhase { if (source != null) { // Store the loaded source on the hit context so that fetch subphases can access it. // Also make it available to scripts by storing it on the shared SearchLookup instance. - hitContext.sourceLookup().setSource(source); + SourceLookup.BytesSourceProvider sourceBytes = new SourceLookup.BytesSourceProvider(source); + hitContext.sourceLookup().setSourceProvider(sourceBytes); SourceLookup scriptSourceLookup = context.getSearchExecutionContext().lookup().source(); scriptSourceLookup.setSegmentAndDocument(subReaderContext, subDocId); - scriptSourceLookup.setSource(source); + scriptSourceLookup.setSourceProvider(sourceBytes); } return hitContext; } @@ -486,8 +487,7 @@ public class FetchPhase { } } - hitContext.sourceLookup().setSource(nestedSourceAsMap); - hitContext.sourceLookup().setSourceContentType(rootSourceContentType); + hitContext.sourceLookup().setSourceProvider(new SourceLookup.MapSourceProvider(nestedSourceAsMap, rootSourceContentType)); } return hitContext; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 8ab13515167e..11155c55e99f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -31,7 +31,7 @@ public interface FetchSubPhase { this.hit = hit; this.readerContext = context; this.docId = docId; - this.sourceLookup = new SourceLookup(); + this.sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); sourceLookup.setSegmentAndDocument(context, docId); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index 4a8f341837a9..8f54ec24eef3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -29,6 +29,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -172,15 +173,20 @@ public class FieldFetcher { documentFields.put(field, docField); } } - collectUnmapped(documentFields, sourceLookup, "", 0); + collectUnmapped(documentFields, () -> sourceLookup.source(), "", 0); return documentFields; } - private void collectUnmapped(Map documentFields, Map source, String parentPath, int lastState) { + private void collectUnmapped( + Map documentFields, + Supplier> source, + String parentPath, + int lastState + ) { // lookup field patterns containing wildcards if (this.unmappedFieldsFetchAutomaton != null) { - for (String key : source.keySet()) { - Object value = source.get(key); + for (String key : source.get().keySet()) { + Object value = source.get().get(key); String currentPath = parentPath + key; if (this.fieldContexts.containsKey(currentPath)) { continue; @@ -196,7 +202,7 @@ public class FieldFetcher { Map objectMap = (Map) value; collectUnmapped( documentFields, - objectMap, + () -> objectMap, currentPath + ".", step(this.unmappedFieldsFetchAutomaton, ".", currentState) ); @@ -227,7 +233,7 @@ public class FieldFetcher { if (this.fieldContexts.containsKey(path)) { continue; // this is actually a mapped field } - List values = XContentMapValues.extractRawValues(path, source); + List values = XContentMapValues.extractRawValues(path, source.get()); if (values.isEmpty() == false) { documentFields.put(path, new DocumentField(path, values)); } @@ -241,7 +247,7 @@ public class FieldFetcher { if (value instanceof Map) { @SuppressWarnings("unchecked") final Map objectMap = (Map) value; - collectUnmapped(documentFields, objectMap, parentPath + ".", step(this.unmappedFieldsFetchAutomaton, ".", lastState)); + collectUnmapped(documentFields, () -> objectMap, parentPath + ".", step(this.unmappedFieldsFetchAutomaton, ".", lastState)); } else if (value instanceof List) { // weird case, but can happen for objects with "enabled" : "false" collectUnmappedList(documentFields, (List) value, parentPath, lastState); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 3b1e53d3061b..264e38e21e96 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -53,11 +53,12 @@ public class SearchLookup { */ public SearchLookup( Function fieldTypeLookup, - TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup, + SourceLookup.SourceProvider sourceProvider ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); - this.sourceLookup = new SourceLookup(); + this.sourceLookup = new SourceLookup(sourceProvider); this.fieldDataLookup = fieldDataLookup; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 46322e7f7c6e..337c82fce935 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -25,26 +25,23 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import static java.util.Collections.emptyMap; - public class SourceLookup implements Map { - - private LeafReader reader; - private CheckedBiConsumer fieldReader; + private SourceProvider sourceProvider; private int docId = -1; - private BytesReference sourceAsBytes; - private Map source; - private XContentType sourceContentType; + public SourceLookup(SourceProvider sourceProvider) { + this.sourceProvider = sourceProvider; + } public XContentType sourceContentType() { - return sourceContentType; + return sourceProvider.sourceContentType(); } public int docId() { @@ -62,101 +59,30 @@ public class SourceLookup implements Map { * number. */ public Map source() { - if (source != null) { - return source; - } - if (sourceAsBytes != null) { - Tuple> tuple = sourceAsMapAndType(sourceAsBytes); - sourceContentType = tuple.v1(); - source = tuple.v2(); - return source; - } - try { - FieldsVisitor sourceFieldVisitor = new FieldsVisitor(true); - fieldReader.accept(docId, sourceFieldVisitor); - BytesReference source = sourceFieldVisitor.source(); - if (source == null) { - this.source = emptyMap(); - this.sourceContentType = null; - } else { - Tuple> tuple = sourceAsMapAndType(source); - this.sourceContentType = tuple.v1(); - this.source = tuple.v2(); - } - } catch (Exception e) { - throw new ElasticsearchParseException("failed to parse / load source", e); - } - return this.source; + return sourceProvider.source(); } - private static Tuple> sourceAsMapAndType(BytesReference source) throws ElasticsearchParseException { - return XContentHelper.convertToMap(source, false); - } - - /** - * Get the source as a {@link Map} of java objects. - *

- * Important: This can lose precision on numbers with a decimal point. It - * converts numbers like {@code "n": 1234.567} to a {@code double} which - * only has 52 bits of precision in the mantissa. This will come up most - * frequently when folks write nanosecond precision dates as a decimal - * number. - */ - public static Map sourceAsMap(BytesReference source) throws ElasticsearchParseException { - return sourceAsMapAndType(source).v2(); + public void setSourceProvider(SourceProvider sourceProvider) { + this.sourceProvider = sourceProvider; } public void setSegmentAndDocument(LeafReaderContext context, int docId) { - // if we are called with the same document, don't invalidate source - if (this.reader == context.reader() && this.docId == docId) { - return; - } - - // only reset reader and fieldReader when reader changes - if (this.reader != context.reader()) { - this.reader = context.reader(); - - // All the docs to fetch are adjacent but Lucene stored fields are optimized - // for random access and don't optimize for sequential access - except for merging. - // So we do a little hack here and pretend we're going to do merges in order to - // get better sequential access. - if (context.reader()instanceof SequentialStoredFieldsLeafReader lf) { - // Avoid eagerly loading the stored fields reader, since this can be expensive - Supplier supplier = new MemoizedSupplier<>(lf::getSequentialStoredFieldsReader); - fieldReader = (d, v) -> supplier.get().visitDocument(d, v); - } else { - fieldReader = context.reader()::document; - } - } - this.source = null; - this.sourceAsBytes = null; + sourceProvider.setSegmentAndDocument(context, docId); this.docId = docId; } - public void setSource(BytesReference source) { - this.sourceAsBytes = source; - } - - public void setSourceContentType(XContentType sourceContentType) { - this.sourceContentType = sourceContentType; - } - - public void setSource(Map source) { - this.source = source; - } - /** * Internal source representation, might be compressed.... */ public BytesReference internalSourceRef() { - return sourceAsBytes; + return sourceProvider.sourceAsBytes(); } /** * Checks if the source has been deserialized as a {@link Map} of java objects. */ public boolean hasSourceAsMap() { - return source != null; + return sourceProvider.hasSourceAsMap(); } /** @@ -174,23 +100,7 @@ public class SourceLookup implements Map { * @return The list of found values or an empty list if none are found */ public List extractRawValuesWithoutCaching(String path) { - if (source != null) { - return XContentMapValues.extractRawValues(path, source); - } - if (sourceAsBytes != null) { - return XContentMapValues.extractRawValues( - path, - XContentHelper.convertToMap(sourceAsBytes, false, null, Set.of(path), null).v2() - ); - } - try { - FieldsVisitor sourceFieldVisitor = new FieldsVisitor(true); - fieldReader.accept(docId, sourceFieldVisitor); - BytesReference source = sourceFieldVisitor.source(); - return XContentMapValues.extractRawValues(path, XContentHelper.convertToMap(source, false, null, Set.of(path), null).v2()); - } catch (Exception e) { - throw new ElasticsearchParseException("failed to parse / load source", e); - } + return sourceProvider.extractRawValuesWithoutCaching(path); } /** @@ -212,6 +122,23 @@ public class SourceLookup implements Map { return context.getFilter().apply(source()); } + private static Tuple> sourceAsMapAndType(BytesReference source) throws ElasticsearchParseException { + return XContentHelper.convertToMap(source, false); + } + + /** + * Get the source as a {@link Map} of java objects. + *

+ * Important: This can lose precision on numbers with a decimal point. It + * converts numbers like {@code "n": 1234.567} to a {@code double} which + * only has 52 bits of precision in the mantissa. This will come up most + * frequently when folks write nanosecond precision dates as a decimal + * number. + */ + public static Map sourceAsMap(BytesReference source) throws ElasticsearchParseException { + return sourceAsMapAndType(source).v2(); + } + @Override public Object get(Object key) { return source().get(key); @@ -272,4 +199,255 @@ public class SourceLookup implements Map { public void clear() { throw new UnsupportedOperationException(); } + + /** + * SourceProvider describes how SourceLookup will access the source. + */ + public interface SourceProvider { + Map source(); + + XContentType sourceContentType(); + + BytesReference sourceAsBytes(); + + List extractRawValuesWithoutCaching(String path); + + boolean hasSourceAsMap(); + + void setSegmentAndDocument(LeafReaderContext context, int docId); + } + + /** + * Null source provider when we expect to be given a real source provider in the future. + * Exceptions expected to be primarily seen by scripts attempting to access synthetic source. + */ + public static class NullSourceProvider implements SourceProvider { + @Override + public Map source() { + throw new IllegalArgumentException("no source available"); + } + + @Override + public XContentType sourceContentType() { + throw new IllegalArgumentException("no source available"); + } + + @Override + public BytesReference sourceAsBytes() { + throw new IllegalArgumentException("no source available"); + } + + @Override + public List extractRawValuesWithoutCaching(String path) { + throw new IllegalArgumentException("no source available"); + } + + @Override + public boolean hasSourceAsMap() { + return false; + } + + @Override + public void setSegmentAndDocument(LeafReaderContext context, int docId) { + // + } + } + + /** + * Provider for source using a given map and optional content type. + */ + public static class MapSourceProvider implements SourceProvider { + private Map source; + private XContentType sourceContentType; + + public MapSourceProvider(Map source) { + this.source = source; + } + + public MapSourceProvider(Map source, @Nullable XContentType sourceContentType) { + this.source = source; + this.sourceContentType = sourceContentType; + } + + @Override + public Map source() { + return source; + } + + @Override + public XContentType sourceContentType() { + return sourceContentType; + } + + @Override + public BytesReference sourceAsBytes() { + throw new UnsupportedOperationException("source as bytes unavailable - empty or already parsed to map"); + } + + @Override + public List extractRawValuesWithoutCaching(String path) { + return XContentMapValues.extractRawValues(path, source); + } + + @Override + public boolean hasSourceAsMap() { + return true; + } + + @Override + public void setSegmentAndDocument(LeafReaderContext context, int docId) { + // + } + } + + /** + * Provider for source using given source bytes. + */ + public static class BytesSourceProvider implements SourceProvider { + private BytesReference sourceAsBytes; + private Map source; + private XContentType sourceContentType; + + public BytesSourceProvider(BytesReference sourceAsBytes) { + this.sourceAsBytes = sourceAsBytes; + } + + void parseSource() { + Tuple> tuple = sourceAsMapAndType(sourceAsBytes); + this.source = tuple.v2(); + this.sourceContentType = tuple.v1(); + } + + @Override + public Map source() { + if (source == null) { + parseSource(); + } + return source; + } + + @Override + public XContentType sourceContentType() { + if (source == null) { + parseSource(); + } + return sourceContentType; + } + + @Override + public BytesReference sourceAsBytes() { + return sourceAsBytes; + } + + @Override + public List extractRawValuesWithoutCaching(String path) { + if (source != null) { + return XContentMapValues.extractRawValues(path, source); + } + return XContentMapValues.extractRawValues( + path, + XContentHelper.convertToMap(sourceAsBytes, false, null, Set.of(path), null).v2() + ); + } + + @Override + public boolean hasSourceAsMap() { + return source != null; + } + + @Override + public void setSegmentAndDocument(LeafReaderContext context, int docId) { + // + } + } + + /** + * Provider for source using a fields visitor. + */ + public static class ReaderSourceProvider implements SourceProvider { + private LeafReader reader; + private CheckedBiConsumer fieldReader; + private int docId = -1; + + private SourceProvider sourceProvider; + + public void setSegmentAndDocument(LeafReaderContext context, int docId) { + // if we are called with the same document, don't invalidate source + if (this.reader == context.reader() && this.docId == docId) { + return; + } + + // only reset reader and fieldReader when reader changes + if (this.reader != context.reader()) { + this.reader = context.reader(); + + // All the docs to fetch are adjacent but Lucene stored fields are optimized + // for random access and don't optimize for sequential access - except for merging. + // So we do a little hack here and pretend we're going to do merges in order to + // get better sequential access. + if (context.reader()instanceof SequentialStoredFieldsLeafReader lf) { + // Avoid eagerly loading the stored fields reader, since this can be expensive + Supplier supplier = new MemoizedSupplier<>(lf::getSequentialStoredFieldsReader); + fieldReader = (d, v) -> supplier.get().visitDocument(d, v); + } else { + fieldReader = context.reader()::document; + } + } + this.sourceProvider = null; + this.docId = docId; + } + + @Override + public Map source() { + if (sourceProvider == null) { + readSourceBytes(); + } + return sourceProvider.source(); + } + + private void readSourceBytes() { + try { + FieldsVisitor sourceFieldVisitor = new FieldsVisitor(true); + fieldReader.accept(docId, sourceFieldVisitor); + BytesReference sourceAsBytes = sourceFieldVisitor.source(); + if (sourceAsBytes == null) { + sourceProvider = new MapSourceProvider(Collections.emptyMap()); + } else { + sourceProvider = new BytesSourceProvider(sourceAsBytes); + ((BytesSourceProvider) sourceProvider).parseSource(); + } + } catch (Exception e) { + throw new ElasticsearchParseException("failed to parse / load source", e); + } + } + + @Override + public XContentType sourceContentType() { + if (sourceProvider == null) { + readSourceBytes(); + } + return sourceProvider.sourceContentType(); + } + + @Override + public BytesReference sourceAsBytes() { + if (sourceProvider == null) { + readSourceBytes(); + } + return sourceProvider.sourceAsBytes(); + } + + @Override + public List extractRawValuesWithoutCaching(String path) { + if (sourceProvider == null) { + readSourceBytes(); + } + return sourceProvider.extractRawValuesWithoutCaching(path); + } + + @Override + public boolean hasSourceAsMap() { + return sourceProvider != null && sourceProvider.hasSourceAsMap(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 282e5009d6d0..b56fb251a3f7 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.InternalSettingsPlugin; @@ -135,7 +136,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { searchLookupSetOnce.set(fdc.lookupSupplier()); return (IndexFieldData.Builder) (cache, breakerService) -> null; }); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH)); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 7ef32966aa13..5f979c1cb58c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -29,6 +29,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -203,6 +204,14 @@ public abstract class AbstractScriptFieldTypeTestCase extends MapperServiceTestC } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) { + return mockContext(allowExpensiveQueries, mappedFieldType, new SourceLookup.ReaderSourceProvider()); + } + + protected static SearchExecutionContext mockContext( + boolean allowExpensiveQueries, + MappedFieldType mappedFieldType, + SourceLookup.SourceProvider sourceProvider + ) { SearchExecutionContext context = mock(SearchExecutionContext.class); if (mappedFieldType != null) { when(context.getFieldType(anyString())).thenReturn(mappedFieldType); @@ -211,7 +220,8 @@ public abstract class AbstractScriptFieldTypeTestCase extends MapperServiceTestC SearchLookup lookup = new SearchLookup( context::getFieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo)) - .build(null, null) + .build(null, null), + sourceProvider ); when(context.lookup()).thenReturn(lookup); when(context.getForField(any(), any())).then(args -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java index 44b66a9e89bd..082c4f79354d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.BooleanFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.List; @@ -57,7 +58,7 @@ public class BooleanFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 5588333465ee..89276437796c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -422,7 +422,7 @@ public class BooleanScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeT case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new BooleanFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit((Boolean) foo); } } @@ -430,7 +430,7 @@ public class BooleanScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeT case "xor_param" -> (fieldName, params, lookup) -> (ctx) -> new BooleanFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit((Boolean) foo ^ ((Boolean) getParams().get("param"))); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 9477c34ddb88..b2ce1482ce02 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -336,7 +337,8 @@ public class CompositeRuntimeFieldTests extends MapperServiceTestCase { mapperService::fieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) - ).build(null, null) + ).build(null, null), + new SourceLookup.ReaderSourceProvider() ); LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index 07383b297963..17e3578a3ea1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.DateFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -66,7 +67,7 @@ public class DateFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())), reader.leaves().get(0) ) { @@ -102,7 +103,7 @@ public class DateFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), DateFormatter.forPattern("epoch_millis") ); DateFieldScript dateFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index cc5537decf2a..a722a1c02211 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -489,7 +489,7 @@ public class DateScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTest ) { @Override public void execute() { - for (Object timestamp : (List) lookup.source().get("timestamp")) { + for (Object timestamp : (List) lookup.source().source().get("timestamp")) { Parse parse = new Parse(this); emit(parse.parse(timestamp)); } @@ -504,7 +504,7 @@ public class DateScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTest ) { @Override public void execute() { - for (Object timestamp : (List) lookup.source().get("timestamp")) { + for (Object timestamp : (List) lookup.source().source().get("timestamp")) { long epoch = (Long) timestamp; ZonedDateTime dt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(epoch), ZoneId.of("UTC")); dt = dt.plus(((Number) params.get("days")).longValue(), ChronoUnit.DAYS); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index 0b45c56581d7..29b68d8f3085 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -64,7 +65,7 @@ public class DoubleFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -99,7 +100,7 @@ public class DoubleFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()) ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index 9b1fba8915e3..02eee8af5050 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -245,7 +245,7 @@ public class DoubleScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTe case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new DoubleFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(((Number) foo).doubleValue()); } } @@ -253,7 +253,7 @@ public class DoubleScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTe case "add_param" -> (fieldName, params, lookup) -> (ctx) -> new DoubleFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(((Number) foo).doubleValue() + ((Number) getParams().get("param")).doubleValue()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java index 93e32df4d10f..bb8f244740e0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.List; @@ -59,7 +60,7 @@ public class GeoPointFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java index 0ae0796e2d28..6537e67cbbde 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java @@ -235,7 +235,7 @@ public class GeoPointScriptFieldTypeTests extends AbstractNonTextScriptFieldType case "fromLatLon" -> (fieldName, params, lookup) -> (ctx) -> new GeoPointFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - Map foo = (Map) lookup.source().get("foo"); + Map foo = (Map) lookup.source().source().get("foo"); emit(((Number) foo.get("lat")).doubleValue(), ((Number) foo.get("lon")).doubleValue()); } }; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index 3949d590b82e..836f6fe3d8af 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -52,7 +53,8 @@ public class IgnoredFieldMapperTests extends MapperServiceTestCase { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 767118aec554..84ada85884f2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Collections; @@ -47,7 +48,11 @@ public class IndexFieldMapperTests extends MapperServiceTestCase { iw.addDocument(mapperService.documentMapper().parse(source).rootDoc()); }, iw -> { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index c73528410833..3b7a35d8971f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.IpFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -65,7 +66,7 @@ public class IpFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -100,7 +101,7 @@ public class IpFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()) ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index e1eb5cad3f1c..72a491188c8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -262,7 +262,7 @@ public class IpScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase { case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new IpFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(foo.toString()); } } @@ -270,7 +270,7 @@ public class IpScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase { case "append_param" -> (fieldName, params, lookup) -> (ctx) -> new IpFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(foo.toString() + getParams().get("param")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 309dcca1ba05..012c6bdb9524 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -292,6 +293,23 @@ public class KeywordScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase } } + public void testSyntheticSourceAccess() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-suffix")); + expectThrows( + IllegalArgumentException.class, + () -> { + searcher.count(fieldType.termQuery("1-suffix", mockContext(true, null, new SourceLookup.NullSourceProvider()))); + } + ); + } + } + } + @Override protected Query randomTermQuery(MappedFieldType ft, SearchExecutionContext ctx) { return ft.termQuery(randomAlphaOfLengthBetween(1, 1000), ctx); @@ -389,7 +407,7 @@ public class KeywordScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase case "read_foo" -> (fieldName, params, lookup) -> ctx -> new StringFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(foo.toString()); } } @@ -397,7 +415,7 @@ public class KeywordScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase case "append_param" -> (fieldName, params, lookup) -> ctx -> new StringFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(foo.toString() + getParams().get("param").toString()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index 859374ea8de6..7aab950f0eaa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -64,7 +65,7 @@ public class LongFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -99,7 +100,7 @@ public class LongFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()) ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 7ad0a18c9d1a..1fa4445fbf23 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -278,7 +278,7 @@ public class LongScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTest return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(((Number) foo).longValue()); } } @@ -287,7 +287,7 @@ public class LongScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTest return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object foo : (List) lookup.source().get("foo")) { + for (Object foo : (List) lookup.source().source().get("foo")) { emit(((Number) foo).longValue() + ((Number) getParams().get("param")).longValue()); } } @@ -298,7 +298,7 @@ public class LongScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTest return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) { @Override public void execute() { - for (Object timestamp : (List) lookup.source().get("timestamp")) { + for (Object timestamp : (List) lookup.source().source().get("timestamp")) { emit(now - ((Number) timestamp).longValue()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LookupRuntimeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LookupRuntimeFieldTypeTests.java index 186d8ee543a0..40d8400c6315 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LookupRuntimeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LookupRuntimeFieldTypeTests.java @@ -55,8 +55,7 @@ public class LookupRuntimeFieldTypeTests extends MapperServiceTestCase { """; var mapperService = createMapperService(mapping); XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("foo", List.of("f1", "f2")).endObject(); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.BytesSourceProvider(BytesReference.bytes(source))); MappedFieldType fieldType = mapperService.fieldType("foo_lookup_field"); ValueFetcher valueFetcher = fieldType.valueFetcher(createSearchExecutionContext(mapperService), null); DocumentField doc = valueFetcher.fetchDocumentField("foo_lookup_field", sourceLookup); @@ -111,8 +110,7 @@ public class LookupRuntimeFieldTypeTests extends MapperServiceTestCase { source.field("foo", List.of()); } source.endObject(); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.BytesSourceProvider(BytesReference.bytes(source))); MappedFieldType fieldType = mapperService.fieldType("foo_lookup_field"); ValueFetcher valueFetcher = fieldType.valueFetcher(createSearchExecutionContext(mapperService), null); DocumentField doc = valueFetcher.fetchDocumentField("foo_lookup_field", sourceLookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index 36cf482507b2..9d726f46154b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FieldFetcher; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import java.util.Collections; @@ -61,7 +62,11 @@ public class PlaceHolderFieldMapperTests extends MapperServiceTestCase { .rootDoc() ); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( searchExecutionContext, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 551ad396e06e..4df0c9823ec8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -77,7 +78,8 @@ public class ProvidedIdFieldMapperTests extends MapperServiceTestCase { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 85eb59df7c44..0ab5c7854c33 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; @@ -70,7 +71,8 @@ public class RoutingFieldMapperTests extends MetadataMapperTestCase { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index a01234c04af7..6d55a3e33917 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -63,7 +64,7 @@ public class StringFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -89,7 +90,7 @@ public class StringFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -129,7 +130,7 @@ public class StringFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); @@ -158,7 +159,7 @@ public class StringFieldScriptTests extends FieldScriptTestCase null, (ft, lookup, fdt) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index 54714ee863b6..b05697088f3d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Collections; @@ -50,7 +51,11 @@ public class VersionFieldMapperTests extends MapperServiceTestCase { iw.addDocument(parsedDoc.rootDoc()); }, iw -> { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index 3738152037b3..b78bfb5f93e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -180,11 +180,10 @@ public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase { when(searchExecutionContext.sourcePath("field.key")).thenReturn(Set.of("field.key")); ValueFetcher fetcher = ft.valueFetcher(searchExecutionContext, null); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap("field", sourceValue)); + SourceLookup lookup = new SourceLookup(new SourceLookup.MapSourceProvider(Collections.singletonMap("field", sourceValue))); assertEquals(List.of("value"), fetcher.fetchValues(lookup, new ArrayList())); - lookup.setSource(Collections.singletonMap("field", null)); + lookup.setSourceProvider(new SourceLookup.MapSourceProvider(Collections.singletonMap("field", null))); assertEquals(List.of(), fetcher.fetchValues(lookup, new ArrayList())); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.valueFetcher(searchExecutionContext, "format")); diff --git a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java index fa7bcf308409..9fb7d6094c57 100644 --- a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -31,7 +32,7 @@ public class CompositeFieldScriptTests extends ESTestCase { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, ftd) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -62,7 +63,7 @@ public class CompositeFieldScriptTests extends ESTestCase { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, ftd) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override @@ -75,7 +76,7 @@ public class CompositeFieldScriptTests extends ESTestCase { StringFieldScript stringFieldScript = new StringFieldScript( "composite.leaf", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, ftd) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java index 384565975013..a449775cb253 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java @@ -18,6 +18,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -39,7 +40,7 @@ public class SortedNumericDocValuesLongFieldScriptTests extends ESTestCase { try (DirectoryReader reader = iw.getReader()) { SortedNumericDocValuesLongFieldScript docValues = new SortedNumericDocValuesLongFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup, ftd) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java index 8a36c68b2d65..d4385b261abd 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java @@ -19,6 +19,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.script.AbstractFieldScript; import org.elasticsearch.script.SortedSetDocValuesStringFieldScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -40,7 +41,7 @@ public class SortedSetDocValuesStringFieldScriptTests extends ESTestCase { try (DirectoryReader reader = iw.getReader()) { SortedSetDocValuesStringFieldScript docValues = new SortedSetDocValuesStringFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup, ftd) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 9859809f46fa..e7604e003d46 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.runtime.StringScriptFieldTermQuery; import java.io.IOException; @@ -635,7 +636,7 @@ public class RangeAggregatorTests extends AggregatorTestCase { */ public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException { long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, new SourceLookup.ReaderSourceProvider()); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 10a03c06cd59..4f3f79701743 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -111,6 +111,7 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.runtime.StringScriptFieldTermQuery; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -2044,7 +2045,7 @@ public class TermsAggregatorTests extends AggregatorTestCase { */ public void testRuntimeFieldTopLevelNotOptimized() throws IOException { long totalDocs = 500; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, new SourceLookup.ReaderSourceProvider()); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index cd9d8dfb5840..85bc4e2d00cf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues; import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -36,7 +37,7 @@ public class ScriptValuesTests extends ESTestCase { int index; FakeAggregationScript(Object[][] values) { - super(Collections.emptyMap(), new SearchLookup(null, null) { + super(Collections.emptyMap(), new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()) { @Override public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index 8249e9a281cd..26791682883f 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -175,7 +176,10 @@ public class FetchSourcePhaseTests extends ESTestCase { MemoryIndex index = new MemoryIndex(); LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0); HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1); - hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source)); + hitContext.sourceLookup() + .setSourceProvider( + source == null ? new SourceLookup.NullSourceProvider() : new SourceLookup.BytesSourceProvider(BytesReference.bytes(source)) + ); FetchSourcePhase phase = new FetchSourcePhase(); FetchSubPhaseProcessor processor = phase.getProcessor(fetchContext); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index d9a08a3426bd..bb51fd74c1c9 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -244,7 +244,7 @@ public class FieldFetcherTests extends MapperServiceTestCase { LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - SourceLookup sourceLookup = new SourceLookup(); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); sourceLookup.setSegmentAndDocument(readerContext, 0); Map fetchedFields = fieldFetcher.fetch(sourceLookup); @@ -284,8 +284,7 @@ public class FieldFetcherTests extends MapperServiceTestCase { public void testEmptyFetch() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject(); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.BytesSourceProvider(BytesReference.bytes(source))); { // make sure that an empty fetch don't deserialize the document FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), List.of()); @@ -1132,7 +1131,7 @@ public class FieldFetcherTests extends MapperServiceTestCase { IndexSearcher searcher = newSearcher(iw); LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - Map fields = fieldFetcher.fetch(new SourceLookup()); + Map fields = fieldFetcher.fetch(new SourceLookup(new SourceLookup.ReaderSourceProvider())); assertEquals(1, fields.size()); DocumentField field = fields.get("runtime_field"); assertEquals(1L, (long) field.getValue()); @@ -1164,7 +1163,7 @@ public class FieldFetcherTests extends MapperServiceTestCase { IndexSearcher searcher = newSearcher(iw); LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - SourceLookup sourceLookup = new SourceLookup(); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); sourceLookup.setSegmentAndDocument(readerContext, 0); Map fields = fieldFetcher.fetch(sourceLookup); assertEquals(1, fields.size()); @@ -1186,8 +1185,9 @@ public class FieldFetcherTests extends MapperServiceTestCase { throws IOException { SourceLookup sourceLookup = null; if (source != null) { - sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); + sourceLookup = new SourceLookup(new SourceLookup.BytesSourceProvider(BytesReference.bytes(source))); + } else { + sourceLookup = new SourceLookup(new SourceLookup.MapSourceProvider(Collections.emptyMap())); } FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); return fieldFetcher.fetch(sourceLookup); diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java index fec64a09c24b..e1f38f6aa69e 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java @@ -11,17 +11,17 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; @@ -34,20 +34,19 @@ public class SourceLookupTests extends ESTestCase { try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { Document doc = new Document(); doc.add(new StringField("field", "value", Field.Store.YES)); + doc.add(new StoredField("_source", new BytesRef("{\"field\": \"value\"}"))); iw.addDocument(doc); try (IndexReader reader = iw.getReader()) { LeafReaderContext readerContext = reader.leaves().get(0); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSegmentAndDocument(readerContext, 42); - sourceLookup.setSource( - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()) - ); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); + sourceLookup.setSegmentAndDocument(readerContext, 0); + sourceLookup.source(); assertNotNull(sourceLookup.internalSourceRef()); // Source should be preserved if we pass in the same reader and document - sourceLookup.setSegmentAndDocument(readerContext, 42); + sourceLookup.setSegmentAndDocument(readerContext, 0); assertNotNull(sourceLookup.internalSourceRef()); // Check that the stored fields reader is not loaded eagerly diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java index 5555b413dfec..b332c5f66def 100644 --- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; @@ -49,7 +50,7 @@ public class ScriptScoreQueryTests extends ESTestCase { private DirectoryReader reader; private IndexSearcher searcher; private LeafReaderContext leafReaderContext; - private final SearchLookup lookup = new SearchLookup(null, null); + private final SearchLookup lookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); @Before public void initSearcher() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index b5755dedfe8d..eb1a57c98677 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.List; @@ -82,7 +83,7 @@ public class GeoPointScriptFieldDistanceFeatureQueryTests extends AbstractScript iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), @@ -91,7 +92,7 @@ public class GeoPointScriptFieldDistanceFeatureQueryTests extends AbstractScript ) { @Override public void execute() { - GeoPoint point = GeoUtils.parseGeoPoint(searchLookup.source().get("location"), true); + GeoPoint point = GeoUtils.parseGeoPoint(searchLookup.source().source().get("location"), true); emit(point.lat(), point.lon()); } }; diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index 8127ff8d3a9a..fcdf236188ea 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.DateFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -68,7 +69,7 @@ public class LongScriptFieldDistanceFeatureQueryTests extends AbstractScriptFiel iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), @@ -78,7 +79,7 @@ public class LongScriptFieldDistanceFeatureQueryTests extends AbstractScriptFiel ) { @Override public void execute() { - for (Object timestamp : (List) searchLookup.source().get("timestamp")) { + for (Object timestamp : (List) searchLookup.source().source().get("timestamp")) { emit(((Number) timestamp).longValue()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index efd38b71e97e..e62f11b49521 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -55,8 +55,7 @@ public abstract class FieldTypeTestCase extends ESTestCase { when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, format); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap(field, sourceValue)); + SourceLookup lookup = new SourceLookup(new SourceLookup.MapSourceProvider(Collections.singletonMap(field, sourceValue))); return fetcher.fetchValues(lookup, new ArrayList<>()); } @@ -67,8 +66,7 @@ public abstract class FieldTypeTestCase extends ESTestCase { when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, null); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap(field, List.of(values))); + SourceLookup lookup = new SourceLookup(new SourceLookup.MapSourceProvider(Collections.singletonMap(field, List.of(values)))); return fetcher.fetchValues(lookup, new ArrayList<>()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index b790a7e62348..ebc126fb6b10 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -333,7 +333,8 @@ public abstract class MapperTestCase extends MapperServiceTestCase { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + new SourceLookup.ReaderSourceProvider() ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); IndexSearcher searcher = newSearcher(iw); @@ -605,7 +606,7 @@ public abstract class MapperTestCase extends MapperServiceTestCase { ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - SourceLookup sourceLookup = new SourceLookup(); + SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); nativeFetcher.setNextReader(ir.leaves().get(0)); @@ -714,7 +715,11 @@ public abstract class MapperTestCase extends MapperServiceTestCase { SourceToParse source = source(this::writeField); ParsedDocument doc = mapperService.documentMapper().parse(source); - SearchLookup lookup = new SearchLookup(f -> fieldType, (f, s, t) -> { throw new UnsupportedOperationException(); }); + SearchLookup lookup = new SearchLookup( + f -> fieldType, + (f, s, t) -> { throw new UnsupportedOperationException(); }, + new SourceLookup.ReaderSourceProvider() + ); withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index 9d712d238138..6006920eb87d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -88,7 +88,7 @@ public class DataTierFieldTypeTests extends MapperServiceTestCase { public void testFetchValue() throws IOException { MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE; - SourceLookup lookup = new SourceLookup(); + SourceLookup lookup = new SourceLookup(new SourceLookup.NullSourceProvider()); List ignoredValues = new ArrayList<>(); ValueFetcher valueFetcher = ft.valueFetcher(createContext(), null); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 09b2e04b9201..95e7545f2552 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.script.DocReader; import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; @@ -124,7 +125,8 @@ public class AggregateDoubleMetricFieldTypeTests extends FieldTypeTestCase { searchExecutionContext::getFieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo) - ).build(null, null) + ).build(null, null), + new SourceLookup.ReaderSourceProvider() ); when(searchExecutionContext.lookup()).thenReturn(lookup); IndexSearcher searcher = newSearcher(reader); diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index c376b9c10ea1..f26553ef2328 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -111,9 +111,8 @@ public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { MappedFieldType fieldType = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); ValueFetcher fetcher = fieldType.valueFetcher(null, null); - SourceLookup missingValueLookup = new SourceLookup(); - SourceLookup nullValueLookup = new SourceLookup(); - nullValueLookup.setSource(Collections.singletonMap("field", null)); + SourceLookup missingValueLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider()); + SourceLookup nullValueLookup = new SourceLookup(new SourceLookup.MapSourceProvider(Collections.singletonMap("field", null))); List ignoredValues = new ArrayList<>(); assertTrue(fetcher.fetchValues(missingValueLookup, ignoredValues).isEmpty());