Include ignored source as part of loading field values in ValueSourceReaderOperator via BlockSourceReader. (#114903)

Currently, in compute engine when loading source if source mode is synthetic, the synthetic source loader is already used. But the ignored_source field isn't always marked as a required source field, causing the source to potentially miss a lot of fields.

This change includes _ignored_source field as a required stored field and allowing keyword fields without doc values or stored fields to be used in case of synthetic source.

Relying on synthetic source to get the values (because a field doesn't have stored fields / doc values) is slow. In case of synthetic source we already keep ignored field/values in a special place, named ignored source. Long term in case of synthetic source we should only load ignored source in case a field has no doc values or stored field. Like is being explored in #114886 Thereby avoiding synthesizing the complete _source in order to get only one field.

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -364,7 +364,8 @@ public class MatchOnlyTextFieldMapper extends FieldMapper {
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
             BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name());
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, sourceMode);
         }
 
         @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -319,7 +319,8 @@ public class ScaledFloatFieldMapper extends FieldMapper {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, sourceMode);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -189,7 +189,8 @@ public abstract class AbstractGeometryFieldMapper<T> extends FieldMapper {
         protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) {
             ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB);
             // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
-            return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), sourceMode);
         }
 
         protected abstract Object nullValueAsSource(T nullValue);

server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java

@@ -22,6 +22,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Loads values from {@code _source}. This whole process is very slow and cast-tastic,
@@ -29,6 +30,14 @@ import java.util.List;
  * slow.
  */
 public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
+
+    // _ignored_source is needed when source mode is synthetic.
+    static final StoredFieldsSpec NEEDS_SOURCE_AND_IGNORED_SOURCE = new StoredFieldsSpec(
+        true,
+        false,
+        Set.of(IgnoredSourceFieldMapper.NAME)
+    );
+
     private final ValueFetcher fetcher;
     private final List<Object> ignoredValues = new ArrayList<>();
     private final DocIdSetIterator iter;
@@ -91,10 +100,12 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
     private abstract static class SourceBlockLoader implements BlockLoader {
         protected final ValueFetcher fetcher;
         private final LeafIteratorLookup lookup;
+        private final SourceFieldMapper.Mode sourceMode;
 
-        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
+        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
             this.fetcher = fetcher;
             this.lookup = lookup;
+            this.sourceMode = sourceMode;
         }
 
         @Override
@@ -104,7 +115,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
 
         @Override
         public final StoredFieldsSpec rowStrideStoredFieldSpec() {
-            return StoredFieldsSpec.NEEDS_SOURCE;
+            return sourceMode == SourceFieldMapper.Mode.SYNTHETIC ? NEEDS_SOURCE_AND_IGNORED_SOURCE : StoredFieldsSpec.NEEDS_SOURCE;
         }
 
         @Override
@@ -140,8 +151,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
      * Load {@code boolean}s from {@code _source}.
      */
     public static class BooleansBlockLoader extends SourceBlockLoader {
-        public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -180,8 +191,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
      * Load {@link BytesRef}s from {@code _source}.
      */
     public static class BytesRefsBlockLoader extends SourceBlockLoader {
-        public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -191,7 +202,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
 
         @Override
         protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) throws IOException {
-            return new BytesRefs(fetcher, iter);
+            return new BytesRefs(fetcher, iter, null);
         }
 
         @Override
@@ -201,8 +212,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
     }
 
     public static class GeometriesBlockLoader extends SourceBlockLoader {
-        public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -212,7 +223,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
 
         @Override
         protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
-            return new Geometries(fetcher, iter);
+            return new Geometries(fetcher, iter, null);
         }
 
         @Override
@@ -224,7 +235,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
     private static class BytesRefs extends BlockSourceReader {
         private final BytesRef scratch = new BytesRef();
 
-        BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter) {
+        BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
             super(fetcher, iter);
         }
 
@@ -241,7 +252,7 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
 
     private static class Geometries extends BlockSourceReader {
 
-        Geometries(ValueFetcher fetcher, DocIdSetIterator iter) {
+        Geometries(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
             super(fetcher, iter);
         }
 
@@ -264,8 +275,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
      * Load {@code double}s from {@code _source}.
      */
     public static class DoublesBlockLoader extends SourceBlockLoader {
-        public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -304,8 +315,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
      * Load {@code int}s from {@code _source}.
      */
     public static class IntsBlockLoader extends SourceBlockLoader {
-        public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -344,8 +355,8 @@ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
      * Load {@code long}s from {@code _source}.
      */
     public static class LongsBlockLoader extends SourceBlockLoader {
-        public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -314,7 +314,7 @@ public class BooleanFieldMapper extends FieldMapper {
             BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup);
+            return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.indexSettings().getIndexMappingSourceMode());
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -792,7 +792,8 @@ public final class DateFieldMapper extends FieldMapper {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -632,18 +632,12 @@ public final class KeywordFieldMapper extends FieldMapper {
             if (hasDocValues()) {
                 return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
             }
-            if (isSyntheticSource) {
-                if (false == isStored()) {
-                    throw new IllegalStateException(
-                        "keyword field ["
-                            + name()
-                            + "] is only supported in synthetic _source index if it creates doc values or stored fields"
-                    );
-                }
+            if (isStored()) {
                 return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
             }
             SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), sourceMode);
         }
 
         private BlockSourceReader.LeafIteratorLookup sourceBlockLoaderLookup(BlockLoaderContext blContext) {

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -461,8 +461,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         FLOAT("float", NumericType.FLOAT) {
@@ -645,8 +649,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         DOUBLE("double", NumericType.DOUBLE) {
@@ -795,8 +803,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         BYTE("byte", NumericType.BYTE) {
@@ -908,8 +920,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1021,8 +1037,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1208,8 +1228,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         LONG("long", NumericType.LONG) {
@@ -1355,8 +1379,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1634,7 +1662,11 @@ public class NumberFieldMapper extends FieldMapper {
 
         abstract BlockLoader blockLoaderFromDocValues(String fieldName);
 
-        abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
+        abstract BlockLoader blockLoaderFromSource(
+            SourceValueFetcher sourceValueFetcher,
+            BlockSourceReader.LeafIteratorLookup lookup,
+            SourceFieldMapper.Mode sourceMode
+        );
     }
 
     public static class NumberFieldType extends SimpleMappedFieldType {
@@ -1773,7 +1805,8 @@ public class NumberFieldMapper extends FieldMapper {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1012,17 +1012,20 @@ public final class TextFieldMapper extends FieldMapper {
             if (isStored()) {
                 return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name());
             }
-            if (isSyntheticSource) {
+            if (isSyntheticSource && syntheticSourceDelegate == null) {
                 /*
                  * When we're in synthetic source mode we don't currently
                  * support text fields that are not stored and are not children
                  * of perfect keyword fields. We'd have to load from the parent
-                 * field and then convert the result to a string.
+                 * field and then convert the result to a string. In this case,
+                 * even if we would synthesize the source, the current field
+                 * would be missing.
                  */
                 return null;
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext));
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), sourceMode);
         }
 
         /**

server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java

@@ -51,7 +51,7 @@ public class BlockSourceReaderTests extends MapperServiceTestCase {
     private void loadBlock(LeafReaderContext ctx, Consumer<TestBlock> test) throws IOException {
         ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"));
         BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field");
-        BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup);
+        BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, null);
         assertThat(loader.columnAtATimeReader(ctx), nullValue());
         BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx);
         assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE));

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -1321,12 +1321,15 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             return mapper.fieldType(loaderFieldName).blockLoader(new MappedFieldType.BlockLoaderContext() {
                 @Override
                 public String indexName() {
-                    throw new UnsupportedOperationException();
+                    return "test_index";
                 }
 
                 @Override
                 public IndexSettings indexSettings() {
-                    throw new UnsupportedOperationException();
+                    var imd = IndexMetadata.builder(indexName())
+                        .settings(MapperTestCase.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                        .build();
+                    return new IndexSettings(imd, Settings.EMPTY);
                 }
 
                 @Override

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java

@@ -26,6 +26,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
@@ -546,7 +547,10 @@ public class ValueSourceReaderTypeConversionTests extends AnyOperatorTestCase {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java

@@ -24,9 +24,11 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.tests.mockfile.HandleLimitFS;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -500,7 +502,10 @@ public class ValuesSourceReaderOperatorTests extends OperatorTestCase {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override

x-pack/plugin/logsdb/build.gradle

@@ -25,7 +25,7 @@ base {
 
 restResources {
   restApi {
-    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities'
+    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities', 'esql.query'
   }
 }
 

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java

@@ -301,6 +301,21 @@ public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChalleng
         assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
+    public void testEsqlTermsAggregationByMethod() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
+
+        indexDocuments(documents);
+
+        final String query = "FROM $index | STATS count(*) BY method | SORT method | LIMIT " + numberOfDocuments;
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
+            .settings(getContenderSettings(), getBaselineSettings())
+            .expected(getEsqlStatsResults(esqlBaseline(query)))
+            .ignoringSort(true)
+            .isEqualTo(getEsqlStatsResults(esqlContender(query)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
+    }
+
     public void testFieldCaps() throws IOException {
         int numberOfDocuments = ESTestCase.randomIntBetween(20, 50);
         final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml

@@ -0,0 +1,305 @@
+---
+setup:
+  - requires:
+      test_runner_features: allowed_warnings_regex
+
+  - do:
+      indices.create:
+        index: my-index
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+                doc_values: false
+                store: false
+              process_id:
+                type: integer
+                doc_values: false
+                store: false
+              http_method:
+                type: keyword
+                doc_values: false
+                store: false
+              is_https:
+                type: boolean
+                doc_values: false
+                store: false
+              location:
+                type: geo_point
+                doc_values: false
+                store: false
+              message:
+                type: text
+                store: false
+                fields:
+                  raw:
+                    type: keyword
+
+  - do:
+      bulk:
+        index: my-index
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." }
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: my-index
+
+---
+"Simple from":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "message.raw"}
+  - match: {columns.7.type: "keyword"}
+  - match: {columns.8.name: "process_id"}
+  - match: {columns.8.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.006 40.7128)"}
+  - match: {values.0.6: "Do. Or do not. There is no try."}
+  - match: {values.0.7: "Do. Or do not. There is no try."}
+  - match: {values.0.8: 102}
+
+---
+"Simple from geo point":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10'
+
+  - match: {columns.0.name: "location"}
+  - match: {columns.0.type: "geo_point"}
+
+  - match: {values.0.0: "POINT (-74.006 40.7128)"}
+  - match: {values.1.0: "POINT (-74.006 40.7128)"}
+  - match: {values.2.0: "POINT (-74.006 40.7128)"}
+  - match: {values.3.0: "POINT (-74.006 40.7128)"}
+  - match: {values.4.0: "POINT (-74.006 40.7128)"}
+  - match: {values.5.0: "POINT (-74.006 40.7128)"}
+
+---
+"Simple from number fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10'
+
+  - match: {columns.0.name: "process_id"}
+  - match: {columns.0.type: "integer"}
+
+  - match: {values.0.0: 102}
+  - match: {values.1.0: 102}
+  - match: {values.2.0: 104}
+  - match: {values.3.0: 101}
+  - match: {values.4.0: 103}
+  - match: {values.5.0: 105}
+
+---
+"Simple from keyword fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10'
+
+  - match: {columns.0.name: "agent_id"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "http_method"}
+  - match: {columns.1.type: "keyword"}
+
+  - match: {values.0.0: "yoda"}
+  - match: {values.0.1: "PUT"}
+  - match: {values.1.0: "darth-vader"}
+  - match: {values.1.1: "POST"}
+  - match: {values.2.0: "yoda"}
+  - match: {values.2.1: "POST"}
+  - match: {values.3.0: "darth-vader"}
+  - match: {values.3.1: "GET"}
+  - match: {values.4.0: "obi-wan"}
+  - match: {values.4.1: "GET"}
+  - match: {values.5.0: "obi-wan"}
+  - match: {values.5.1: "GET"}
+
+---
+"Simple from boolean fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10'
+
+  - match: {columns.0.name: "is_https"}
+  - match: {columns.0.type: "boolean"}
+
+  - match: {values.0.0: false}
+  - match: {values.1.0: true}
+  - match: {values.2.0: false}
+  - match: {values.3.0: false}
+  - match: {values.4.0: false}
+  - match: {values.5.0: false}
+
+---
+"Simple from text fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+
+---
+"message field without keyword multi-field":
+  - do:
+      indices.create:
+        index: my-index2
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+                doc_values: false
+                store: false
+              process_id:
+                type: integer
+                doc_values: false
+                store: false
+              http_method:
+                type: keyword
+                doc_values: false
+                store: false
+              is_https:
+                type: boolean
+                doc_values: false
+                store: false
+              location:
+                type: geo_point
+                doc_values: false
+                store: false
+              message:
+                type: text
+                store: false
+
+  - do:
+      bulk:
+        index: my-index2
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "That's no moon. It's a space station." }
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "process_id"}
+  - match: {columns.7.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.006 40.7128)"}
+  - match: {values.0.6: null}          # null is expected, because text fields aren't stored in ignored source
+  - match: {values.0.7: 102}
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  # null is expected, because text fields aren't stored in ignored source
+  - match: {values.0.0: null}
+  - match: {values.1.0: null}
+  - match: {values.2.0: null}
+  - match: {values.3.0: null}
+  - match: {values.4.0: null}
+  - match: {values.5.0: null}

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml

@@ -0,0 +1,177 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: my-index
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              is_https:
+                type: boolean
+              location:
+                type: geo_point
+              message:
+                type: text
+
+  - do:
+      bulk:
+        index: my-index
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." }
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: my-index
+
+---
+"Simple from":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "host.name"}
+  - match: {columns.2.type: "keyword"}
+  - match: {columns.3.name: "http_method" }
+  - match: {columns.3.type: "keyword" }
+  - match: {columns.4.name: "is_https"}
+  - match: {columns.4.type: "boolean"}
+  - match: {columns.5.name: "location"}
+  - match: {columns.5.type: "geo_point"}
+  - match: {columns.6.name: "message"}
+  - match: {columns.6.type: "text"}
+  - match: {columns.7.name: "process_id"}
+  - match: {columns.7.type: "integer"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "yoda"}
+  - match: {values.0.2: "bar"}
+  - match: {values.0.3: "PUT"}
+  - match: {values.0.4: false}
+  - match: {values.0.5: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.0.6: "Do. Or do not. There is no try."}
+  - match: {values.0.7: 102}
+
+---
+"Simple from geo point":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10'
+
+  - match: {columns.0.name: "location"}
+  - match: {columns.0.type: "geo_point"}
+
+  - match: {values.0.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.1.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.2.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.3.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.4.0: "POINT (-74.00600004941225 40.712799984030426)"}
+  - match: {values.5.0: "POINT (-74.00600004941225 40.712799984030426)"}
+
+---
+"Simple from number fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10'
+
+  - match: {columns.0.name: "process_id"}
+  - match: {columns.0.type: "integer"}
+
+  - match: {values.0.0: 102}
+  - match: {values.1.0: 102}
+  - match: {values.2.0: 104}
+  - match: {values.3.0: 101}
+  - match: {values.4.0: 103}
+  - match: {values.5.0: 105}
+
+---
+"Simple from keyword fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10'
+
+  - match: {columns.0.name: "agent_id"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "http_method"}
+  - match: {columns.1.type: "keyword"}
+
+  - match: {values.0.0: "yoda"}
+  - match: {values.0.1: "PUT"}
+  - match: {values.1.0: "darth-vader"}
+  - match: {values.1.1: "POST"}
+  - match: {values.2.0: "yoda"}
+  - match: {values.2.1: "POST"}
+  - match: {values.3.0: "darth-vader"}
+  - match: {values.3.1: "GET"}
+  - match: {values.4.0: "obi-wan"}
+  - match: {values.4.1: "GET"}
+  - match: {values.5.0: "obi-wan"}
+  - match: {values.5.1: "GET"}
+
+---
+"Simple from boolean fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10'
+
+  - match: {columns.0.name: "is_https"}
+  - match: {columns.0.type: "boolean"}
+
+  - match: {values.0.0: false}
+  - match: {values.1.0: true}
+  - match: {values.2.0: false}
+  - match: {values.3.0: false}
+  - match: {values.4.0: false}
+  - match: {values.5.0: false}
+
+---
+"Simple from text fields":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -339,7 +339,8 @@ public class UnsignedLongFieldMapper extends FieldMapper {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, sourceMode);
         }
 
         @Override