Expose the bitset filter cache in the MappingParserContext (#109298)

Add the bitset filter cache in the MappingParserContext

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.benchmark.index.mapper;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -21,10 +22,12 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.Script;
@@ -52,6 +55,13 @@ public class MapperServiceFactory {
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         MapperService mapperService = new MapperService(
             () -> TransportVersion.current(),
             indexSettings,
@@ -73,6 +83,7 @@ public class MapperServiceFactory {
                     throw new UnsupportedOperationException();
                 }
             },
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
 

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -189,6 +189,7 @@ public class QueryParserHelperBenchmark {
                     throw new UnsupportedOperationException();
                 }
             },
+            query -> { throw new UnsupportedOperationException(); },
             MapperMetrics.NOOP
         );
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -187,6 +187,9 @@ public class IndexMetadataVerifier {
                     () -> null,
                     indexSettings.getMode().idFieldMapperWithoutFieldData(),
                     scriptService,
+                    query -> {
+                        throw new UnsupportedOperationException("IndexMetadataVerifier");
+                    },
                     mapperMetrics
                 )
             ) {

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -652,6 +652,9 @@ public final class IndexModule {
             },
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             scriptService,
+            query -> {
+                throw new UnsupportedOperationException("no index query shard context available");
+            },
             mapperMetrics
         );
     }

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -212,6 +212,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         this.indexAnalyzers = indexAnalyzers;
         if (needsMapperService(indexSettings, indexCreationContext)) {
             assert indexAnalyzers != null;
+            this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.mapperService = new MapperService(
                 clusterService,
                 indexSettings,
@@ -223,6 +224,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()),
                 idFieldMapper,
                 scriptService,
+                bitsetFilterCache::getBitSetProducer,
                 mapperMetrics
             );
             this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService);
@@ -238,7 +240,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                 this.indexSortSupplier = () -> null;
             }
             indexFieldData.setListener(new FieldDataCacheListener(this));
-            this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
             this.indexCache = new IndexCache(queryCache, bitsetFilterCache);
         } else {

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
@@ -167,6 +169,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
         ScriptCompiler scriptCompiler,
+        Function<Query, BitSetProducer> bitSetProducer,
         MapperMetrics mapperMetrics
     ) {
         this(
@@ -179,6 +182,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             searchExecutionContextSupplier,
             idFieldMapper,
             scriptCompiler,
+            bitSetProducer,
             mapperMetrics
         );
     }
@@ -194,6 +198,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
         ScriptCompiler scriptCompiler,
+        Function<Query, BitSetProducer> bitSetProducer,
         MapperMetrics mapperMetrics
     ) {
         super(indexSettings);
@@ -210,7 +215,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             scriptCompiler,
             indexAnalyzers,
             indexSettings,
-            idFieldMapper
+            idFieldMapper,
+            bitSetProducer
         );
         this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get());
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(

server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -37,6 +39,7 @@ public class MappingParserContext {
     private final IndexAnalyzers indexAnalyzers;
     private final IndexSettings indexSettings;
     private final IdFieldMapper idFieldMapper;
+    private final Function<Query, BitSetProducer> bitSetProducer;
     private final long mappingObjectDepthLimit;
     private long mappingObjectDepth = 0;
 
@@ -50,7 +53,8 @@ public class MappingParserContext {
         ScriptCompiler scriptCompiler,
         IndexAnalyzers indexAnalyzers,
         IndexSettings indexSettings,
-        IdFieldMapper idFieldMapper
+        IdFieldMapper idFieldMapper,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
         this.similarityLookupService = similarityLookupService;
         this.typeParsers = typeParsers;
@@ -63,6 +67,7 @@ public class MappingParserContext {
         this.indexSettings = indexSettings;
         this.idFieldMapper = idFieldMapper;
         this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit();
+        this.bitSetProducer = bitSetProducer;
     }
 
     public IndexAnalyzers getIndexAnalyzers() {
@@ -132,6 +137,10 @@ public class MappingParserContext {
         return scriptCompiler;
     }
 
+    public BitSetProducer bitSetProducer(Query query) {
+        return bitSetProducer.apply(query);
+    }
+
     void incrementMappingObjectDepth() throws MapperParsingException {
         mappingObjectDepth++;
         if (mappingObjectDepth > mappingObjectDepthLimit) {
@@ -159,7 +168,8 @@ public class MappingParserContext {
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
-                in.idFieldMapper
+                in.idFieldMapper,
+                in.bitSetProducer
             );
         }
 
@@ -188,7 +198,8 @@ public class MappingParserContext {
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
-                in.idFieldMapper
+                in.idFieldMapper,
+                in.bitSetProducer
             );
             this.dateFormatter = dateFormatter;
         }

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -19,15 +19,18 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.ScriptCompiler;
@@ -107,6 +110,13 @@ public class CodecTests extends ESTestCase {
             Collections.emptyMap(),
             MapperPlugin.NOOP_FIELD_FILTER
         );
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         MapperService service = new MapperService(
             () -> TransportVersion.current(),
             settings,
@@ -117,6 +127,7 @@ public class CodecTests extends ESTestCase {
             () -> null,
             settings.getMode().idFieldMapperWithoutFieldData(),
             ScriptCompiler.NONE,
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
         return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE);

server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -17,6 +18,8 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.ScriptService;
@@ -43,6 +46,13 @@ public class MappingParserTests extends MapperServiceTestCase {
         IndexAnalyzers indexAnalyzers = createIndexAnalyzers();
         SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         Supplier<MappingParserContext> mappingParserContextSupplier = () -> new MappingParserContext(
             similarityService::getSimilarity,
             type -> mapperRegistry.getMapperParser(type, indexSettings.getIndexVersionCreated()),
@@ -55,7 +65,8 @@ public class MappingParserTests extends MapperServiceTestCase {
             scriptService,
             indexAnalyzers,
             indexSettings,
-            indexSettings.getMode().idFieldMapperWithoutFieldData()
+            indexSettings.getMode().idFieldMapperWithoutFieldData(),
+            bitsetFilterCache::getBitSetProducer
         );
 
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(

server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java

@@ -277,7 +277,10 @@ public class ParametrizedMapperTests extends MapperServiceTestCase {
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData()
+            mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(),
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
         if (fromDynamicTemplate) {
             pc = pc.createDynamicTemplateContext(null);

server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java

@@ -97,7 +97,10 @@ public class TypeParsersTests extends ESTestCase {
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            ProvidedIdFieldMapper.NO_FIELD_DATA
+            ProvidedIdFieldMapper.NO_FIELD_DATA,
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
 
         TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext);
@@ -128,7 +131,10 @@ public class TypeParsersTests extends ESTestCase {
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            ProvidedIdFieldMapper.NO_FIELD_DATA
+            ProvidedIdFieldMapper.NO_FIELD_DATA,
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {

server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java

@@ -548,7 +548,10 @@ public class SearchExecutionContextTests extends ESTestCase {
                 ScriptCompiler.NONE,
                 indexAnalyzers,
                 indexSettings,
-                indexSettings.getMode().buildIdFieldMapper(() -> true)
+                indexSettings.getMode().buildIdFieldMapper(() -> true),
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             )
         );
         when(mapperService.isMultiField(anyString())).then(

test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java

@@ -8,15 +8,18 @@
 
 package org.elasticsearch.index;
 
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.ScriptCompiler;
@@ -58,6 +61,13 @@ public class MapperTestUtils {
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
         IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         return new MapperService(
             () -> TransportVersion.current(),
             indexSettings,
@@ -68,6 +78,7 @@ public class MapperTestUtils {
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             ScriptCompiler.NONE,
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
     }

test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java

@@ -55,6 +55,9 @@ public class TranslogHandler implements Engine.TranslogRecoveryRunner {
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             null,
+            query -> {
+                throw new UnsupportedOperationException("The bitset filter cache is not available in translog operations");
+            },
             MapperMetrics.NOOP
         );
     }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -254,6 +254,14 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
                 getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
             ).getMapperRegistry();
 
+            BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+                @Override
+                public void onCache(ShardId shardId, Accountable accountable) {}
+
+                @Override
+                public void onRemoval(ShardId shardId, Accountable accountable) {}
+            });
+
             return new MapperService(
                 () -> TransportVersion.current(),
                 indexSettings,
@@ -266,6 +274,7 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
                 },
                 indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
                 scriptCompiler,
+                bitsetFilterCache::getBitSetProducer,
                 mapperMetrics
             );
         }

test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java

@@ -63,7 +63,10 @@ public class TestDocumentParserContext extends DocumentParserContext {
                 null,
                 (type, name) -> Lucene.STANDARD_ANALYZER,
                 MapperTestCase.createIndexSettings(IndexVersion.current(), settings),
-                null
+                null,
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             ),
             source,
             mappingLookup.getMapping().getRoot(),

test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java

@@ -1284,7 +1284,10 @@ public abstract class AggregatorTestCase extends ESTestCase {
                 ScriptCompiler.NONE,
                 null,
                 indexSettings,
-                null
+                null,
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             );
         }
 

test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java

@@ -467,6 +467,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
             IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings);
             scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts);
             similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap());
+            this.bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
+                @Override
+                public void onCache(ShardId shardId, Accountable accountable) {}
+
+                @Override
+                public void onRemoval(ShardId shardId, Accountable accountable) {}
+            });
             MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
             mapperService = new MapperService(
                 clusterService,
@@ -478,23 +485,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
                 () -> createShardContext(null),
                 idxSettings.getMode().idFieldMapperWithoutFieldData(),
                 ScriptCompiler.NONE,
+                bitsetFilterCache::getBitSetProducer,
                 MapperMetrics.NOOP
             );
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
             });
             indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, new NoneCircuitBreakerService());
-            bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
-                @Override
-                public void onCache(ShardId shardId, Accountable accountable) {
-
-                }
-
-                @Override
-                public void onRemoval(ShardId shardId, Accountable accountable) {
-
-                }
-            });
-
             if (registerType) {
                 mapperService.merge(
                     "_doc",