github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Expose the bitset filter cache in the MappingParserContext (#109298)

Browse source 
Add the bitset filter cache in the MappingParserContext
This commit is contained in:
Jim Ferenczi 2024-06-05 11:43:07 +01:00 • committed by GitHub
parent 32dfd1866e
commit 564549af8f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 119 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

11
benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.benchmark.index.mapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.IndexMetadata;
@ -21,10 +22,12 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.LowercaseNormalizer;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.mapper.MapperMetrics;
import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.Script;
@ -52,6 +55,13 @@ public class MapperServiceFactory {
MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
MapperService mapperService = new MapperService(
() -> TransportVersion.current(),
indexSettings,
@ -73,6 +83,7 @@ public class MapperServiceFactory {
throw new UnsupportedOperationException();
}
},
bitsetFilterCache::getBitSetProducer,
MapperMetrics.NOOP
);

1
benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
View file

@ -189,6 +189,7 @@ public class QueryParserHelperBenchmark {
throw new UnsupportedOperationException();
}
},
query -> { throw new UnsupportedOperationException(); },
MapperMetrics.NOOP
);

3
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
View file

@ -187,6 +187,9 @@ public class IndexMetadataVerifier {
() -> null,
indexSettings.getMode().idFieldMapperWithoutFieldData(),
scriptService,
query -> {
throw new UnsupportedOperationException("IndexMetadataVerifier");
},
mapperMetrics
)
) {

3
server/src/main/java/org/elasticsearch/index/IndexModule.java
View file

@ -652,6 +652,9 @@ public final class IndexModule {
},
indexSettings.getMode().idFieldMapperWithoutFieldData(),
scriptService,
query -> {
throw new UnsupportedOperationException("no index query shard context available");
},
mapperMetrics
);
}

3
server/src/main/java/org/elasticsearch/index/IndexService.java
View file

@ -212,6 +212,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
this.indexAnalyzers = indexAnalyzers;
if (needsMapperService(indexSettings, indexCreationContext)) {
assert indexAnalyzers != null;
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
this.mapperService = new MapperService(
clusterService,
indexSettings,
@ -223,6 +224,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
() -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()),
idFieldMapper,
scriptService,
bitsetFilterCache::getBitSetProducer,
mapperMetrics
);
this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService);
@ -238,7 +240,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
this.indexSortSupplier = () -> null;
}
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
this.indexCache = new IndexCache(queryCache, bitsetFilterCache);
} else {

8
server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View file

@ -8,6 +8,8 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.BitSetProducer;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
@ -167,6 +169,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
Supplier<SearchExecutionContext> searchExecutionContextSupplier,
IdFieldMapper idFieldMapper,
ScriptCompiler scriptCompiler,
Function<Query, BitSetProducer> bitSetProducer,
MapperMetrics mapperMetrics
) {
this(
@ -179,6 +182,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
searchExecutionContextSupplier,
idFieldMapper,
scriptCompiler,
bitSetProducer,
mapperMetrics
);
}
@ -194,6 +198,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
Supplier<SearchExecutionContext> searchExecutionContextSupplier,
IdFieldMapper idFieldMapper,
ScriptCompiler scriptCompiler,
Function<Query, BitSetProducer> bitSetProducer,
MapperMetrics mapperMetrics
) {
super(indexSettings);
@ -210,7 +215,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
scriptCompiler,
indexAnalyzers,
indexSettings,
idFieldMapper
idFieldMapper,
bitSetProducer
);
this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get());
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(

17
server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
View file

@ -8,6 +8,8 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.BitSetProducer;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
@ -37,6 +39,7 @@ public class MappingParserContext {
private final IndexAnalyzers indexAnalyzers;
private final IndexSettings indexSettings;
private final IdFieldMapper idFieldMapper;
private final Function<Query, BitSetProducer> bitSetProducer;
private final long mappingObjectDepthLimit;
private long mappingObjectDepth = 0;
@ -50,7 +53,8 @@ public class MappingParserContext {
ScriptCompiler scriptCompiler,
IndexAnalyzers indexAnalyzers,
IndexSettings indexSettings,
IdFieldMapper idFieldMapper
IdFieldMapper idFieldMapper,
Function<Query, BitSetProducer> bitSetProducer
) {
this.similarityLookupService = similarityLookupService;
this.typeParsers = typeParsers;
@ -63,6 +67,7 @@ public class MappingParserContext {
this.indexSettings = indexSettings;
this.idFieldMapper = idFieldMapper;
this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit();
this.bitSetProducer = bitSetProducer;
}
public IndexAnalyzers getIndexAnalyzers() {
@ -132,6 +137,10 @@ public class MappingParserContext {
return scriptCompiler;
}
public BitSetProducer bitSetProducer(Query query) {
return bitSetProducer.apply(query);
}
void incrementMappingObjectDepth() throws MapperParsingException {
mappingObjectDepth++;
if (mappingObjectDepth > mappingObjectDepthLimit) {
@ -159,7 +168,8 @@ public class MappingParserContext {
in.scriptCompiler,
in.indexAnalyzers,
in.indexSettings,
in.idFieldMapper
in.idFieldMapper,
in.bitSetProducer
);
}
@ -188,7 +198,8 @@ public class MappingParserContext {
in.scriptCompiler,
in.indexAnalyzers,
in.indexSettings,
in.idFieldMapper
in.idFieldMapper,
in.bitSetProducer
);
this.dateFormatter = dateFormatter;
}

11
server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
View file

@ -19,15 +19,18 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.mapper.MapperMetrics;
import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.script.ScriptCompiler;
@ -107,6 +110,13 @@ public class CodecTests extends ESTestCase {
Collections.emptyMap(),
MapperPlugin.NOOP_FIELD_FILTER
);
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
MapperService service = new MapperService(
() -> TransportVersion.current(),
settings,
@ -117,6 +127,7 @@ public class CodecTests extends ESTestCase {
() -> null,
settings.getMode().idFieldMapperWithoutFieldData(),
ScriptCompiler.NONE,
bitsetFilterCache::getBitSetProducer,
MapperMetrics.NOOP
);
return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE);

13
server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
View file

@ -8,6 +8,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.bytes.BytesReference;
@ -17,6 +18,8 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.ScriptService;
@ -43,6 +46,13 @@ public class MappingParserTests extends MapperServiceTestCase {
IndexAnalyzers indexAnalyzers = createIndexAnalyzers();
SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap());
MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
Supplier<MappingParserContext> mappingParserContextSupplier = () -> new MappingParserContext(
similarityService::getSimilarity,
type -> mapperRegistry.getMapperParser(type, indexSettings.getIndexVersionCreated()),
@ -55,7 +65,8 @@ public class MappingParserTests extends MapperServiceTestCase {
scriptService,
indexAnalyzers,
indexSettings,
indexSettings.getMode().idFieldMapperWithoutFieldData()
indexSettings.getMode().idFieldMapperWithoutFieldData(),
bitsetFilterCache::getBitSetProducer
);
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(

5
server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
View file

@ -277,7 +277,10 @@ public class ParametrizedMapperTests extends MapperServiceTestCase {
ScriptCompiler.NONE,
mapperService.getIndexAnalyzers(),
mapperService.getIndexSettings(),
mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData()
mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(),
query -> {
throw new UnsupportedOperationException();
}
);
if (fromDynamicTemplate) {
pc = pc.createDynamicTemplateContext(null);

10
server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
View file

@ -97,7 +97,10 @@ public class TypeParsersTests extends ESTestCase {
ScriptCompiler.NONE,
mapperService.getIndexAnalyzers(),
mapperService.getIndexSettings(),
ProvidedIdFieldMapper.NO_FIELD_DATA
ProvidedIdFieldMapper.NO_FIELD_DATA,
query -> {
throw new UnsupportedOperationException();
}
);
TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext);
@ -128,7 +131,10 @@ public class TypeParsersTests extends ESTestCase {
ScriptCompiler.NONE,
mapperService.getIndexAnalyzers(),
mapperService.getIndexSettings(),
ProvidedIdFieldMapper.NO_FIELD_DATA
ProvidedIdFieldMapper.NO_FIELD_DATA,
query -> {
throw new UnsupportedOperationException();
}
);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {

5
server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
View file

@ -548,7 +548,10 @@ public class SearchExecutionContextTests extends ESTestCase {
ScriptCompiler.NONE,
indexAnalyzers,
indexSettings,
indexSettings.getMode().buildIdFieldMapper(() -> true)
indexSettings.getMode().buildIdFieldMapper(() -> true),
query -> {
throw new UnsupportedOperationException();
}
)
);
when(mapperService.isMultiField(anyString())).then(

11
test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View file

@ -8,15 +8,18 @@
package org.elasticsearch.index;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.mapper.MapperMetrics;
import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.ScriptCompiler;
@ -58,6 +61,13 @@ public class MapperTestUtils {
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
return new MapperService(
() -> TransportVersion.current(),
indexSettings,
@ -68,6 +78,7 @@ public class MapperTestUtils {
() -> null,
indexSettings.getMode().idFieldMapperWithoutFieldData(),
ScriptCompiler.NONE,
bitsetFilterCache::getBitSetProducer,
MapperMetrics.NOOP
);
}

3
test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
View file

@ -55,6 +55,9 @@ public class TranslogHandler implements Engine.TranslogRecoveryRunner {
() -> null,
indexSettings.getMode().idFieldMapperWithoutFieldData(),
null,
query -> {
throw new UnsupportedOperationException("The bitset filter cache is not available in translog operations");
},
MapperMetrics.NOOP
);
}

9
test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
View file

@ -254,6 +254,14 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
).getMapperRegistry();
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
return new MapperService(
() -> TransportVersion.current(),
indexSettings,
@ -266,6 +274,7 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
},
indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
scriptCompiler,
bitsetFilterCache::getBitSetProducer,
mapperMetrics
);
}

5
test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
View file

@ -63,7 +63,10 @@ public class TestDocumentParserContext extends DocumentParserContext {
null,
(type, name) -> Lucene.STANDARD_ANALYZER,
MapperTestCase.createIndexSettings(IndexVersion.current(), settings),
null
null,
query -> {
throw new UnsupportedOperationException();
}
),
source,
mappingLookup.getMapping().getRoot(),

5
test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
View file

@ -1284,7 +1284,10 @@ public abstract class AggregatorTestCase extends ESTestCase {
ScriptCompiler.NONE,
null,
indexSettings,
null
null,
query -> {
throw new UnsupportedOperationException();
}
);
}

20
test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
View file

@ -467,6 +467,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings);
scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts);
similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap());
this.bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {}
});
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
mapperService = new MapperService(
clusterService,
@ -478,23 +485,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
() -> createShardContext(null),
idxSettings.getMode().idFieldMapperWithoutFieldData(),
ScriptCompiler.NONE,
bitsetFilterCache::getBitSetProducer,
MapperMetrics.NOOP
);
IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
});
indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, new NoneCircuitBreakerService());
bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
@Override
public void onCache(ShardId shardId, Accountable accountable) {
}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {
}
});
if (registerType) {
mapperService.merge(
"_doc",