github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-17 20:05:09 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use Lucene101 postings format by default (#126080)

Browse source 
Update the PerFieldFormatSupplier so that new standard indices use the
Lucene101PostingsFormat instead of the current default ES812PostingsFormat.

Currently, use of the new codec is gated behind a feature flag.
This commit is contained in:
Jordan Powers 2025-04-04 12:41:27 -07:00 • committed by GitHub
parent fce28cc82a
commit 4c174a891f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 38 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
View file

@ -37,6 +37,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
.feature(FeatureFlag.TIME_SERIES_MODE)
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
.feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
.build();
public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

1
rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
View file

@ -37,6 +37,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
.feature(FeatureFlag.TIME_SERIES_MODE)
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
.feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
.build();
public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

1
server/src/main/java/org/elasticsearch/index/IndexVersions.java
View file

@ -158,6 +158,7 @@ public class IndexVersions {
public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS = def(9_018_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG = def(9_019_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0);
public static final IndexVersion USE_LUCENE101_POSTINGS_FORMAT = def(9_021_0_00, Version.LUCENE_10_1_0);
/*
* STOP! READ THIS FIRST! No, really,
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

21
server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
View file

@ -12,11 +12,14 @@ package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.FeatureFlag;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
import org.elasticsearch.index.codec.postings.ES812PostingsFormat;
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
@ -31,19 +34,33 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
* vectors.
*/
public class PerFieldFormatSupplier {
public static final FeatureFlag USE_LUCENE101_POSTINGS_FORMAT = new FeatureFlag("use_lucene101_postings_format");
private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
private static final Lucene101PostingsFormat lucene101PostingsFormat = new Lucene101PostingsFormat();
private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
private final MapperService mapperService;
private final PostingsFormat defaultPostingsFormat;
public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
this.mapperService = mapperService;
this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
if (mapperService != null
&& USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
&& mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.USE_LUCENE101_POSTINGS_FORMAT)
&& mapperService.getIndexSettings().getMode() == IndexMode.STANDARD) {
defaultPostingsFormat = lucene101PostingsFormat;
} else {
// our own posting format using PFOR
defaultPostingsFormat = es812PostingsFormat;
}
}
public PostingsFormat getPostingsFormatForField(String field) {
@ -60,8 +77,8 @@ public class PerFieldFormatSupplier {
return completionPostingsFormat;
}
}
// return our own posting format using PFOR
return es812PostingsFormat;
return defaultPostingsFormat;
}
boolean useBloomFilter(String field) {

15
server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
View file

@ -9,6 +9,8 @@
package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
@ -86,11 +88,15 @@ public class PerFieldMapperCodecTests extends ESTestCase {
""";
public void testUseBloomFilter() throws IOException {
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, randomBoolean(), false);
boolean timeSeries = randomBoolean();
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, timeSeries, false);
assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
assertThat(perFieldMapperCodec.getPostingsFormatForField("_id"), instanceOf(ES87BloomFilterPostingsFormat.class));
assertThat(perFieldMapperCodec.useBloomFilter("another_field"), is(false));
assertThat(perFieldMapperCodec.getPostingsFormatForField("another_field"), instanceOf(ES812PostingsFormat.class));
Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
&& timeSeries == false ? Lucene101PostingsFormat.class : ES812PostingsFormat.class;
assertThat(perFieldMapperCodec.getPostingsFormatForField("another_field"), instanceOf(expectedPostingsFormat));
}
public void testUseBloomFilterWithTimestampFieldEnabled() throws IOException {
@ -104,7 +110,10 @@ public class PerFieldMapperCodecTests extends ESTestCase {
public void testUseBloomFilterWithTimestampFieldEnabled_noTimeSeriesMode() throws IOException {
PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, false);
assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(false));
assertThat(perFieldMapperCodec.getPostingsFormatForField("_id"), instanceOf(ES812PostingsFormat.class));
Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
? Lucene101PostingsFormat.class
: ES812PostingsFormat.class;
assertThat(perFieldMapperCodec.getPostingsFormatForField("_id"), instanceOf(expectedPostingsFormat));
}
public void testUseBloomFilterWithTimestampFieldEnabled_disableBloomFilter() throws IOException {

3
test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
View file

@ -19,7 +19,8 @@ public enum FeatureFlag {
TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null),
FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null),
SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null),
DOC_VALUES_SKIPPER("es.doc_values_skipper_feature_flag_enabled=true", Version.fromString("8.18.1"), null);
DOC_VALUES_SKIPPER("es.doc_values_skipper_feature_flag_enabled=true", Version.fromString("8.18.1"), null),
USE_LUCENE101_POSTINGS_FORMAT("es.use_lucene101_postings_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null);
public final String systemProperty;
public final Version from;

1
x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java
View file

@ -25,6 +25,7 @@ public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase {
.setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial")
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
.feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
.build();
public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

1
x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
View file

@ -51,6 +51,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest
.feature(FeatureFlag.TIME_SERIES_MODE)
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
.feature(FeatureFlag.DOC_VALUES_SKIPPER)
.feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
.build();
public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {