mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
Upgrade to Lucene-9.3.0-snapshot-2d05f5c623e (#88284)
To include LUCENE-10620 - which passes Weight to Collector
This commit is contained in:
Nhat Nguyen
GitHub
parent
945cb27782
commit
bd69f90fff
58 changed files with 60 additions and 58 deletions
|
|
@ -1,5 +1,5 @@
|
|||
elasticsearch = 8.4.0
|
||||
lucene = 9.3.0-snapshot-f3025a56e31
|
||||
lucene = 9.3.0-snapshot-2d05f5c623e
|
||||
|
||||
bundled_jdk_vendor = openjdk
|
||||
bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
a2b0a4e1d763c5575f289ef7b4d8f109e7ee895f
|
||||
|
|
@ -1 +0,0 @@
|
|||
edcd74ce93bda0e2007428dc989f0d855449e154
|
||||
|
|
@ -0,0 +1 @@
|
|||
37603372705778c2c755d055ac72f835af0ebe51
|
||||
|
|
@ -1 +0,0 @@
|
|||
6f4e079ad40b708b1b7cd6f41300e6bdb71ade94
|
||||
|
|
@ -0,0 +1 @@
|
|||
899069409ff5a448c641700a0371ce792a0635b1
|
||||
|
|
@ -1 +0,0 @@
|
|||
6dd3dedd93129e518eaccd806cd16b067f27d414
|
||||
|
|
@ -0,0 +1 @@
|
|||
3416a5a2becfd987049453c939afa2624c3db6c7
|
||||
|
|
@ -1 +0,0 @@
|
|||
d510d06738c5e0db79875bb07699bf3acc3b31cc
|
||||
|
|
@ -0,0 +1 @@
|
|||
87772b1d9907c63287d7bc79cf8fa3e9a46c9c77
|
||||
|
|
@ -1 +0,0 @@
|
|||
57e329682ef82a15901c76e5fdbc5ee233689f2d
|
||||
|
|
@ -0,0 +1 @@
|
|||
a58272a5eb0137dbcb030a7f2cd7925dd717c3e8
|
||||
|
|
@ -1 +0,0 @@
|
|||
bd19c2df3970762d730e79a57398010d4dac3ea9
|
||||
|
|
@ -0,0 +1 @@
|
|||
e15673b088608c7b7e2b5ab491665f9ea92941b2
|
||||
|
|
@ -1 +0,0 @@
|
|||
4b59d393f25bed87d0d8df0f851e0c7d07960d23
|
||||
|
|
@ -0,0 +1 @@
|
|||
f997bf7bdd07a5684e017294764b0a2ca398f948
|
||||
|
|
@ -1 +0,0 @@
|
|||
e9aaeb24bcdaf0f9da2a1f8500511ecb56306223
|
||||
|
|
@ -0,0 +1 @@
|
|||
7cdabbc7114cb7f2925407dd7e18fe6e36fbfc02
|
||||
|
|
@ -1 +0,0 @@
|
|||
d3bba726a5b29eae4dfba27e1e65fdd6a1f4cc2f
|
||||
|
|
@ -0,0 +1 @@
|
|||
56cd7a7ae2f9db55fc4150f60745f900a1018e35
|
||||
|
|
@ -1 +0,0 @@
|
|||
5a161d98f65f6d5b0aa5bd881c37d3054db89b09
|
||||
|
|
@ -0,0 +1 @@
|
|||
d8a80d858ca656735e3f3e0e3803f9130a504f58
|
||||
|
|
@ -1 +0,0 @@
|
|||
f148a80992ae25e893b8959efd818cbb0d3e2121
|
||||
|
|
@ -0,0 +1 @@
|
|||
ef6b338e156eacb5ae63daa9638899b4b0c44c15
|
||||
|
|
@ -1 +0,0 @@
|
|||
f8fb2da223a48e2f54a5c94924850b007e5a1f57
|
||||
|
|
@ -0,0 +1 @@
|
|||
25886039976045fb8f9859e12f525eedbb64860a
|
||||
|
|
@ -1 +0,0 @@
|
|||
48b0e121903b227a8730d23ef56879ce04cdfedf
|
||||
|
|
@ -0,0 +1 @@
|
|||
98b0f69f9d285dcf822bcd90974e8aa4972187b9
|
||||
|
|
@ -1 +0,0 @@
|
|||
690ea3dcc4606a4599b3e2b2e3510cff7059950c
|
||||
|
|
@ -0,0 +1 @@
|
|||
6f6126445ac9b86ef93c6ab26907a69931ada790
|
||||
|
|
@ -1 +0,0 @@
|
|||
32cdfad19d7bbdc7f3052bf6c9d9871263cfd1c8
|
||||
|
|
@ -0,0 +1 @@
|
|||
dd6fafd8701b01a3f6279447f7d7c410b1230c13
|
||||
|
|
@ -1 +0,0 @@
|
|||
366231787c733fc0c4fb373eaf507a66ba339dd6
|
||||
|
|
@ -0,0 +1 @@
|
|||
2bc4615335ba485b2152cd10d21ca0cb768b651e
|
||||
|
|
@ -1 +0,0 @@
|
|||
08837eab0f0836bba2f82827acc3eab3006b16b9
|
||||
|
|
@ -0,0 +1 @@
|
|||
3972925a8a1e304e803491881e67c12cbcda4617
|
||||
|
|
@ -1 +0,0 @@
|
|||
bc550835d4a040f858f88f845dcb66573ca3c19e
|
||||
|
|
@ -0,0 +1 @@
|
|||
18c2adc5e45d08d65e64d31402219b67d7bc5c61
|
||||
|
|
@ -1 +0,0 @@
|
|||
506d4ddd2ac3386df063f279113006694889ae53
|
||||
|
|
@ -0,0 +1 @@
|
|||
bc9e4af7d35799ba941993be0242101f79719cc6
|
||||
|
|
@ -1 +0,0 @@
|
|||
1aa664e0ac15d7d12723674d4b65d8969be9f8c2
|
||||
|
|
@ -0,0 +1 @@
|
|||
f62b8f566e37fd6cf150f13d866fcc646c210e35
|
||||
|
|
@ -1 +0,0 @@
|
|||
6350f41cb9117e4499c15bd060ec2cf7b48970d9
|
||||
|
|
@ -0,0 +1 @@
|
|||
081ff7330c13bfcf0110e7140badfc8be3df910c
|
||||
|
|
@ -1 +0,0 @@
|
|||
b36d0d959d2decffe5f9ea68130f28470637ca5c
|
||||
|
|
@ -0,0 +1 @@
|
|||
e3240b6ab35e61a947d94978697816d102b1bd13
|
||||
|
|
@ -1 +0,0 @@
|
|||
b0220d2ce0e68879535246013601a09323106875
|
||||
|
|
@ -86,7 +86,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
public class Lucene {
|
||||
public static final String LATEST_CODEC = "Lucene92";
|
||||
public static final String LATEST_CODEC = "Lucene93";
|
||||
|
||||
public static final String SOFT_DELETES_FIELD = "__soft_deletes";
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
package org.elasticsearch.index.codec;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
|
||||
import org.elasticsearch.core.Nullable;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
||||
|
|
@ -34,11 +34,11 @@ public class CodecService {
|
|||
public CodecService(@Nullable MapperService mapperService) {
|
||||
final var codecs = new HashMap<String, Codec>();
|
||||
if (mapperService == null) {
|
||||
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
|
||||
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Lucene92Codec.Mode.BEST_COMPRESSION));
|
||||
codecs.put(DEFAULT_CODEC, new Lucene93Codec());
|
||||
codecs.put(BEST_COMPRESSION_CODEC, new Lucene93Codec(Lucene93Codec.Mode.BEST_COMPRESSION));
|
||||
} else {
|
||||
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_SPEED, mapperService));
|
||||
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_COMPRESSION, mapperService));
|
||||
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_SPEED, mapperService));
|
||||
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_COMPRESSION, mapperService));
|
||||
}
|
||||
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
|
||||
for (String codec : Codec.availableCodecs()) {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
|
|||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
|
||||
import org.elasticsearch.common.lucene.Lucene;
|
||||
import org.elasticsearch.index.mapper.Mapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
|
@ -27,7 +27,7 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
|
|||
* per index in real time via the mapping API. If no specific postings format or vector format is
|
||||
* configured for a specific field the default postings or vector format is used.
|
||||
*/
|
||||
public class PerFieldMapperCodec extends Lucene92Codec {
|
||||
public class PerFieldMapperCodec extends Lucene93Codec {
|
||||
private final MapperService mapperService;
|
||||
|
||||
private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
package org.elasticsearch.index.mapper.vectors;
|
||||
|
||||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat;
|
||||
import org.apache.lucene.document.BinaryDocValuesField;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.KnnVectorField;
|
||||
|
|
@ -529,7 +529,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
return null; // use default format
|
||||
} else {
|
||||
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
|
||||
return new Lucene92HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
|
||||
return new Lucene93HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,7 +162,10 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
|
||||
weight = wrapWeight(weight);
|
||||
collector.setWeight(weight);
|
||||
for (LeafReaderContext ctx : leaves) { // search each subreader
|
||||
searchLeaf(ctx, weight, collector);
|
||||
}
|
||||
|
|
@ -176,7 +179,6 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
|
|||
*/
|
||||
private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
|
||||
cancellable.checkCancelled();
|
||||
weight = wrapWeight(weight);
|
||||
final LeafCollector leafCollector;
|
||||
try {
|
||||
leafCollector = collector.getLeafCollector(ctx);
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
|
|||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
||||
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
|
||||
import org.apache.lucene.document.BinaryDocValuesField;
|
||||
|
|
@ -297,7 +297,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
|||
public void testCompletionField() throws Exception {
|
||||
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
||||
.setUseCompoundFile(false)
|
||||
.setCodec(new Lucene92Codec(Lucene92Codec.Mode.BEST_SPEED) {
|
||||
.setCodec(new Lucene93Codec(Lucene93Codec.Mode.BEST_SPEED) {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
if (field.startsWith("suggest_")) {
|
||||
|
|
@ -384,25 +384,25 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
|||
enum CodecMode {
|
||||
BEST_SPEED {
|
||||
@Override
|
||||
Lucene92Codec.Mode mode() {
|
||||
return Lucene92Codec.Mode.BEST_SPEED;
|
||||
Lucene93Codec.Mode mode() {
|
||||
return Lucene93Codec.Mode.BEST_SPEED;
|
||||
}
|
||||
},
|
||||
|
||||
BEST_COMPRESSION {
|
||||
@Override
|
||||
Lucene92Codec.Mode mode() {
|
||||
return Lucene92Codec.Mode.BEST_COMPRESSION;
|
||||
Lucene93Codec.Mode mode() {
|
||||
return Lucene93Codec.Mode.BEST_COMPRESSION;
|
||||
}
|
||||
};
|
||||
|
||||
abstract Lucene92Codec.Mode mode();
|
||||
abstract Lucene93Codec.Mode mode();
|
||||
}
|
||||
|
||||
static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
|
||||
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
||||
.setUseCompoundFile(randomBoolean())
|
||||
.setCodec(new Lucene92Codec(codecMode.mode()));
|
||||
.setCodec(new Lucene93Codec(codecMode.mode()));
|
||||
try (IndexWriter writer = new IndexWriter(directory, config)) {
|
||||
for (int i = 0; i < numDocs; i++) {
|
||||
final Document doc = new Document();
|
||||
|
|
@ -580,7 +580,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
|||
try (DirectoryReader reader = DirectoryReader.open(source)) {
|
||||
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
|
||||
.setUseCompoundFile(randomBoolean())
|
||||
.setCodec(new Lucene92Codec(mode.mode()) {
|
||||
.setCodec(new Lucene93Codec(mode.mode()) {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
return new Lucene90PostingsFormat();
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ package org.elasticsearch.index.codec;
|
|||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
|
|
@ -41,21 +41,21 @@ public class CodecTests extends ESTestCase {
|
|||
public void testResolveDefaultCodecs() throws Exception {
|
||||
CodecService codecService = createCodecService();
|
||||
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
|
||||
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
|
||||
assertThat(codecService.codec("default"), instanceOf(Lucene93Codec.class));
|
||||
}
|
||||
|
||||
public void testDefault() throws Exception {
|
||||
Codec codec = createCodecService().codec("default");
|
||||
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
|
||||
assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_SPEED, codec);
|
||||
}
|
||||
|
||||
public void testBestCompression() throws Exception {
|
||||
Codec codec = createCodecService().codec("best_compression");
|
||||
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
|
||||
assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_COMPRESSION, codec);
|
||||
}
|
||||
|
||||
// write some docs with it, inspect .si to see this was the used compression
|
||||
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
|
||||
private void assertStoredFieldsCompressionEquals(Lucene93Codec.Mode expected, Codec actual) throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(null);
|
||||
iwc.setCodec(actual);
|
||||
|
|
@ -67,7 +67,7 @@ public class CodecTests extends ESTestCase {
|
|||
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
|
||||
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
|
||||
assertNotNull(v);
|
||||
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
|
||||
assertEquals(expected, Lucene93Codec.Mode.valueOf(v));
|
||||
ir.close();
|
||||
dir.close();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
package org.elasticsearch.index.engine;
|
||||
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93Codec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
|
|
@ -45,7 +45,7 @@ public class CompletionStatsCacheTests extends ESTestCase {
|
|||
public void testCompletionStatsCache() throws IOException, InterruptedException {
|
||||
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
|
||||
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
|
||||
indexWriterConfig.setCodec(new Lucene92Codec() {
|
||||
indexWriterConfig.setCodec(new Lucene93Codec() {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
return postingsFormat; // all fields are suggest fields
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
|||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat;
|
||||
import org.apache.lucene.document.BinaryDocValuesField;
|
||||
import org.apache.lucene.document.KnnVectorField;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
|
|
@ -40,8 +40,8 @@ import java.nio.ByteBuffer;
|
|||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
||||
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN;
|
||||
import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
||||
import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_MAX_CONN;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
|
@ -462,8 +462,8 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
|
|||
Codec codec = codecService.codec("default");
|
||||
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
|
||||
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
|
||||
assertThat(knnVectorsFormat, instanceOf(Lucene92HnswVectorsFormat.class));
|
||||
String expectedString = "lucene92HnswVectorsFormat(name = lucene92HnswVectorsFormat, maxConn = "
|
||||
assertThat(knnVectorsFormat, instanceOf(Lucene93HnswVectorsFormat.class));
|
||||
String expectedString = "Lucene93HnswVectorsFormat(name=Lucene93HnswVectorsFormat, maxConn="
|
||||
+ m
|
||||
+ ", beamWidth="
|
||||
+ efConstruction
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
87772b1d9907c63287d7bc79cf8fa3e9a46c9c77
|
||||
|
|
@ -1 +0,0 @@
|
|||
57e329682ef82a15901c76e5fdbc5ee233689f2d
|
||||
Loading…
Add table
Add a link
Reference in a new issue