mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
Upgrade to new Lucene snapshot (#87932)
This PR uses Lucene-9.3 snapshot in Elasticsearch 8.4. Noticeable changes in this Lucene snapshot: - Merge-on-refresh (disabled) - No more pathological merging - SortedSetDocValues#count for value_count aggs
This commit is contained in:
Nhat Nguyen
GitHub
parent
26c1d33ca4
commit
c2dc6e6ef4
66 changed files with 64 additions and 44 deletions
|
|
@ -1,5 +1,5 @@
|
|||
elasticsearch = 8.4.0
|
||||
lucene = 9.2.0
|
||||
lucene = 9.3.0-snapshot-f3025a56e31
|
||||
|
||||
bundled_jdk_vendor = openjdk
|
||||
bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
|
||||
|
||||
:lucene_version: 9.2.0
|
||||
:lucene_version_path: 9_2_0
|
||||
:lucene_version: 9.3.0
|
||||
:lucene_version_path: 9_3_0
|
||||
:jdk: 11.0.2
|
||||
:jdk_major: 11
|
||||
:build_type: tar
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
6a1b5573015bfbfd94d7324012a3177217af7705
|
||||
|
|
@ -0,0 +1 @@
|
|||
edcd74ce93bda0e2007428dc989f0d855449e154
|
||||
|
|
@ -1 +0,0 @@
|
|||
12e8ba1ca93695819d0251a16584880deac58ae0
|
||||
|
|
@ -0,0 +1 @@
|
|||
6f4e079ad40b708b1b7cd6f41300e6bdb71ade94
|
||||
|
|
@ -1 +0,0 @@
|
|||
47e15ef3815554c73cff7163c70115ea1f18818f
|
||||
|
|
@ -0,0 +1 @@
|
|||
6dd3dedd93129e518eaccd806cd16b067f27d414
|
||||
|
|
@ -1 +0,0 @@
|
|||
22308d4eaab8bf8a2b16cfc9eff97bfc2fb5a508
|
||||
|
|
@ -0,0 +1 @@
|
|||
d510d06738c5e0db79875bb07699bf3acc3b31cc
|
||||
|
|
@ -1 +0,0 @@
|
|||
832f62c39c8c2a77097e2d2d4438bd1642f11f29
|
||||
|
|
@ -0,0 +1 @@
|
|||
57e329682ef82a15901c76e5fdbc5ee233689f2d
|
||||
|
|
@ -1 +0,0 @@
|
|||
f3314a95b461d30e048a932f81ff3c5808dd145f
|
||||
|
|
@ -0,0 +1 @@
|
|||
bd19c2df3970762d730e79a57398010d4dac3ea9
|
||||
|
|
@ -1 +0,0 @@
|
|||
e4fd55524bf85aa7d1ec86f8680faa7b07d95fb4
|
||||
|
|
@ -0,0 +1 @@
|
|||
4b59d393f25bed87d0d8df0f851e0c7d07960d23
|
||||
|
|
@ -1 +0,0 @@
|
|||
85c59dcdd7ac761b7f384475aa687a0ae0afaab2
|
||||
|
|
@ -0,0 +1 @@
|
|||
e9aaeb24bcdaf0f9da2a1f8500511ecb56306223
|
||||
|
|
@ -1 +0,0 @@
|
|||
8bfdb8ff2824a585be6d91d80a52a6d4d15c35e8
|
||||
|
|
@ -0,0 +1 @@
|
|||
d3bba726a5b29eae4dfba27e1e65fdd6a1f4cc2f
|
||||
|
|
@ -1 +0,0 @@
|
|||
a050c43f529572590d8dd5a5bc9f7b64119795b4
|
||||
|
|
@ -0,0 +1 @@
|
|||
5a161d98f65f6d5b0aa5bd881c37d3054db89b09
|
||||
|
|
@ -1 +0,0 @@
|
|||
d2a148922ee01da3f653e931cb572d6dfec1ba3b
|
||||
|
|
@ -0,0 +1 @@
|
|||
f148a80992ae25e893b8959efd818cbb0d3e2121
|
||||
|
|
@ -1 +0,0 @@
|
|||
da636dedae3155ef186b5eaa543093da069ddab1
|
||||
|
|
@ -0,0 +1 @@
|
|||
f8fb2da223a48e2f54a5c94924850b007e5a1f57
|
||||
|
|
@ -1 +0,0 @@
|
|||
97f362ff458b03850b3e0fb45a6cc2773ddbfbfa
|
||||
|
|
@ -0,0 +1 @@
|
|||
48b0e121903b227a8730d23ef56879ce04cdfedf
|
||||
|
|
@ -1 +0,0 @@
|
|||
da43e5472e43db68b8c74f05e63d900ecedc1631
|
||||
|
|
@ -0,0 +1 @@
|
|||
690ea3dcc4606a4599b3e2b2e3510cff7059950c
|
||||
|
|
@ -1 +0,0 @@
|
|||
b1ea8b82a036cbff93a9c849cbf574c6730a7b13
|
||||
|
|
@ -0,0 +1 @@
|
|||
32cdfad19d7bbdc7f3052bf6c9d9871263cfd1c8
|
||||
|
|
@ -1 +0,0 @@
|
|||
c447cad35d879bd656f8a0aeb3114c08e25ca1b1
|
||||
|
|
@ -0,0 +1 @@
|
|||
366231787c733fc0c4fb373eaf507a66ba339dd6
|
||||
|
|
@ -1 +0,0 @@
|
|||
4652557ef1d68b0046f0bb28762ede953f6367ef
|
||||
|
|
@ -0,0 +1 @@
|
|||
08837eab0f0836bba2f82827acc3eab3006b16b9
|
||||
|
|
@ -1 +0,0 @@
|
|||
6c9aa37760c11c033f154170c15c2b1961b7a886
|
||||
|
|
@ -0,0 +1 @@
|
|||
bc550835d4a040f858f88f845dcb66573ca3c19e
|
||||
|
|
@ -1 +0,0 @@
|
|||
c51ef9a5894dfb4548bbf80d1a271cfe8e86cbf6
|
||||
|
|
@ -0,0 +1 @@
|
|||
506d4ddd2ac3386df063f279113006694889ae53
|
||||
|
|
@ -1 +0,0 @@
|
|||
fcb32402e0cba93454675cb631d59264968b32a4
|
||||
|
|
@ -0,0 +1 @@
|
|||
1aa664e0ac15d7d12723674d4b65d8969be9f8c2
|
||||
|
|
@ -1 +0,0 @@
|
|||
7bbcadf643c6bed8a15d789c71cd89a8c9dddf31
|
||||
|
|
@ -0,0 +1 @@
|
|||
6350f41cb9117e4499c15bd060ec2cf7b48970d9
|
||||
|
|
@ -1 +0,0 @@
|
|||
aedb9a641278845f81cb004d6bc557eb43f69a57
|
||||
|
|
@ -0,0 +1 @@
|
|||
b36d0d959d2decffe5f9ea68130f28470637ca5c
|
||||
|
|
@ -1 +0,0 @@
|
|||
608e3851216dc1d8d85f9389c71241f2b395f1ea
|
||||
|
|
@ -0,0 +1 @@
|
|||
b0220d2ce0e68879535246013601a09323106875
|
||||
|
|
@ -114,7 +114,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final Version V_8_2_3 = new Version(8_02_03_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
|
||||
public static final Version V_8_2_4 = new Version(8_02_04_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
|
||||
public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
|
||||
public static final Version V_8_4_0 = new Version(8_04_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
|
||||
public static final Version V_8_4_0 = new Version(8_04_00_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
|
||||
public static final Version CURRENT = V_8_4_0;
|
||||
|
||||
private static final Map<Integer, Version> idToVersion;
|
||||
|
|
|
|||
|
|
@ -2408,6 +2408,8 @@ public class InternalEngine extends Engine {
|
|||
mergePolicy = new ShuffleForcedMergePolicy(mergePolicy);
|
||||
}
|
||||
iwc.setMergePolicy(mergePolicy);
|
||||
// TODO: Introduce an index setting for setMaxFullFlushMergeWaitMillis
|
||||
iwc.setMaxFullFlushMergeWaitMillis(-1);
|
||||
iwc.setSimilarity(engineConfig.getSimilarity());
|
||||
iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
|
||||
iwc.setCodec(engineConfig.getCodec());
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -190,8 +190,8 @@ public class MultiOrdinals extends Ordinals {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return currentEndOffset - currentOffset;
|
||||
public int docValueCount() {
|
||||
return Math.toIntExact(currentEndOffset - currentOffset);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -460,7 +460,7 @@ class DocumentLeafReader extends LeafReader {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return values.size();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ public class KeyedFlattenedLeafFieldData implements LeafOrdinalsFieldData {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return delegate.docValueCount();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ public enum MissingValues {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
|
|
@ -319,7 +319,7 @@ public enum MissingValues {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -517,7 +517,9 @@ public class LuceneTests extends ESTestCase {
|
|||
public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
|
||||
.setMergePolicy(new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, MatchAllDocsQuery::new, newMergePolicy()));
|
||||
.setMergePolicy(new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, MatchAllDocsQuery::new, newMergePolicy()))
|
||||
// disable merges on refresh as we will verify the deleted documents
|
||||
.setMaxFullFlushMergeWaitMillis(-1);
|
||||
IndexWriter writer = new IndexWriter(dir, config);
|
||||
int numDocs = between(1, 10);
|
||||
List<String> liveDocs = new ArrayList<>();
|
||||
|
|
|
|||
|
|
@ -200,7 +200,7 @@ public class KeyedFlattenedLeafFieldDataTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return documentOrds.length;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -724,7 +724,7 @@ public class MultiValueModeTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return array[doc].length;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ public class BinaryRangeAggregatorTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return ords.length;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ public class IncludeExcludeTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ public class MissingValuesTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
return ords[doc].length;
|
||||
}
|
||||
};
|
||||
|
|
@ -232,7 +232,7 @@ public class MissingValuesTests extends ESTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
public int docValueCount() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
832f62c39c8c2a77097e2d2d4438bd1642f11f29
|
||||
|
|
@ -0,0 +1 @@
|
|||
57e329682ef82a15901c76e5fdbc5ee233689f2d
|
||||
|
|
@ -56,6 +56,12 @@ public abstract class LegacySortedSetDocValues {
|
|||
*/
|
||||
public abstract long nextOrd();
|
||||
|
||||
/**
|
||||
* Retrieves the number of unique ords for the current document. This must always be greater than
|
||||
* zero. It is illegal to call this method after advanceExact(int) returned {@code false}.
|
||||
*/
|
||||
public abstract int docValueCount();
|
||||
|
||||
/**
|
||||
* Sets iteration to the specified docID
|
||||
* @param docID document ID
|
||||
|
|
|
|||
|
|
@ -99,8 +99,8 @@ public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues {
|
|||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
throw new UnsupportedOperationException();
|
||||
public int docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1351,6 +1351,11 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
|
|||
return binary.get(ord);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int docValueCount() {
|
||||
return Math.toIntExact(endOffset - startOffset);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueCount() {
|
||||
return valueCount;
|
||||
|
|
@ -1415,6 +1420,11 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
|
|||
return valueCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int docValueCount() {
|
||||
return Math.toIntExact(endOffset - startOffset);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long lookupTerm(BytesRef key) {
|
||||
if (binary instanceof CompressedBinaryDocValues) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue