Upgrade to new Lucene snapshot (#87932)

This PR uses Lucene-9.3 snapshot in Elasticsearch 8.4. Noticeable changes in this Lucene snapshot:

- Merge-on-refresh (disabled)
- No more pathological merging
- SortedSetDocValues#count for value_count aggs

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.4.0
-lucene            = 9.2.0
+lucene            = 9.3.0-snapshot-f3025a56e31
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.2.0
+:lucene_version:        9.3.0
-:lucene_version_path:   9_2_0
+:lucene_version_path:   9_3_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

modules/lang-expression/licenses/lucene-codecs-9.2.0.jar.sha1

@@ -1 +0,0 @@
-6a1b5573015bfbfd94d7324012a3177217af7705

modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+edcd74ce93bda0e2007428dc989f0d855449e154

modules/lang-expression/licenses/lucene-expressions-9.2.0.jar.sha1

@@ -1 +0,0 @@
-12e8ba1ca93695819d0251a16584880deac58ae0

modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+6f4e079ad40b708b1b7cd6f41300e6bdb71ade94

modules/legacy-geo/licenses/lucene-spatial-extras-9.2.0.jar.sha1

@@ -1 +0,0 @@
-47e15ef3815554c73cff7163c70115ea1f18818f

modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+6dd3dedd93129e518eaccd806cd16b067f27d414

modules/legacy-geo/licenses/lucene-spatial3d-9.2.0.jar.sha1

@@ -1 +0,0 @@
-22308d4eaab8bf8a2b16cfc9eff97bfc2fb5a508

modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+d510d06738c5e0db79875bb07699bf3acc3b31cc

plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0.jar.sha1

@@ -1 +0,0 @@
-832f62c39c8c2a77097e2d2d4438bd1642f11f29

plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+57e329682ef82a15901c76e5fdbc5ee233689f2d

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0.jar.sha1

@@ -1 +0,0 @@
-f3314a95b461d30e048a932f81ff3c5808dd145f

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+bd19c2df3970762d730e79a57398010d4dac3ea9

plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0.jar.sha1

@@ -1 +0,0 @@
-e4fd55524bf85aa7d1ec86f8680faa7b07d95fb4

plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+4b59d393f25bed87d0d8df0f851e0c7d07960d23

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0.jar.sha1

@@ -1 +0,0 @@
-85c59dcdd7ac761b7f384475aa687a0ae0afaab2

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+e9aaeb24bcdaf0f9da2a1f8500511ecb56306223

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0.jar.sha1

@@ -1 +0,0 @@
-8bfdb8ff2824a585be6d91d80a52a6d4d15c35e8

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+d3bba726a5b29eae4dfba27e1e65fdd6a1f4cc2f

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0.jar.sha1

@@ -1 +0,0 @@
-a050c43f529572590d8dd5a5bc9f7b64119795b4

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+5a161d98f65f6d5b0aa5bd881c37d3054db89b09

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0.jar.sha1

@@ -1 +0,0 @@
-d2a148922ee01da3f653e931cb572d6dfec1ba3b

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+f148a80992ae25e893b8959efd818cbb0d3e2121

server/licenses/lucene-analysis-common-9.2.0.jar.sha1

@@ -1 +0,0 @@
-da636dedae3155ef186b5eaa543093da069ddab1

server/licenses/lucene-analysis-common-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+f8fb2da223a48e2f54a5c94924850b007e5a1f57

server/licenses/lucene-backward-codecs-9.2.0.jar.sha1

@@ -1 +0,0 @@
-97f362ff458b03850b3e0fb45a6cc2773ddbfbfa

server/licenses/lucene-backward-codecs-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+48b0e121903b227a8730d23ef56879ce04cdfedf

server/licenses/lucene-core-9.2.0.jar.sha1

@@ -1 +0,0 @@
-da43e5472e43db68b8c74f05e63d900ecedc1631

server/licenses/lucene-core-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+690ea3dcc4606a4599b3e2b2e3510cff7059950c

server/licenses/lucene-grouping-9.2.0.jar.sha1

@@ -1 +0,0 @@
-b1ea8b82a036cbff93a9c849cbf574c6730a7b13

server/licenses/lucene-grouping-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+32cdfad19d7bbdc7f3052bf6c9d9871263cfd1c8

server/licenses/lucene-highlighter-9.2.0.jar.sha1

@@ -1 +0,0 @@
-c447cad35d879bd656f8a0aeb3114c08e25ca1b1

server/licenses/lucene-highlighter-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+366231787c733fc0c4fb373eaf507a66ba339dd6

server/licenses/lucene-join-9.2.0.jar.sha1

@@ -1 +0,0 @@
-4652557ef1d68b0046f0bb28762ede953f6367ef

server/licenses/lucene-join-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+08837eab0f0836bba2f82827acc3eab3006b16b9

server/licenses/lucene-memory-9.2.0.jar.sha1

@@ -1 +0,0 @@
-6c9aa37760c11c033f154170c15c2b1961b7a886

server/licenses/lucene-memory-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+bc550835d4a040f858f88f845dcb66573ca3c19e

server/licenses/lucene-misc-9.2.0.jar.sha1

@@ -1 +0,0 @@
-c51ef9a5894dfb4548bbf80d1a271cfe8e86cbf6

server/licenses/lucene-misc-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+506d4ddd2ac3386df063f279113006694889ae53

server/licenses/lucene-queries-9.2.0.jar.sha1

@@ -1 +0,0 @@
-fcb32402e0cba93454675cb631d59264968b32a4

server/licenses/lucene-queries-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+1aa664e0ac15d7d12723674d4b65d8969be9f8c2

server/licenses/lucene-queryparser-9.2.0.jar.sha1

@@ -1 +0,0 @@
-7bbcadf643c6bed8a15d789c71cd89a8c9dddf31

server/licenses/lucene-queryparser-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+6350f41cb9117e4499c15bd060ec2cf7b48970d9

server/licenses/lucene-sandbox-9.2.0.jar.sha1

@@ -1 +0,0 @@
-aedb9a641278845f81cb004d6bc557eb43f69a57

server/licenses/lucene-sandbox-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+b36d0d959d2decffe5f9ea68130f28470637ca5c

server/licenses/lucene-suggest-9.2.0.jar.sha1

@@ -1 +0,0 @@
-608e3851216dc1d8d85f9389c71241f2b395f1ea

server/licenses/lucene-suggest-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+b0220d2ce0e68879535246013601a09323106875

server/src/main/java/org/elasticsearch/Version.java

@@ -114,7 +114,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_8_2_3 = new Version(8_02_03_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_8_2_4 = new Version(8_02_04_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
-    public static final Version V_8_4_0 = new Version(8_04_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
+    public static final Version V_8_4_0 = new Version(8_04_00_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
     public static final Version CURRENT = V_8_4_0;
 
     private static final Map<Integer, Version> idToVersion;

server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -2408,6 +2408,8 @@ public class InternalEngine extends Engine {
             mergePolicy = new ShuffleForcedMergePolicy(mergePolicy);
         }
         iwc.setMergePolicy(mergePolicy);
+        // TODO: Introduce an index setting for setMaxFullFlushMergeWaitMillis
+        iwc.setMaxFullFlushMergeWaitMillis(-1);
         iwc.setSimilarity(engineConfig.getSimilarity());
         iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
         iwc.setCodec(engineConfig.getCodec());

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java

@@ -60,7 +60,7 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
     }
 
     @Override
-    public long docValueCount() {
+    public int docValueCount() {
         return values.docValueCount();
     }
 

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java

@@ -190,8 +190,8 @@ public class MultiOrdinals extends Ordinals {
         }
 
         @Override
-        public long docValueCount() {
+        public int docValueCount() {
-            return currentEndOffset - currentOffset;
+            return Math.toIntExact(currentEndOffset - currentOffset);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java

@@ -460,7 +460,7 @@ class DocumentLeafReader extends LeafReader {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return values.size();
             }
 

server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java

@@ -210,7 +210,7 @@ public class KeyedFlattenedLeafFieldData implements LeafOrdinalsFieldData {
         }
 
         @Override
-        public long docValueCount() {
+        public int docValueCount() {
             return delegate.docValueCount();
         }
 

server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java

@@ -280,7 +280,7 @@ public enum MissingValues {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return values.docValueCount();
             }
 
@@ -319,7 +319,7 @@ public enum MissingValues {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return values.docValueCount();
             }
 

server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java

@@ -517,7 +517,9 @@ public class LuceneTests extends ESTestCase {
     public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
-            .setMergePolicy(new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, MatchAllDocsQuery::new, newMergePolicy()));
+            .setMergePolicy(new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, MatchAllDocsQuery::new, newMergePolicy()))
+            // disable merges on refresh as we will verify the deleted documents
+            .setMaxFullFlushMergeWaitMillis(-1);
         IndexWriter writer = new IndexWriter(dir, config);
         int numDocs = between(1, 10);
         List<String> liveDocs = new ArrayList<>();

server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java

@@ -200,7 +200,7 @@ public class KeyedFlattenedLeafFieldDataTests extends ESTestCase {
         }
 
         @Override
-        public long docValueCount() {
+        public int docValueCount() {
             return documentOrds.length;
         }
 

server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java

@@ -724,7 +724,7 @@ public class MultiValueModeTests extends ESTestCase {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return array[doc].length;
             }
 

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java

@@ -48,7 +48,7 @@ public class BinaryRangeAggregatorTests extends ESTestCase {
         }
 
         @Override
-        public long docValueCount() {
+        public int docValueCount() {
             return ords.length;
         }
 

server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java

@@ -67,7 +67,7 @@ public class IncludeExcludeTests extends ESTestCase {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return 1;
             }
 

server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java

@@ -131,7 +131,7 @@ public class MissingValuesTests extends ESTestCase {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 return ords[doc].length;
             }
         };
@@ -232,7 +232,7 @@ public class MissingValuesTests extends ESTestCase {
             }
 
             @Override
-            public long docValueCount() {
+            public int docValueCount() {
                 throw new UnsupportedOperationException();
             }
 

x-pack/plugin/ml/licenses/lucene-analysis-icu-9.2.0.jar.sha1

@@ -1 +0,0 @@
-832f62c39c8c2a77097e2d2d4438bd1642f11f29

x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-f3025a56e31.jar.sha1

@@ -0,0 +1 @@
+57e329682ef82a15901c76e5fdbc5ee233689f2d

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValues.java

@@ -56,6 +56,12 @@ public abstract class LegacySortedSetDocValues {
      */
     public abstract long nextOrd();
 
+    /**
+     * Retrieves the number of unique ords for the current document. This must always be greater than
+     * zero. It is illegal to call this method after advanceExact(int) returned {@code false}.
+     */
+    public abstract int docValueCount();
+
     /**
      * Sets iteration to the specified docID
      * @param docID document ID

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java

@@ -99,8 +99,8 @@ public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues {
     }
 
     @Override
-    public long docValueCount() {
+    public int docValueCount() {
-        throw new UnsupportedOperationException();
+        return values.docValueCount();
     }
 
     @Override

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene54/Lucene54DocValuesProducer.java

@@ -1351,6 +1351,11 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
                 return binary.get(ord);
             }
 
+            @Override
+            public int docValueCount() {
+                return Math.toIntExact(endOffset - startOffset);
+            }
+
             @Override
             public long getValueCount() {
                 return valueCount;
@@ -1415,6 +1420,11 @@ final class Lucene54DocValuesProducer extends DocValuesProducer implements Close
                 return valueCount;
             }
 
+            @Override
+            public int docValueCount() {
+                return Math.toIntExact(endOffset - startOffset);
+            }
+
             @Override
             public long lookupTerm(BytesRef key) {
                 if (binary instanceof CompressedBinaryDocValues) {