Upgrade to Lucene 9.2 snapshot efa5d6f4d43 (#86227)

Notable changes include:

count implementations for MultiRangeQuery and IndexSortedNumericDocValuesRangeQuery, which may speed up certain aggregations
more efficient decoding of docids in BKD reader

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.3.0
-lucene            = 9.1.0
+lucene            = 9.2.0-snapshot-efa5d6f4d43
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 18+36

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.1.0
+:lucene_version:        9.2.0
-:lucene_version_path:   9_1_0
+:lucene_version_path:   9_2_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/86227.yaml

@@ -0,0 +1,5 @@
+pr: 86227
+summary: Upgrade to Lucene 9.2 snapshot
+area: Search
+type: upgrade
+issues: []

libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java

@@ -26,7 +26,6 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.URI;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
@@ -158,10 +157,8 @@ public class IOUtilsTests extends ESTestCase {
         for (int i = 0; i < numberOfLocations; i++) {
             if (exception && randomBoolean()) {
                 final Path location = createTempDir();
-                final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
+                final FilterFileSystemProvider fsProvider = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
-                    URI.create("file:///")
+                final Path wrapped = fsProvider.wrapPath(location);
-                );
-                final Path wrapped = new FilterPath(location, fs);
                 locations[i] = wrapped.resolve(randomAlphaOfLength(8));
                 Files.createDirectory(locations[i]);
                 locationsThrowingException.add(locations[i]);
@@ -246,8 +243,8 @@ public class IOUtilsTests extends ESTestCase {
 
     public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
         final Path path = createTempDir().toRealPath();
-        final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
+        final FilterFileSystemProvider fsProvider = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
-        final Path wrapped = new FilterPath(path, fs);
+        final Path wrapped = fsProvider.wrapPath(path);
         if (Constants.WINDOWS) {
             // no exception, we early return and do not even try to open the directory
             fsync(wrapped, true);

modules/lang-expression/licenses/lucene-expressions-9.1.0.jar.sha1

@@ -1 +0,0 @@
-2711abb758d101fc738c35a6867ee7559da5308b

modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+1755a2c955a53ae73096ec2b72b8982d676554b9

modules/legacy-geo/licenses/lucene-spatial-extras-9.1.0.jar.sha1

@@ -1 +0,0 @@
-94d7d107c399cd11d407b94fa62f5677fe86f63b

modules/legacy-geo/licenses/lucene-spatial-extras-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+563fc02413da6a9e9f6b5afdaf92df66253d16ce

plugins/analysis-icu/licenses/lucene-analysis-icu-9.1.0.jar.sha1

@@ -1 +0,0 @@
-e9b429da553560fa0c363ffc04c774f957c56e14

plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+1943e5a73bfed85067862388e931bb1702764547

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.1.0.jar.sha1

@@ -1 +0,0 @@
-b247f8a877237b4663e4ab7d86fae21c68a58ea5

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+aa21165b2cbc19ba0cc0b0b3ec675ca2577de80d

plugins/analysis-nori/licenses/lucene-analysis-nori-9.1.0.jar.sha1

@@ -1 +0,0 @@
-30e24b42fb0440911e702a531f4373bf397eb8c6

plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+57e5f7a8a10ae291e94db06d0262a757d08952dd

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.1.0.jar.sha1

@@ -1 +0,0 @@
-18a321d93836ea2856a5302d192e9dc99c647c6e

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+350720d6ebb8cb0b6296069a35f68d4081992b46

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.1.0.jar.sha1

@@ -1 +0,0 @@
-41c847f39a15bb8495be8c9d8a098974be15f74b

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+ec86d5ef7d47ae178dd34f52c0df39bd9b9c4344

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.1.0.jar.sha1

@@ -1 +0,0 @@
-ee7995231b181aa0a01f5aef8775562e269f5ef7

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+43ffa0c82753b644b66dfc931cae50ca0d351492

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.1.0.jar.sha1

@@ -1 +0,0 @@
-575c458431396baa7f01a546173807f27b12a087

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+8fab9f95101a853faaf86eef58f43f5517e5b679

server/licenses/lucene-analysis-common-9.1.0.jar.sha1

@@ -1 +0,0 @@
-240e3997fb139ff001e022124c89b686b5a8498d

server/licenses/lucene-analysis-common-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+38f13d4a20b4a9aec0307004ee18b87845c5e6e8

server/licenses/lucene-backward-codecs-9.1.0.jar.sha1

@@ -1 +0,0 @@
-de23bdacb09e8b39cbe876ff79c7a5b2ecc1faa6

server/licenses/lucene-backward-codecs-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+c9ac8533aa40378755539d3722b1ffed08b53b7c

server/licenses/lucene-core-9.1.0.jar.sha1

@@ -1 +0,0 @@
-0375603f1dacd8266526404faf0088a2ac8ec2ff

server/licenses/lucene-core-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+8be8ebe2a770686fede2c213fd81d4bb0b9f1590

server/licenses/lucene-grouping-9.1.0.jar.sha1

@@ -1 +0,0 @@
-703308505e62fa7dcb0bf64fdb6d95d335941bdc

server/licenses/lucene-grouping-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+686821eb6732242e3bedebd67788ded567b63470

server/licenses/lucene-highlighter-9.1.0.jar.sha1

@@ -1 +0,0 @@
-7f1925f6ef985000399a277ca17b8f67d3056838

server/licenses/lucene-highlighter-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+5d7032e3c10e79e907748478cb1662335ce42345

server/licenses/lucene-join-9.1.0.jar.sha1

@@ -1 +0,0 @@
-e7d39da8e623c99ee8da8bcc0185b2d908aca4b3

server/licenses/lucene-join-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+2a249b6127d5c05c67e16d3e1d3a542d5440629c

server/licenses/lucene-memory-9.1.0.jar.sha1

@@ -1 +0,0 @@
-209166fd48dae3261ccf26990fe600332b8fb373

server/licenses/lucene-memory-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+ad395a9f5a29b793bd431d790bb34bac9dce3b80

server/licenses/lucene-misc-9.1.0.jar.sha1

@@ -1 +0,0 @@
-905d93b6389060cf4b0cb464ffa8fa2db81b60e7

server/licenses/lucene-misc-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+08f39a2036656e70844d68227df79f7eb160d171

server/licenses/lucene-queries-9.1.0.jar.sha1

@@ -1 +0,0 @@
-c50fc971573910ea239ee6f275e9257b6b6bdd48

server/licenses/lucene-queries-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+beb6b98e38261d349da54d7cca367847925a2646

server/licenses/lucene-queryparser-9.1.0.jar.sha1

@@ -1 +0,0 @@
-383eb69b12f9d9c98c44237155f50c870c9a34b9

server/licenses/lucene-queryparser-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+5adb8b965bae11f0aa3ce7fa43558d6d1eecffeb

server/licenses/lucene-sandbox-9.1.0.jar.sha1

@@ -1 +0,0 @@
-0c728684e750a63f881998fbe27afd897f739762

server/licenses/lucene-sandbox-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+22acf5c5f112bed98fa4dfffbf09d5f17ec298e5

server/licenses/lucene-spatial3d-9.1.0.jar.sha1

@@ -1 +0,0 @@
-7717b300bc14dfa9eb4b7d5970d8e25a60010e64

server/licenses/lucene-spatial3d-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+f57269c7c419a5046792da61cfd1d532f9914390

server/licenses/lucene-suggest-9.1.0.jar.sha1

@@ -1 +0,0 @@
-957fca507eba94dbc3ef0d02377839be49bbe619

server/licenses/lucene-suggest-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+0ac869cf9e6a5045b2578a0f7fbf2acc88f3ceaf

server/src/main/java/org/elasticsearch/Version.java

@@ -109,7 +109,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_8_1_3 = new Version(8_01_03_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
     public static final Version V_8_2_0 = new Version(8_02_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_8_2_1 = new Version(8_02_01_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
     public static final Version CURRENT = V_8_3_0;
 
     private static final Map<Integer, Version> idToVersion;

server/src/main/java/org/elasticsearch/index/engine/LazySoftDeletesDirectoryReaderWrapper.java

@@ -11,6 +11,7 @@ package org.elasticsearch.index.engine;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.CodecReader;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FilterCodecReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
@@ -23,7 +24,6 @@ import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.lucene.Lucene;
@@ -227,7 +227,7 @@ public final class LazySoftDeletesDirectoryReaderWrapper extends FilterDirectory
         private Bits init() throws IOException {
             assert Thread.holdsLock(this);
 
-            DocIdSetIterator iterator = DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(field, reader);
+            DocIdSetIterator iterator = getDocValuesDocIdSetIterator(field, reader);
             assert iterator != null;
             Bits liveDocs = reader.getLiveDocs();
             final FixedBitSet bits;
@@ -274,4 +274,39 @@ public final class LazySoftDeletesDirectoryReaderWrapper extends FilterDirectory
             super(delegate);
         }
     }
+
+    /**
+     * Returns a {@link DocIdSetIterator} from the given field or null if the field doesn't exist in
+     * the reader or if the reader has no doc values for the field.
+     */
+    private static DocIdSetIterator getDocValuesDocIdSetIterator(String field, LeafReader reader) throws IOException {
+        FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+        final DocIdSetIterator iterator;
+        if (fieldInfo != null) {
+            switch (fieldInfo.getDocValuesType()) {
+                case NONE:
+                    iterator = null;
+                    break;
+                case NUMERIC:
+                    iterator = reader.getNumericDocValues(field);
+                    break;
+                case BINARY:
+                    iterator = reader.getBinaryDocValues(field);
+                    break;
+                case SORTED:
+                    iterator = reader.getSortedDocValues(field);
+                    break;
+                case SORTED_NUMERIC:
+                    iterator = reader.getSortedNumericDocValues(field);
+                    break;
+                case SORTED_SET:
+                    iterator = reader.getSortedSetDocValues(field);
+                    break;
+                default:
+                    throw new AssertionError();
+            }
+            return iterator;
+        }
+        return null;
+    }
 }

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java

@@ -59,6 +59,11 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
         }
     }
 
+    @Override
+    public long docValueCount() {
+        return values.docValueCount();
+    }
+
     @Override
     public BytesRef lookupOrd(long globalOrd) throws IOException {
         final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java

@@ -189,6 +189,11 @@ public class MultiOrdinals extends Ordinals {
             }
         }
 
+        @Override
+        public long docValueCount() {
+            return currentEndOffset - currentOffset;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return values.lookupOrd(ord);

server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java

@@ -459,6 +459,11 @@ class DocumentLeafReader extends LeafReader {
                 return i;
             }
 
+            @Override
+            public long docValueCount() {
+                return values.size();
+            }
+
             @Override
             public BytesRef lookupOrd(long ord) {
                 return values.get((int) ord);

server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java

@@ -209,6 +209,11 @@ public class KeyedFlattenedLeafFieldData implements LeafOrdinalsFieldData {
             }
         }
 
+        @Override
+        public long docValueCount() {
+            return delegate.docValueCount();
+        }
+
         @Override
         public boolean advanceExact(int target) throws IOException {
             if (delegate.advanceExact(target)) {

server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java

@@ -275,6 +275,11 @@ public enum MissingValues {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public boolean advanceExact(int doc) throws IOException {
                 hasOrds = values.advanceExact(doc);
@@ -309,6 +314,11 @@ public enum MissingValues {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public long getValueCount() {
                 return 1 + values.getValueCount();

server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java

@@ -199,6 +199,11 @@ public class KeyedFlattenedLeafFieldDataTests extends ESTestCase {
             return documentOrds[index++];
         }
 
+        @Override
+        public long docValueCount() {
+            return documentOrds.length;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return allTerms[(int) ord];

server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java

@@ -723,6 +723,11 @@ public class MultiValueModeTests extends ESTestCase {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return array[doc].length;
+            }
+
             @Override
             public boolean advanceExact(int docID) {
                 this.doc = docID;

server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java

@@ -876,7 +876,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas
                                             .entry(
                                                 "filters",
                                                 matchesList().item(
-                                                    matchesMap().entry("query", "DocValuesFieldExistsQuery [field=f]")
+                                                    matchesMap().entry("query", "FieldExistsQuery [field=f]")
                                                         .entry("specialized_for", "docvalues_field_exists")
                                                         .entry("results_from_metadata", greaterThan(0))
                                                 )

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java

@@ -47,6 +47,11 @@ public class BinaryRangeAggregatorTests extends ESTestCase {
             return ords[i++];
         }
 
+        @Override
+        public long docValueCount() {
+            return ords.length;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return terms[(int) ord];

server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java

@@ -66,6 +66,11 @@ public class IncludeExcludeTests extends ESTestCase {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return 1;
+            }
+
             @Override
             public BytesRef lookupOrd(long ord) {
                 assertEquals(0, ord);

server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java

@@ -131,6 +131,11 @@ public class MissingValuesTests extends ESTestCase {
                     return NO_MORE_ORDS;
                 }
             }
+
+            @Override
+            public long docValueCount() {
+                return ords[doc].length;
+            }
         };
 
         final BytesRef existingMissing = RandomPicks.randomFrom(random(), values);
@@ -228,6 +233,11 @@ public class MissingValuesTests extends ESTestCase {
                 throw new UnsupportedOperationException();
             }
 
+            @Override
+            public long docValueCount() {
+                throw new UnsupportedOperationException();
+            }
+
             @Override
             public BytesRef lookupOrd(long ord) throws IOException {
                 return values[Math.toIntExact(ord)];

x-pack/plugin/ml/licenses/lucene-analysis-icu-9.1.0.jar.sha1

@@ -1 +0,0 @@
-e9b429da553560fa0c363ffc04c774f957c56e14

x-pack/plugin/ml/licenses/lucene-analysis-icu-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+1943e5a73bfed85067862388e931bb1702764547

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacySortedSetDocValuesWrapper.java

@@ -98,6 +98,11 @@ public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues {
         return result;
     }
 
+    @Override
+    public long docValueCount() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public BytesRef lookupOrd(long ord) {
         return values.lookupOrd((int) ord);

x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java

@@ -602,7 +602,7 @@ public class WildcardFieldMapperTests extends MapperTestCase {
         String suboptimalTests[][] = {
             // TODO short wildcards like a* OR b* aren't great so we just drop them.
             // Ideally we would attach to successors to create (acd OR bcd)
-            { "[ab]cd", "+(+cc_ +c__) +*:*" } };
+            { "[ab]cd", "+cc_ +c__" } };
         for (String[] test : suboptimalTests) {
             String regex = test[0];
             String expectedAccelerationQueryString = test[1].replaceAll("_", "" + WildcardFieldMapper.TOKEN_START_OR_END_CHAR);