mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
Upgrade to Lucene 9.2 snapshot efa5d6f4d43 (#86227)
Notable changes include: count implementations for MultiRangeQuery and IndexSortedNumericDocValuesRangeQuery, which may speed up certain aggregations more efficient decoding of docids in BKD reader
This commit is contained in:
Alan Woodward
GitHub
parent
e590e900a4
commit
4d076eee20
65 changed files with 140 additions and 38 deletions
|
|
@ -1,5 +1,5 @@
|
|||
elasticsearch = 8.3.0
|
||||
lucene = 9.1.0
|
||||
lucene = 9.2.0-snapshot-efa5d6f4d43
|
||||
|
||||
bundled_jdk_vendor = adoptium
|
||||
bundled_jdk = 18+36
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
|
||||
|
||||
:lucene_version: 9.1.0
|
||||
:lucene_version_path: 9_1_0
|
||||
:lucene_version: 9.2.0
|
||||
:lucene_version_path: 9_2_0
|
||||
:jdk: 11.0.2
|
||||
:jdk_major: 11
|
||||
:build_type: tar
|
||||
|
|
|
|||
5
docs/changelog/86227.yaml
Normal file
5
docs/changelog/86227.yaml
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
pr: 86227
|
||||
summary: Upgrade to Lucene 9.2 snapshot
|
||||
area: Search
|
||||
type: upgrade
|
||||
issues: []
|
||||
|
|
@ -26,7 +26,6 @@ import org.elasticsearch.test.ESTestCase;
|
|||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
|
|
@ -158,10 +157,8 @@ public class IOUtilsTests extends ESTestCase {
|
|||
for (int i = 0; i < numberOfLocations; i++) {
|
||||
if (exception && randomBoolean()) {
|
||||
final Path location = createTempDir();
|
||||
final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
|
||||
URI.create("file:///")
|
||||
);
|
||||
final Path wrapped = new FilterPath(location, fs);
|
||||
final FilterFileSystemProvider fsProvider = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
|
||||
final Path wrapped = fsProvider.wrapPath(location);
|
||||
locations[i] = wrapped.resolve(randomAlphaOfLength(8));
|
||||
Files.createDirectory(locations[i]);
|
||||
locationsThrowingException.add(locations[i]);
|
||||
|
|
@ -246,8 +243,8 @@ public class IOUtilsTests extends ESTestCase {
|
|||
|
||||
public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
|
||||
final Path path = createTempDir().toRealPath();
|
||||
final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
|
||||
final Path wrapped = new FilterPath(path, fs);
|
||||
final FilterFileSystemProvider fsProvider = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
|
||||
final Path wrapped = fsProvider.wrapPath(path);
|
||||
if (Constants.WINDOWS) {
|
||||
// no exception, we early return and do not even try to open the directory
|
||||
fsync(wrapped, true);
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
2711abb758d101fc738c35a6867ee7559da5308b
|
||||
|
|
@ -0,0 +1 @@
|
|||
1755a2c955a53ae73096ec2b72b8982d676554b9
|
||||
|
|
@ -1 +0,0 @@
|
|||
94d7d107c399cd11d407b94fa62f5677fe86f63b
|
||||
|
|
@ -0,0 +1 @@
|
|||
563fc02413da6a9e9f6b5afdaf92df66253d16ce
|
||||
|
|
@ -1 +0,0 @@
|
|||
e9b429da553560fa0c363ffc04c774f957c56e14
|
||||
|
|
@ -0,0 +1 @@
|
|||
1943e5a73bfed85067862388e931bb1702764547
|
||||
|
|
@ -1 +0,0 @@
|
|||
b247f8a877237b4663e4ab7d86fae21c68a58ea5
|
||||
|
|
@ -0,0 +1 @@
|
|||
aa21165b2cbc19ba0cc0b0b3ec675ca2577de80d
|
||||
|
|
@ -1 +0,0 @@
|
|||
30e24b42fb0440911e702a531f4373bf397eb8c6
|
||||
|
|
@ -0,0 +1 @@
|
|||
57e5f7a8a10ae291e94db06d0262a757d08952dd
|
||||
|
|
@ -1 +0,0 @@
|
|||
18a321d93836ea2856a5302d192e9dc99c647c6e
|
||||
|
|
@ -0,0 +1 @@
|
|||
350720d6ebb8cb0b6296069a35f68d4081992b46
|
||||
|
|
@ -1 +0,0 @@
|
|||
41c847f39a15bb8495be8c9d8a098974be15f74b
|
||||
|
|
@ -0,0 +1 @@
|
|||
ec86d5ef7d47ae178dd34f52c0df39bd9b9c4344
|
||||
|
|
@ -1 +0,0 @@
|
|||
ee7995231b181aa0a01f5aef8775562e269f5ef7
|
||||
|
|
@ -0,0 +1 @@
|
|||
43ffa0c82753b644b66dfc931cae50ca0d351492
|
||||
|
|
@ -1 +0,0 @@
|
|||
575c458431396baa7f01a546173807f27b12a087
|
||||
|
|
@ -0,0 +1 @@
|
|||
8fab9f95101a853faaf86eef58f43f5517e5b679
|
||||
|
|
@ -1 +0,0 @@
|
|||
240e3997fb139ff001e022124c89b686b5a8498d
|
||||
|
|
@ -0,0 +1 @@
|
|||
38f13d4a20b4a9aec0307004ee18b87845c5e6e8
|
||||
|
|
@ -1 +0,0 @@
|
|||
de23bdacb09e8b39cbe876ff79c7a5b2ecc1faa6
|
||||
|
|
@ -0,0 +1 @@
|
|||
c9ac8533aa40378755539d3722b1ffed08b53b7c
|
||||
|
|
@ -1 +0,0 @@
|
|||
0375603f1dacd8266526404faf0088a2ac8ec2ff
|
||||
|
|
@ -0,0 +1 @@
|
|||
8be8ebe2a770686fede2c213fd81d4bb0b9f1590
|
||||
|
|
@ -1 +0,0 @@
|
|||
703308505e62fa7dcb0bf64fdb6d95d335941bdc
|
||||
|
|
@ -0,0 +1 @@
|
|||
686821eb6732242e3bedebd67788ded567b63470
|
||||
|
|
@ -1 +0,0 @@
|
|||
7f1925f6ef985000399a277ca17b8f67d3056838
|
||||
|
|
@ -0,0 +1 @@
|
|||
5d7032e3c10e79e907748478cb1662335ce42345
|
||||
|
|
@ -1 +0,0 @@
|
|||
e7d39da8e623c99ee8da8bcc0185b2d908aca4b3
|
||||
|
|
@ -0,0 +1 @@
|
|||
2a249b6127d5c05c67e16d3e1d3a542d5440629c
|
||||
|
|
@ -1 +0,0 @@
|
|||
209166fd48dae3261ccf26990fe600332b8fb373
|
||||
|
|
@ -0,0 +1 @@
|
|||
ad395a9f5a29b793bd431d790bb34bac9dce3b80
|
||||
|
|
@ -1 +0,0 @@
|
|||
905d93b6389060cf4b0cb464ffa8fa2db81b60e7
|
||||
|
|
@ -0,0 +1 @@
|
|||
08f39a2036656e70844d68227df79f7eb160d171
|
||||
|
|
@ -1 +0,0 @@
|
|||
c50fc971573910ea239ee6f275e9257b6b6bdd48
|
||||
|
|
@ -0,0 +1 @@
|
|||
beb6b98e38261d349da54d7cca367847925a2646
|
||||
|
|
@ -1 +0,0 @@
|
|||
383eb69b12f9d9c98c44237155f50c870c9a34b9
|
||||
|
|
@ -0,0 +1 @@
|
|||
5adb8b965bae11f0aa3ce7fa43558d6d1eecffeb
|
||||
|
|
@ -1 +0,0 @@
|
|||
0c728684e750a63f881998fbe27afd897f739762
|
||||
|
|
@ -0,0 +1 @@
|
|||
22acf5c5f112bed98fa4dfffbf09d5f17ec298e5
|
||||
|
|
@ -1 +0,0 @@
|
|||
7717b300bc14dfa9eb4b7d5970d8e25a60010e64
|
||||
|
|
@ -0,0 +1 @@
|
|||
f57269c7c419a5046792da61cfd1d532f9914390
|
||||
|
|
@ -1 +0,0 @@
|
|||
957fca507eba94dbc3ef0d02377839be49bbe619
|
||||
|
|
@ -0,0 +1 @@
|
|||
0ac869cf9e6a5045b2578a0f7fbf2acc88f3ceaf
|
||||
|
|
@ -109,7 +109,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final Version V_8_1_3 = new Version(8_01_03_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
|
||||
public static final Version V_8_2_0 = new Version(8_02_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
|
||||
public static final Version V_8_2_1 = new Version(8_02_01_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
|
||||
public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
|
||||
public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
|
||||
public static final Version CURRENT = V_8_3_0;
|
||||
|
||||
private static final Map<Integer, Version> idToVersion;
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ package org.elasticsearch.index.engine;
|
|||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.CodecReader;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FilterCodecReader;
|
||||
import org.apache.lucene.index.FilterDirectoryReader;
|
||||
import org.apache.lucene.index.FilterLeafReader;
|
||||
|
|
@ -23,7 +24,6 @@ import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
|
|||
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.DocValuesFieldExistsQuery;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.elasticsearch.common.lucene.Lucene;
|
||||
|
|
@ -227,7 +227,7 @@ public final class LazySoftDeletesDirectoryReaderWrapper extends FilterDirectory
|
|||
private Bits init() throws IOException {
|
||||
assert Thread.holdsLock(this);
|
||||
|
||||
DocIdSetIterator iterator = DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(field, reader);
|
||||
DocIdSetIterator iterator = getDocValuesDocIdSetIterator(field, reader);
|
||||
assert iterator != null;
|
||||
Bits liveDocs = reader.getLiveDocs();
|
||||
final FixedBitSet bits;
|
||||
|
|
@ -274,4 +274,39 @@ public final class LazySoftDeletesDirectoryReaderWrapper extends FilterDirectory
|
|||
super(delegate);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link DocIdSetIterator} from the given field or null if the field doesn't exist in
|
||||
* the reader or if the reader has no doc values for the field.
|
||||
*/
|
||||
private static DocIdSetIterator getDocValuesDocIdSetIterator(String field, LeafReader reader) throws IOException {
|
||||
FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
|
||||
final DocIdSetIterator iterator;
|
||||
if (fieldInfo != null) {
|
||||
switch (fieldInfo.getDocValuesType()) {
|
||||
case NONE:
|
||||
iterator = null;
|
||||
break;
|
||||
case NUMERIC:
|
||||
iterator = reader.getNumericDocValues(field);
|
||||
break;
|
||||
case BINARY:
|
||||
iterator = reader.getBinaryDocValues(field);
|
||||
break;
|
||||
case SORTED:
|
||||
iterator = reader.getSortedDocValues(field);
|
||||
break;
|
||||
case SORTED_NUMERIC:
|
||||
iterator = reader.getSortedNumericDocValues(field);
|
||||
break;
|
||||
case SORTED_SET:
|
||||
iterator = reader.getSortedSetDocValues(field);
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError();
|
||||
}
|
||||
return iterator;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,11 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long globalOrd) throws IOException {
|
||||
final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);
|
||||
|
|
|
|||
|
|
@ -189,6 +189,11 @@ public class MultiOrdinals extends Ordinals {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return currentEndOffset - currentOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
return values.lookupOrd(ord);
|
||||
|
|
|
|||
|
|
@ -459,6 +459,11 @@ class DocumentLeafReader extends LeafReader {
|
|||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return values.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
return values.get((int) ord);
|
||||
|
|
|
|||
|
|
@ -209,6 +209,11 @@ public class KeyedFlattenedLeafFieldData implements LeafOrdinalsFieldData {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return delegate.docValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean advanceExact(int target) throws IOException {
|
||||
if (delegate.advanceExact(target)) {
|
||||
|
|
|
|||
|
|
@ -275,6 +275,11 @@ public enum MissingValues {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean advanceExact(int doc) throws IOException {
|
||||
hasOrds = values.advanceExact(doc);
|
||||
|
|
@ -309,6 +314,11 @@ public enum MissingValues {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return values.docValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueCount() {
|
||||
return 1 + values.getValueCount();
|
||||
|
|
|
|||
|
|
@ -199,6 +199,11 @@ public class KeyedFlattenedLeafFieldDataTests extends ESTestCase {
|
|||
return documentOrds[index++];
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return documentOrds.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
return allTerms[(int) ord];
|
||||
|
|
|
|||
|
|
@ -723,6 +723,11 @@ public class MultiValueModeTests extends ESTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return array[doc].length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean advanceExact(int docID) {
|
||||
this.doc = docID;
|
||||
|
|
|
|||
|
|
@ -876,7 +876,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas
|
|||
.entry(
|
||||
"filters",
|
||||
matchesList().item(
|
||||
matchesMap().entry("query", "DocValuesFieldExistsQuery [field=f]")
|
||||
matchesMap().entry("query", "FieldExistsQuery [field=f]")
|
||||
.entry("specialized_for", "docvalues_field_exists")
|
||||
.entry("results_from_metadata", greaterThan(0))
|
||||
)
|
||||
|
|
|
|||
|
|
@ -47,6 +47,11 @@ public class BinaryRangeAggregatorTests extends ESTestCase {
|
|||
return ords[i++];
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return ords.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
return terms[(int) ord];
|
||||
|
|
|
|||
|
|
@ -66,6 +66,11 @@ public class IncludeExcludeTests extends ESTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
assertEquals(0, ord);
|
||||
|
|
|
|||
|
|
@ -131,6 +131,11 @@ public class MissingValuesTests extends ESTestCase {
|
|||
return NO_MORE_ORDS;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
return ords[doc].length;
|
||||
}
|
||||
};
|
||||
|
||||
final BytesRef existingMissing = RandomPicks.randomFrom(random(), values);
|
||||
|
|
@ -228,6 +233,11 @@ public class MissingValuesTests extends ESTestCase {
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) throws IOException {
|
||||
return values[Math.toIntExact(ord)];
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
e9b429da553560fa0c363ffc04c774f957c56e14
|
||||
|
|
@ -0,0 +1 @@
|
|||
1943e5a73bfed85067862388e931bb1702764547
|
||||
|
|
@ -98,6 +98,11 @@ public final class LegacySortedSetDocValuesWrapper extends SortedSetDocValues {
|
|||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long docValueCount() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef lookupOrd(long ord) {
|
||||
return values.lookupOrd((int) ord);
|
||||
|
|
|
|||
|
|
@ -602,7 +602,7 @@ public class WildcardFieldMapperTests extends MapperTestCase {
|
|||
String suboptimalTests[][] = {
|
||||
// TODO short wildcards like a* OR b* aren't great so we just drop them.
|
||||
// Ideally we would attach to successors to create (acd OR bcd)
|
||||
{ "[ab]cd", "+(+cc_ +c__) +*:*" } };
|
||||
{ "[ab]cd", "+cc_ +c__" } };
|
||||
for (String[] test : suboptimalTests) {
|
||||
String regex = test[0];
|
||||
String expectedAccelerationQueryString = test[1].replaceAll("_", "" + WildcardFieldMapper.TOKEN_START_OR_END_CHAR);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue