mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
Upgrade to lucene snapshot 978eef5459c (#86852)
Final (hopefully!) snapshot before the 9.2.0 release * Update test to expect persian tokenfilter - will be exposed later * Fix KnnVectorQueryBuilderTests::doAssertLuceneQuery Co-authored-by: Mayya Sharipova <mayya.sharipova@elastic.co>
This commit is contained in:
parent
b7f49345e1
commit
0418e8a9d8
59 changed files with 74 additions and 61 deletions
|
@ -9,7 +9,6 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
|
||||||
com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
|
com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
|
||||||
|
|
||||||
org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
|
org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
|
||||||
org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
|
|
||||||
org.junit.Ignore @ Use AwaitsFix instead
|
org.junit.Ignore @ Use AwaitsFix instead
|
||||||
org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
|
org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
|
||||||
com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
|
com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
elasticsearch = 8.3.0
|
elasticsearch = 8.3.0
|
||||||
lucene = 9.2.0-snapshot-efa5d6f4d43
|
lucene = 9.2.0-snapshot-978eef5459c
|
||||||
|
|
||||||
bundled_jdk_vendor = openjdk
|
bundled_jdk_vendor = openjdk
|
||||||
bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
|
bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
|
||||||
|
|
5
docs/changelog/86852.yaml
Normal file
5
docs/changelog/86852.yaml
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
pr: 86852
|
||||||
|
summary: Upgrade to lucene snapshot 978eef5459c
|
||||||
|
area: Search
|
||||||
|
type: upgrade
|
||||||
|
issues: []
|
|
@ -0,0 +1 @@
|
||||||
|
f19903613908590bb237683064c82a4834e64ba5
|
|
@ -1 +0,0 @@
|
||||||
1755a2c955a53ae73096ec2b72b8982d676554b9
|
|
|
@ -0,0 +1 @@
|
||||||
|
050d713d9dc35226d040c580d48a157c21953e6f
|
|
@ -1 +0,0 @@
|
||||||
563fc02413da6a9e9f6b5afdaf92df66253d16ce
|
|
|
@ -0,0 +1 @@
|
||||||
|
071d4e394dc5b9135e815054710466673092c725
|
|
@ -1 +0,0 @@
|
||||||
1943e5a73bfed85067862388e931bb1702764547
|
|
|
@ -0,0 +1 @@
|
||||||
|
3b3084be8519720c80c910062219370934673169
|
|
@ -1 +0,0 @@
|
||||||
aa21165b2cbc19ba0cc0b0b3ec675ca2577de80d
|
|
|
@ -0,0 +1 @@
|
||||||
|
d46b121b3da6d87e37470e462b3acf000cc7e8fc
|
|
@ -1 +0,0 @@
|
||||||
57e5f7a8a10ae291e94db06d0262a757d08952dd
|
|
|
@ -0,0 +1 @@
|
||||||
|
b649f34cb35d2007f9460ab808493a0908698e18
|
|
@ -1 +0,0 @@
|
||||||
350720d6ebb8cb0b6296069a35f68d4081992b46
|
|
|
@ -0,0 +1 @@
|
||||||
|
80cf1edcd52219348441fde9b6535577ff6b5055
|
|
@ -1 +0,0 @@
|
||||||
ec86d5ef7d47ae178dd34f52c0df39bd9b9c4344
|
|
|
@ -0,0 +1 @@
|
||||||
|
34549e8867baa25bbc5c88160bbe1490520496ff
|
|
@ -1 +0,0 @@
|
||||||
43ffa0c82753b644b66dfc931cae50ca0d351492
|
|
|
@ -0,0 +1 @@
|
||||||
|
7dac3ff3c24683c8fc057a930ed5cd346b33b587
|
|
@ -1 +0,0 @@
|
||||||
8fab9f95101a853faaf86eef58f43f5517e5b679
|
|
|
@ -0,0 +1 @@
|
||||||
|
54a32bd0b8410fa1793b5fb9aa37070a50d34406
|
|
@ -1 +0,0 @@
|
||||||
38f13d4a20b4a9aec0307004ee18b87845c5e6e8
|
|
|
@ -0,0 +1 @@
|
||||||
|
1f9e1809056e553590f82d2fab240516dba163a9
|
|
@ -1 +0,0 @@
|
||||||
c9ac8533aa40378755539d3722b1ffed08b53b7c
|
|
|
@ -0,0 +1 @@
|
||||||
|
2a5725388d220bb62fbfe24c0bd42e00142dfb4a
|
|
@ -1 +0,0 @@
|
||||||
8be8ebe2a770686fede2c213fd81d4bb0b9f1590
|
|
|
@ -0,0 +1 @@
|
||||||
|
a059178643306db25088c02bce8c4b6ff51c157a
|
|
@ -1 +0,0 @@
|
||||||
686821eb6732242e3bedebd67788ded567b63470
|
|
|
@ -0,0 +1 @@
|
||||||
|
38c3310e99d31e621b431fe48bda5489055ee5c2
|
|
@ -1 +0,0 @@
|
||||||
5d7032e3c10e79e907748478cb1662335ce42345
|
|
|
@ -0,0 +1 @@
|
||||||
|
95fe8d6803fb8caf7097b50a6e15fd823600aa49
|
|
@ -1 +0,0 @@
|
||||||
2a249b6127d5c05c67e16d3e1d3a542d5440629c
|
|
|
@ -0,0 +1 @@
|
||||||
|
ed27757832d5188a0a7fed5646cb670cffc3e526
|
|
@ -1 +0,0 @@
|
||||||
ad395a9f5a29b793bd431d790bb34bac9dce3b80
|
|
|
@ -0,0 +1 @@
|
||||||
|
b851d0d5123d6f61ccbf0cfd9274b174baf6f670
|
|
@ -1 +0,0 @@
|
||||||
08f39a2036656e70844d68227df79f7eb160d171
|
|
|
@ -0,0 +1 @@
|
||||||
|
5eb5de43f515f29d9c15e3dead7278d4edb60890
|
|
@ -1 +0,0 @@
|
||||||
beb6b98e38261d349da54d7cca367847925a2646
|
|
|
@ -0,0 +1 @@
|
||||||
|
e7ade1e53845c76917a523dde6038c82e4b50e26
|
|
@ -1 +0,0 @@
|
||||||
5adb8b965bae11f0aa3ce7fa43558d6d1eecffeb
|
|
|
@ -0,0 +1 @@
|
||||||
|
8fc04d2909ca3649d34a9b4e75c02370f5414bdf
|
|
@ -1 +0,0 @@
|
||||||
22acf5c5f112bed98fa4dfffbf09d5f17ec298e5
|
|
|
@ -0,0 +1 @@
|
||||||
|
36e70112545743795e81c507ccf9d2214621f9ea
|
|
@ -1 +0,0 @@
|
||||||
f57269c7c419a5046792da61cfd1d532f9914390
|
|
|
@ -0,0 +1 @@
|
||||||
|
4b46de9ca199c319a9807c63a728776f96dfc22f
|
|
@ -1 +0,0 @@
|
||||||
0ac869cf9e6a5045b2578a0f7fbf2acc88f3ceaf
|
|
|
@ -86,7 +86,7 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public class Lucene {
|
public class Lucene {
|
||||||
public static final String LATEST_CODEC = "Lucene91";
|
public static final String LATEST_CODEC = "Lucene92";
|
||||||
|
|
||||||
public static final String SOFT_DELETES_FIELD = "__soft_deletes";
|
public static final String SOFT_DELETES_FIELD = "__soft_deletes";
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,7 @@
|
||||||
package org.elasticsearch.index.codec;
|
package org.elasticsearch.index.codec;
|
||||||
|
|
||||||
import org.apache.lucene.codecs.Codec;
|
import org.apache.lucene.codecs.Codec;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
|
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||||
import org.elasticsearch.core.Nullable;
|
import org.elasticsearch.core.Nullable;
|
||||||
import org.elasticsearch.index.mapper.MapperService;
|
import org.elasticsearch.index.mapper.MapperService;
|
||||||
|
|
||||||
|
@ -34,11 +34,11 @@ public class CodecService {
|
||||||
public CodecService(@Nullable MapperService mapperService) {
|
public CodecService(@Nullable MapperService mapperService) {
|
||||||
final var codecs = new HashMap<String, Codec>();
|
final var codecs = new HashMap<String, Codec>();
|
||||||
if (mapperService == null) {
|
if (mapperService == null) {
|
||||||
codecs.put(DEFAULT_CODEC, new Lucene91Codec());
|
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
|
||||||
codecs.put(BEST_COMPRESSION_CODEC, new Lucene91Codec(Lucene91Codec.Mode.BEST_COMPRESSION));
|
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Lucene92Codec.Mode.BEST_COMPRESSION));
|
||||||
} else {
|
} else {
|
||||||
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene91Codec.Mode.BEST_SPEED, mapperService));
|
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_SPEED, mapperService));
|
||||||
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene91Codec.Mode.BEST_COMPRESSION, mapperService));
|
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_COMPRESSION, mapperService));
|
||||||
}
|
}
|
||||||
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
|
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
|
||||||
for (String codec : Codec.availableCodecs()) {
|
for (String codec : Codec.availableCodecs()) {
|
||||||
|
|
|
@ -13,7 +13,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
|
||||||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||||
import org.apache.lucene.codecs.PostingsFormat;
|
import org.apache.lucene.codecs.PostingsFormat;
|
||||||
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
|
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||||
import org.elasticsearch.common.lucene.Lucene;
|
import org.elasticsearch.common.lucene.Lucene;
|
||||||
import org.elasticsearch.index.mapper.MapperService;
|
import org.elasticsearch.index.mapper.MapperService;
|
||||||
|
|
||||||
|
@ -25,7 +25,7 @@ import org.elasticsearch.index.mapper.MapperService;
|
||||||
* per index in real time via the mapping API. If no specific postings format or vector format is
|
* per index in real time via the mapping API. If no specific postings format or vector format is
|
||||||
* configured for a specific field the default postings or vector format is used.
|
* configured for a specific field the default postings or vector format is used.
|
||||||
*/
|
*/
|
||||||
public class PerFieldMapperCodec extends Lucene91Codec {
|
public class PerFieldMapperCodec extends Lucene92Codec {
|
||||||
private final MapperService mapperService;
|
private final MapperService mapperService;
|
||||||
|
|
||||||
private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
|
private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
|
||||||
|
|
|
@ -12,7 +12,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
|
||||||
import org.apache.lucene.codecs.PostingsFormat;
|
import org.apache.lucene.codecs.PostingsFormat;
|
||||||
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
|
||||||
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
|
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
|
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||||
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
|
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
|
||||||
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
|
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
|
||||||
import org.apache.lucene.document.BinaryDocValuesField;
|
import org.apache.lucene.document.BinaryDocValuesField;
|
||||||
|
@ -266,7 +266,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
||||||
public void testCompletionField() throws Exception {
|
public void testCompletionField() throws Exception {
|
||||||
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
||||||
.setUseCompoundFile(false)
|
.setUseCompoundFile(false)
|
||||||
.setCodec(new Lucene91Codec(Lucene91Codec.Mode.BEST_SPEED) {
|
.setCodec(new Lucene92Codec(Lucene92Codec.Mode.BEST_SPEED) {
|
||||||
@Override
|
@Override
|
||||||
public PostingsFormat getPostingsFormatForField(String field) {
|
public PostingsFormat getPostingsFormatForField(String field) {
|
||||||
if (field.startsWith("suggest_")) {
|
if (field.startsWith("suggest_")) {
|
||||||
|
@ -353,25 +353,25 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
||||||
enum CodecMode {
|
enum CodecMode {
|
||||||
BEST_SPEED {
|
BEST_SPEED {
|
||||||
@Override
|
@Override
|
||||||
Lucene91Codec.Mode mode() {
|
Lucene92Codec.Mode mode() {
|
||||||
return Lucene91Codec.Mode.BEST_SPEED;
|
return Lucene92Codec.Mode.BEST_SPEED;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
BEST_COMPRESSION {
|
BEST_COMPRESSION {
|
||||||
@Override
|
@Override
|
||||||
Lucene91Codec.Mode mode() {
|
Lucene92Codec.Mode mode() {
|
||||||
return Lucene91Codec.Mode.BEST_COMPRESSION;
|
return Lucene92Codec.Mode.BEST_COMPRESSION;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
abstract Lucene91Codec.Mode mode();
|
abstract Lucene92Codec.Mode mode();
|
||||||
}
|
}
|
||||||
|
|
||||||
static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
|
static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
|
||||||
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
|
||||||
.setUseCompoundFile(randomBoolean())
|
.setUseCompoundFile(randomBoolean())
|
||||||
.setCodec(new Lucene91Codec(codecMode.mode()));
|
.setCodec(new Lucene92Codec(codecMode.mode()));
|
||||||
try (IndexWriter writer = new IndexWriter(directory, config)) {
|
try (IndexWriter writer = new IndexWriter(directory, config)) {
|
||||||
for (int i = 0; i < numDocs; i++) {
|
for (int i = 0; i < numDocs; i++) {
|
||||||
final Document doc = new Document();
|
final Document doc = new Document();
|
||||||
|
@ -549,7 +549,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
|
||||||
try (DirectoryReader reader = DirectoryReader.open(source)) {
|
try (DirectoryReader reader = DirectoryReader.open(source)) {
|
||||||
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
|
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
|
||||||
.setUseCompoundFile(randomBoolean())
|
.setUseCompoundFile(randomBoolean())
|
||||||
.setCodec(new Lucene91Codec(mode.mode()) {
|
.setCodec(new Lucene92Codec(mode.mode()) {
|
||||||
@Override
|
@Override
|
||||||
public PostingsFormat getPostingsFormatForField(String field) {
|
public PostingsFormat getPostingsFormatForField(String field) {
|
||||||
return new Lucene90PostingsFormat();
|
return new Lucene90PostingsFormat();
|
||||||
|
|
|
@ -10,7 +10,7 @@ package org.elasticsearch.index.codec;
|
||||||
|
|
||||||
import org.apache.lucene.codecs.Codec;
|
import org.apache.lucene.codecs.Codec;
|
||||||
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
|
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
|
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.index.DirectoryReader;
|
import org.apache.lucene.index.DirectoryReader;
|
||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
|
@ -41,21 +41,21 @@ public class CodecTests extends ESTestCase {
|
||||||
public void testResolveDefaultCodecs() throws Exception {
|
public void testResolveDefaultCodecs() throws Exception {
|
||||||
CodecService codecService = createCodecService();
|
CodecService codecService = createCodecService();
|
||||||
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
|
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
|
||||||
assertThat(codecService.codec("default"), instanceOf(Lucene91Codec.class));
|
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDefault() throws Exception {
|
public void testDefault() throws Exception {
|
||||||
Codec codec = createCodecService().codec("default");
|
Codec codec = createCodecService().codec("default");
|
||||||
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_SPEED, codec);
|
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBestCompression() throws Exception {
|
public void testBestCompression() throws Exception {
|
||||||
Codec codec = createCodecService().codec("best_compression");
|
Codec codec = createCodecService().codec("best_compression");
|
||||||
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_COMPRESSION, codec);
|
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
|
||||||
}
|
}
|
||||||
|
|
||||||
// write some docs with it, inspect .si to see this was the used compression
|
// write some docs with it, inspect .si to see this was the used compression
|
||||||
private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Codec actual) throws Exception {
|
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
|
||||||
Directory dir = newDirectory();
|
Directory dir = newDirectory();
|
||||||
IndexWriterConfig iwc = newIndexWriterConfig(null);
|
IndexWriterConfig iwc = newIndexWriterConfig(null);
|
||||||
iwc.setCodec(actual);
|
iwc.setCodec(actual);
|
||||||
|
@ -67,7 +67,7 @@ public class CodecTests extends ESTestCase {
|
||||||
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
|
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
|
||||||
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
|
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
|
||||||
assertNotNull(v);
|
assertNotNull(v);
|
||||||
assertEquals(expected, Lucene91Codec.Mode.valueOf(v));
|
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
|
||||||
ir.close();
|
ir.close();
|
||||||
dir.close();
|
dir.close();
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,7 @@
|
||||||
package org.elasticsearch.index.engine;
|
package org.elasticsearch.index.engine;
|
||||||
|
|
||||||
import org.apache.lucene.codecs.PostingsFormat;
|
import org.apache.lucene.codecs.PostingsFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
|
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.index.DirectoryReader;
|
import org.apache.lucene.index.DirectoryReader;
|
||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
|
@ -45,7 +45,7 @@ public class CompletionStatsCacheTests extends ESTestCase {
|
||||||
public void testCompletionStatsCache() throws IOException, InterruptedException {
|
public void testCompletionStatsCache() throws IOException, InterruptedException {
|
||||||
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
|
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
|
||||||
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
|
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
|
||||||
indexWriterConfig.setCodec(new Lucene91Codec() {
|
indexWriterConfig.setCodec(new Lucene92Codec() {
|
||||||
@Override
|
@Override
|
||||||
public PostingsFormat getPostingsFormatForField(String field) {
|
public PostingsFormat getPostingsFormatForField(String field) {
|
||||||
return postingsFormat; // all fields are suggest fields
|
return postingsFormat; // all fields are suggest fields
|
||||||
|
|
|
@ -194,7 +194,8 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
|
||||||
// LUCENE-10248
|
// LUCENE-10248
|
||||||
entry("spanishpluralstem", Void.class),
|
entry("spanishpluralstem", Void.class),
|
||||||
// LUCENE-10352
|
// LUCENE-10352
|
||||||
entry("daitchmokotoffsoundex", Void.class)
|
entry("daitchmokotoffsoundex", Void.class),
|
||||||
|
entry("persianstem", Void.class)
|
||||||
);
|
);
|
||||||
|
|
||||||
static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(
|
static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
071d4e394dc5b9135e815054710466673092c725
|
|
@ -1 +0,0 @@
|
||||||
1943e5a73bfed85067862388e931bb1702764547
|
|
|
@ -8,7 +8,7 @@
|
||||||
package org.elasticsearch.xpack.vectors.mapper;
|
package org.elasticsearch.xpack.vectors.mapper;
|
||||||
|
|
||||||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat;
|
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
|
||||||
import org.apache.lucene.document.BinaryDocValuesField;
|
import org.apache.lucene.document.BinaryDocValuesField;
|
||||||
import org.apache.lucene.document.Field;
|
import org.apache.lucene.document.Field;
|
||||||
import org.apache.lucene.document.KnnVectorField;
|
import org.apache.lucene.document.KnnVectorField;
|
||||||
|
@ -528,7 +528,7 @@ public class DenseVectorFieldMapper extends FieldMapper implements PerFieldKnnVe
|
||||||
return null; // use default format
|
return null; // use default format
|
||||||
} else {
|
} else {
|
||||||
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
|
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
|
||||||
return new Lucene91HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
|
return new Lucene92HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,7 +11,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||||
|
|
||||||
import org.apache.lucene.codecs.Codec;
|
import org.apache.lucene.codecs.Codec;
|
||||||
import org.apache.lucene.codecs.KnnVectorsFormat;
|
import org.apache.lucene.codecs.KnnVectorsFormat;
|
||||||
import org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat;
|
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
|
||||||
import org.apache.lucene.document.BinaryDocValuesField;
|
import org.apache.lucene.document.BinaryDocValuesField;
|
||||||
import org.apache.lucene.document.KnnVectorField;
|
import org.apache.lucene.document.KnnVectorField;
|
||||||
import org.apache.lucene.index.IndexableField;
|
import org.apache.lucene.index.IndexableField;
|
||||||
|
@ -42,8 +42,8 @@ import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
||||||
import static org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat.DEFAULT_MAX_CONN;
|
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN;
|
||||||
import static org.hamcrest.Matchers.containsString;
|
import static org.hamcrest.Matchers.containsString;
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
import static org.hamcrest.Matchers.equalTo;
|
||||||
import static org.hamcrest.Matchers.instanceOf;
|
import static org.hamcrest.Matchers.instanceOf;
|
||||||
|
@ -469,8 +469,8 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
|
||||||
Codec codec = codecService.codec("default");
|
Codec codec = codecService.codec("default");
|
||||||
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
|
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
|
||||||
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
|
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
|
||||||
assertThat(knnVectorsFormat, instanceOf(Lucene91HnswVectorsFormat.class));
|
assertThat(knnVectorsFormat, instanceOf(Lucene92HnswVectorsFormat.class));
|
||||||
String expectedString = "Lucene91HnswVectorsFormat(name = Lucene91HnswVectorsFormat, maxConn = "
|
String expectedString = "lucene92HnswVectorsFormat(name = lucene92HnswVectorsFormat, maxConn = "
|
||||||
+ m
|
+ m
|
||||||
+ ", beamWidth="
|
+ ", beamWidth="
|
||||||
+ efConstruction
|
+ efConstruction
|
||||||
|
|
|
@ -7,6 +7,8 @@
|
||||||
|
|
||||||
package org.elasticsearch.xpack.vectors.query;
|
package org.elasticsearch.xpack.vectors.query;
|
||||||
|
|
||||||
|
import org.apache.lucene.search.BooleanClause;
|
||||||
|
import org.apache.lucene.search.BooleanQuery;
|
||||||
import org.apache.lucene.search.KnnVectorQuery;
|
import org.apache.lucene.search.KnnVectorQuery;
|
||||||
import org.apache.lucene.search.Query;
|
import org.apache.lucene.search.Query;
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
|
@ -37,7 +39,6 @@ import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static org.hamcrest.Matchers.containsString;
|
import static org.hamcrest.Matchers.containsString;
|
||||||
import static org.hamcrest.Matchers.equalTo;
|
|
||||||
import static org.hamcrest.Matchers.instanceOf;
|
import static org.hamcrest.Matchers.instanceOf;
|
||||||
|
|
||||||
public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
|
public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
|
||||||
|
@ -98,12 +99,19 @@ public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQ
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) {
|
protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
|
||||||
assertTrue(query instanceof KnnVectorQuery);
|
assertTrue(query instanceof KnnVectorQuery);
|
||||||
KnnVectorQuery knnVectorQuery = (KnnVectorQuery) query;
|
KnnVectorQuery knnVectorQuery = (KnnVectorQuery) query;
|
||||||
|
|
||||||
|
BooleanQuery.Builder builder = new BooleanQuery.Builder();
|
||||||
|
for (QueryBuilder qb : queryBuilder.filterQueries()) {
|
||||||
|
builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
|
||||||
|
}
|
||||||
|
BooleanQuery booleanQuery = builder.build();
|
||||||
|
Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
|
||||||
// The field should always be resolved to the concrete field
|
// The field should always be resolved to the concrete field
|
||||||
assertThat(knnVectorQuery, equalTo(new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands())));
|
Query knnVectorQueryBuilt = new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
|
||||||
|
assertEquals(knnVectorQuery, knnVectorQueryBuilt);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testWrongDimension() {
|
public void testWrongDimension() {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue