mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-19 04:45:07 -04:00
Merge remote-tracking branch 'upstream/main' into mp-rest-tests
This commit is contained in:
Niels Bauman
commit
92af261250
92 changed files with 2139 additions and 391 deletions
|
|
@ -39,7 +39,6 @@ import org.openjdk.jmh.annotations.Param;
|
|||
import org.openjdk.jmh.annotations.Scope;
|
||||
import org.openjdk.jmh.annotations.Setup;
|
||||
import org.openjdk.jmh.annotations.State;
|
||||
import org.openjdk.jmh.annotations.TearDown;
|
||||
import org.openjdk.jmh.annotations.Threads;
|
||||
import org.openjdk.jmh.annotations.Warmup;
|
||||
import org.openjdk.jmh.profile.AsyncProfiler;
|
||||
|
|
@ -51,9 +50,8 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
|
|||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
@BenchmarkMode(Mode.SingleShotTime)
|
||||
@OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
|
|
@ -71,23 +69,10 @@ public class TSDBDocValuesMergeBenchmark {
|
|||
LogConfigurator.setNodeName("test");
|
||||
}
|
||||
|
||||
@Param("20431204")
|
||||
private int nDocs;
|
||||
|
||||
@Param("1000")
|
||||
private int deltaTime;
|
||||
|
||||
@Param("42")
|
||||
private int seed;
|
||||
|
||||
private static final String TIMESTAMP_FIELD = "@timestamp";
|
||||
private static final String HOSTNAME_FIELD = "host.name";
|
||||
private static final long BASE_TIMESTAMP = 1704067200000L;
|
||||
|
||||
private IndexWriter indexWriterWithoutOptimizedMerge;
|
||||
private IndexWriter indexWriterWithOptimizedMerge;
|
||||
private ExecutorService executorService;
|
||||
|
||||
public static void main(String[] args) throws RunnerException {
|
||||
final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
|
||||
.addProfiler(AsyncProfiler.class)
|
||||
|
|
@ -96,78 +81,168 @@ public class TSDBDocValuesMergeBenchmark {
|
|||
new Runner(options).run();
|
||||
}
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void setup() throws IOException {
|
||||
executorService = Executors.newSingleThreadExecutor();
|
||||
@State(Scope.Benchmark)
|
||||
public static class StateDenseWithoutOptimizeMerge {
|
||||
|
||||
final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
|
||||
final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
|
||||
@Param("20431204")
|
||||
private int nDocs;
|
||||
|
||||
@Param("1000")
|
||||
private int deltaTime;
|
||||
|
||||
@Param("42")
|
||||
private int seed;
|
||||
|
||||
private Directory directory;
|
||||
private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(false);
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void setup() throws IOException {
|
||||
directory = FSDirectory.open(Files.createTempDirectory("temp2-"));
|
||||
createIndex(directory, iwc.get(), false, nDocs, deltaTime, seed);
|
||||
}
|
||||
|
||||
indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
|
||||
indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
|
||||
}
|
||||
|
||||
private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
|
||||
final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
|
||||
@Benchmark
|
||||
public void forceMergeDenseWithoutOptimizedMerge(StateDenseWithoutOptimizeMerge state) throws IOException {
|
||||
forceMerge(state.directory, state.iwc.get());
|
||||
}
|
||||
|
||||
@State(Scope.Benchmark)
|
||||
public static class StateDenseWithOptimizeMerge {
|
||||
|
||||
@Param("20431204")
|
||||
private int nDocs;
|
||||
|
||||
@Param("1000")
|
||||
private int deltaTime;
|
||||
|
||||
@Param("42")
|
||||
private int seed;
|
||||
|
||||
private Directory directory;
|
||||
private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(true);
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void setup() throws IOException {
|
||||
directory = FSDirectory.open(Files.createTempDirectory("temp1-"));
|
||||
createIndex(directory, iwc.get(), false, nDocs, deltaTime, seed);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void forceMergeDenseWithOptimizedMerge(StateDenseWithOptimizeMerge state) throws IOException {
|
||||
forceMerge(state.directory, state.iwc.get());
|
||||
}
|
||||
|
||||
@State(Scope.Benchmark)
|
||||
public static class StateSparseWithoutOptimizeMerge {
|
||||
|
||||
@Param("20431204")
|
||||
private int nDocs;
|
||||
|
||||
@Param("1000")
|
||||
private int deltaTime;
|
||||
|
||||
@Param("42")
|
||||
private int seed;
|
||||
|
||||
private Directory directory;
|
||||
private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(false);
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void setup() throws IOException {
|
||||
directory = FSDirectory.open(Files.createTempDirectory("temp4-"));
|
||||
createIndex(directory, iwc.get(), true, nDocs, deltaTime, seed);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void forceMergeSparseWithoutOptimizedMerge(StateSparseWithoutOptimizeMerge state) throws IOException {
|
||||
forceMerge(state.directory, state.iwc.get());
|
||||
}
|
||||
|
||||
@State(Scope.Benchmark)
|
||||
public static class StateSparseWithOptimizeMerge {
|
||||
|
||||
@Param("20431204")
|
||||
private int nDocs;
|
||||
|
||||
@Param("1000")
|
||||
private int deltaTime;
|
||||
|
||||
@Param("42")
|
||||
private int seed;
|
||||
|
||||
private Directory directory;
|
||||
private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(true);
|
||||
|
||||
@Setup(Level.Trial)
|
||||
public void setup() throws IOException {
|
||||
directory = FSDirectory.open(Files.createTempDirectory("temp3-"));
|
||||
createIndex(directory, iwc.get(), true, nDocs, deltaTime, seed);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void forceMergeSparseWithOptimizedMerge(StateSparseWithOptimizeMerge state) throws IOException {
|
||||
forceMerge(state.directory, state.iwc.get());
|
||||
}
|
||||
|
||||
private void forceMerge(Directory directory, IndexWriterConfig config) throws IOException {
|
||||
try (var indexWriter = new IndexWriter(directory, config)) {
|
||||
indexWriter.forceMerge(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void createIndex(Directory directory, IndexWriterConfig iwc, boolean sparse, int nDocs, int deltaTime, int seed)
|
||||
throws IOException {
|
||||
long counter1 = 0;
|
||||
long counter2 = 10_000_000;
|
||||
long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
|
||||
long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
|
||||
int numHosts = 1000;
|
||||
int numHosts = 10000;
|
||||
String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
|
||||
|
||||
final Random random = new Random(seed);
|
||||
IndexWriter indexWriter = new IndexWriter(directory, iwc);
|
||||
for (int i = 0; i < nDocs; i++) {
|
||||
final Document doc = new Document();
|
||||
try (var indexWriter = new IndexWriter(directory, iwc)) {
|
||||
for (int i = 0; i < nDocs; i++) {
|
||||
final Document doc = new Document();
|
||||
|
||||
final int batchIndex = i / numHosts;
|
||||
final String hostName = "host-" + batchIndex;
|
||||
// Slightly vary the timestamp in each document
|
||||
final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
|
||||
final int batchIndex = i % numHosts;
|
||||
final String hostName = "host-" + batchIndex;
|
||||
// Slightly vary the timestamp in each document
|
||||
final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
|
||||
|
||||
doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
|
||||
doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
|
||||
doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
|
||||
doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
|
||||
doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
|
||||
doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
|
||||
int numTags = tags.length % (i + 1);
|
||||
for (int j = 0; j < numTags; j++) {
|
||||
doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
|
||||
}
|
||||
|
||||
indexWriter.addDocument(doc);
|
||||
}
|
||||
indexWriter.commit();
|
||||
return indexWriter;
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void forceMergeWithoutOptimizedMerge() throws IOException {
|
||||
forceMerge(indexWriterWithoutOptimizedMerge);
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void forceMergeWithOptimizedMerge() throws IOException {
|
||||
forceMerge(indexWriterWithOptimizedMerge);
|
||||
}
|
||||
|
||||
private void forceMerge(final IndexWriter indexWriter) throws IOException {
|
||||
indexWriter.forceMerge(1);
|
||||
}
|
||||
|
||||
@TearDown(Level.Trial)
|
||||
public void tearDown() {
|
||||
if (executorService != null) {
|
||||
executorService.shutdown();
|
||||
try {
|
||||
if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
|
||||
executorService.shutdownNow();
|
||||
doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
|
||||
doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
|
||||
if (sparse == false || random.nextBoolean()) {
|
||||
doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
|
||||
}
|
||||
if (sparse == false || random.nextBoolean()) {
|
||||
doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
|
||||
}
|
||||
if (sparse == false || random.nextBoolean()) {
|
||||
doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
|
||||
}
|
||||
if (sparse == false || random.nextBoolean()) {
|
||||
doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
|
||||
}
|
||||
if (sparse == false || random.nextBoolean()) {
|
||||
int numTags = tags.length % (i + 1);
|
||||
for (int j = 0; j < numTags; j++) {
|
||||
doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
|
||||
}
|
||||
}
|
||||
indexWriter.addDocument(doc);
|
||||
|
||||
if (i % 10000 == 0) {
|
||||
indexWriter.commit();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
executorService.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -271,8 +271,7 @@ class ServerCli extends EnvironmentAwareCommand {
|
|||
.withProcessInfo(processInfo)
|
||||
.withServerArgs(args)
|
||||
.withTempDir(tempDir)
|
||||
.withJvmOptions(jvmOptions)
|
||||
.withWorkingDir(args.logsDir());
|
||||
.withJvmOptions(jvmOptions);
|
||||
return serverProcessBuilder.start();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
package org.elasticsearch.server.cli;
|
||||
|
||||
import org.elasticsearch.bootstrap.ServerArgs;
|
||||
import org.elasticsearch.cli.ExitCodes;
|
||||
import org.elasticsearch.cli.ProcessInfo;
|
||||
import org.elasticsearch.cli.Terminal;
|
||||
import org.elasticsearch.cli.UserException;
|
||||
|
|
@ -21,6 +22,8 @@ import org.elasticsearch.core.SuppressForbidden;
|
|||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -44,7 +47,6 @@ public class ServerProcessBuilder {
|
|||
private ServerArgs serverArgs;
|
||||
private ProcessInfo processInfo;
|
||||
private List<String> jvmOptions;
|
||||
private Path workingDir;
|
||||
private Terminal terminal;
|
||||
|
||||
// this allows mocking the process building by tests
|
||||
|
|
@ -84,11 +86,6 @@ public class ServerProcessBuilder {
|
|||
return this;
|
||||
}
|
||||
|
||||
public ServerProcessBuilder withWorkingDir(Path workingDir) {
|
||||
this.workingDir = workingDir;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Specifies the {@link Terminal} to use for reading input and writing output from/to the cli console
|
||||
*/
|
||||
|
|
@ -141,6 +138,17 @@ public class ServerProcessBuilder {
|
|||
return start(ProcessBuilder::start);
|
||||
}
|
||||
|
||||
private void ensureWorkingDirExists() throws UserException {
|
||||
Path workingDir = serverArgs.logsDir();
|
||||
try {
|
||||
Files.createDirectories(workingDir);
|
||||
} catch (FileAlreadyExistsException e) {
|
||||
throw new UserException(ExitCodes.CONFIG, "Logs dir [" + workingDir + "] exists but is not a directory", e);
|
||||
} catch (IOException e) {
|
||||
throw new UserException(ExitCodes.CONFIG, "Unable to create logs dir [" + workingDir + "]", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static void checkRequiredArgument(Object argument, String argumentName) {
|
||||
if (argument == null) {
|
||||
throw new IllegalStateException(
|
||||
|
|
@ -157,12 +165,14 @@ public class ServerProcessBuilder {
|
|||
checkRequiredArgument(jvmOptions, "jvmOptions");
|
||||
checkRequiredArgument(terminal, "terminal");
|
||||
|
||||
ensureWorkingDirExists();
|
||||
|
||||
Process jvmProcess = null;
|
||||
ErrorPumpThread errorPump;
|
||||
|
||||
boolean success = false;
|
||||
try {
|
||||
jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), workingDir, processStarter);
|
||||
jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), serverArgs.logsDir(), processStarter);
|
||||
errorPump = new ErrorPumpThread(terminal, jvmProcess.getErrorStream());
|
||||
errorPump.start();
|
||||
sendArgs(serverArgs, jvmProcess.getOutputStream());
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ public class ServerProcessTests extends ESTestCase {
|
|||
protected final Map<String, String> sysprops = new HashMap<>();
|
||||
protected final Map<String, String> envVars = new HashMap<>();
|
||||
Path esHomeDir;
|
||||
Path workingDir;
|
||||
Path logsDir;
|
||||
Settings.Builder nodeSettings;
|
||||
ProcessValidator processValidator;
|
||||
MainMethod mainCallback;
|
||||
|
|
@ -94,8 +94,8 @@ public class ServerProcessTests extends ESTestCase {
|
|||
sysprops.put("os.name", "Linux");
|
||||
sysprops.put("java.home", "javahome");
|
||||
sysprops.put("es.path.home", esHomeDir.toString());
|
||||
logsDir = esHomeDir.resolve("logs");
|
||||
envVars.clear();
|
||||
workingDir = createTempDir();
|
||||
nodeSettings = Settings.builder();
|
||||
processValidator = null;
|
||||
mainCallback = null;
|
||||
|
|
@ -207,15 +207,7 @@ public class ServerProcessTests extends ESTestCase {
|
|||
}
|
||||
|
||||
ServerArgs createServerArgs(boolean daemonize, boolean quiet) {
|
||||
return new ServerArgs(
|
||||
daemonize,
|
||||
quiet,
|
||||
null,
|
||||
secrets,
|
||||
nodeSettings.build(),
|
||||
esHomeDir.resolve("config"),
|
||||
esHomeDir.resolve("logs")
|
||||
);
|
||||
return new ServerArgs(daemonize, quiet, null, secrets, nodeSettings.build(), esHomeDir.resolve("config"), logsDir);
|
||||
}
|
||||
|
||||
ServerProcess startProcess(boolean daemonize, boolean quiet) throws Exception {
|
||||
|
|
@ -231,8 +223,7 @@ public class ServerProcessTests extends ESTestCase {
|
|||
.withProcessInfo(pinfo)
|
||||
.withServerArgs(createServerArgs(daemonize, quiet))
|
||||
.withJvmOptions(List.of())
|
||||
.withTempDir(ServerProcessUtils.setupTempDir(pinfo))
|
||||
.withWorkingDir(workingDir);
|
||||
.withTempDir(ServerProcessUtils.setupTempDir(pinfo));
|
||||
return serverProcessBuilder.start(starter);
|
||||
}
|
||||
|
||||
|
|
@ -241,7 +232,7 @@ public class ServerProcessTests extends ESTestCase {
|
|||
assertThat(pb.redirectInput(), equalTo(ProcessBuilder.Redirect.PIPE));
|
||||
assertThat(pb.redirectOutput(), equalTo(ProcessBuilder.Redirect.INHERIT));
|
||||
assertThat(pb.redirectError(), equalTo(ProcessBuilder.Redirect.PIPE));
|
||||
assertThat(String.valueOf(pb.directory()), equalTo(workingDir.toString())); // leave default, which is working directory
|
||||
assertThat(String.valueOf(pb.directory()), equalTo(esHomeDir.resolve("logs").toString()));
|
||||
};
|
||||
mainCallback = (args, stdin, stderr, exitCode) -> {
|
||||
try (PrintStream err = new PrintStream(stderr, true, StandardCharsets.UTF_8)) {
|
||||
|
|
@ -315,8 +306,7 @@ public class ServerProcessTests extends ESTestCase {
|
|||
.withProcessInfo(createProcessInfo())
|
||||
.withServerArgs(createServerArgs(false, false))
|
||||
.withJvmOptions(List.of("-Dfoo1=bar", "-Dfoo2=baz"))
|
||||
.withTempDir(Path.of("."))
|
||||
.withWorkingDir(workingDir);
|
||||
.withTempDir(Path.of("."));
|
||||
serverProcessBuilder.start(starter).waitFor();
|
||||
}
|
||||
|
||||
|
|
@ -433,4 +423,26 @@ public class ServerProcessTests extends ESTestCase {
|
|||
int exitCode = server.waitFor();
|
||||
assertThat(exitCode, equalTo(-9));
|
||||
}
|
||||
|
||||
public void testLogsDirIsFile() throws Exception {
|
||||
Files.createFile(logsDir);
|
||||
var e = expectThrows(UserException.class, this::runForeground);
|
||||
assertThat(e.getMessage(), containsString("exists but is not a directory"));
|
||||
}
|
||||
|
||||
public void testLogsDirCreateParents() throws Exception {
|
||||
Path testDir = createTempDir();
|
||||
logsDir = testDir.resolve("subdir/logs");
|
||||
processValidator = pb -> assertThat(String.valueOf(pb.directory()), equalTo(logsDir.toString()));
|
||||
runForeground();
|
||||
}
|
||||
|
||||
public void testLogsCreateFailure() throws Exception {
|
||||
Path testDir = createTempDir();
|
||||
Path parentFile = testDir.resolve("exists");
|
||||
Files.createFile(parentFile);
|
||||
logsDir = parentFile.resolve("logs");
|
||||
var e = expectThrows(UserException.class, this::runForeground);
|
||||
assertThat(e.getMessage(), containsString("Unable to create logs dir"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
5
docs/changelog/126629.yaml
Normal file
5
docs/changelog/126629.yaml
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
pr: 126629
|
||||
summary: Default new `semantic_text` fields to use BBQ when models are compatible
|
||||
area: Relevance
|
||||
type: enhancement
|
||||
issues: []
|
||||
6
docs/changelog/126770.yaml
Normal file
6
docs/changelog/126770.yaml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
pr: 126770
|
||||
summary: Remove empty results before merging
|
||||
area: Search
|
||||
type: bug
|
||||
issues:
|
||||
- 126742
|
||||
|
|
@ -94,11 +94,7 @@ The following are the types of thread pools and their respective parameters:
|
|||
|
||||
### `fixed` [fixed-thread-pool]
|
||||
|
||||
The `fixed` thread pool holds a fixed size of threads to handle the requests with a queue (optionally bounded) for pending requests that have no threads to service them.
|
||||
|
||||
The `size` parameter controls the number of threads.
|
||||
|
||||
The `queue_size` allows to control the size of the queue of pending requests that have no threads to execute them. By default, it is set to `-1` which means its unbounded. When a request comes in and the queue is full, it will abort the request.
|
||||
A `fixed` thread pool holds a fixed number of threads as determined by the `size` parameter. If a task is submitted to a `fixed` thread pool and there are fewer than `size` busy threads in the pool then the task will execute immediately. If all the threads are busy when a task is submitted then it will be held in a queue for later execution. The `queue_size` parameter controls the maximum size of this queue. A `queue_size` of `-1` means that the queue is unbounded, but most `fixed` thread pools specify a bound on their queue size by default. If a bounded queue is full then it will reject further work, which typically causes the corresponding requests to fail.
|
||||
|
||||
```yaml
|
||||
thread_pool:
|
||||
|
|
@ -114,6 +110,8 @@ The `scaling` thread pool holds a dynamic number of threads. This number is prop
|
|||
|
||||
The `keep_alive` parameter determines how long a thread should be kept around in the thread pool without it doing any work.
|
||||
|
||||
If a task is submitted to a `scaling` thread pool when its maximum number of threads are already busy with other tasks, the new task will be held in a queue for later execution. The queue in a `scaling` thread pool is always unbounded.
|
||||
|
||||
```yaml
|
||||
thread_pool:
|
||||
warmer:
|
||||
|
|
|
|||
|
|
@ -560,11 +560,11 @@ Refer to [*Semantic re-ranking*](docs-content://solutions/search/ranking/semanti
|
|||
|
||||
### Prerequisites [_prerequisites_15]
|
||||
|
||||
To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which is based on [Elastic Rerank](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-rerank.html) and serves as the default if no `inference_id` is provided. This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be configured with a machine learning model capable of computing text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
|
||||
To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which uses the [Elastic Rerank model](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-rerank.md) and serves as the default if no `inference_id` is provided. This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be configured with a machine learning model capable of computing text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
|
||||
|
||||
You have the following options:
|
||||
|
||||
* Use the built-in [Elastic Rerank](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) cross-encoder model via the inference API’s {{es}} service. For an example of creating an endpoint using the Elastic Rerank model, refer to [this guide](https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html#inference-example-elastic-reranker).
|
||||
* Use the built-in [Elastic Rerank](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-rerank.md) cross-encoder model via the inference API’s {{es}} service. See [this example](https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html#inference-example-elastic-reranker) for creating an endpoint using the Elastic Rerank model.
|
||||
* Use the [Cohere Rerank inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
|
||||
* Use the [Google Vertex AI inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
|
||||
* Upload a model to {{es}} with [Eland](eland://reference/machine-learning.md#ml-nlp-pytorch) using the `text_similarity` NLP task type.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
```esql
|
||||
FROM books
|
||||
| WHERE MULTI_MATCH("Faulkner", author, description, {"fuzziness": 1})
|
||||
| WHERE MULTI_MATCH("Faulkner", author, description)
|
||||
| KEEP book_no, author
|
||||
| SORT book_no
|
||||
| LIMIT 5
|
||||
|
|
|
|||
|
|
@ -731,7 +731,7 @@
|
|||
}
|
||||
],
|
||||
"examples" : [
|
||||
"FROM books\n| WHERE MULTI_MATCH(\"Faulkner\", author, description, {\"fuzziness\": 1})\n| KEEP book_no, author\n| SORT book_no\n| LIMIT 5",
|
||||
"FROM books\n| WHERE MULTI_MATCH(\"Faulkner\", author, description)\n| KEEP book_no, author\n| SORT book_no\n| LIMIT 5",
|
||||
"FROM books\n| WHERE MULTI_MATCH(\"Hobbit Back Again\", title, description, {\"operator\": \"AND\"})\n| KEEP title;"
|
||||
],
|
||||
"preview" : true,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### GREATEST
|
||||
Returns the maximum value from multiple columns. This is similar to [`MV_MAX`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/mv-functions#esql-mv_max)
|
||||
Returns the maximum value from multiple columns. This is similar to [`MV_MAX`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/mv-functions#esql-mv_max)
|
||||
except it is intended to run on multiple columns at once.
|
||||
|
||||
```esql
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### LEAST
|
||||
Returns the minimum value from multiple columns. This is similar to [`MV_MIN`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/mv-functions#esql-mv_min) except it is intended to run on multiple columns at once.
|
||||
Returns the minimum value from multiple columns. This is similar to [`MV_MIN`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/mv-functions#esql-mv_min) except it is intended to run on multiple columns at once.
|
||||
|
||||
```esql
|
||||
ROW a = 10, b = 20
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MATCH
|
||||
Use `MATCH` to perform a [match query](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-match-query) on the specified field.
|
||||
Use `MATCH` to perform a [match query](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-match-query) on the specified field.
|
||||
Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.
|
||||
|
||||
Match can be used on fields from the text family like [text](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/text) and [semantic_text](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/semantic-text),
|
||||
Match can be used on fields from the text family like [text](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/text) and [semantic_text](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/semantic-text),
|
||||
as well as other field types like keyword, boolean, dates, and numeric types.
|
||||
|
||||
Match can use [function named parameters](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-syntax#esql-function-named-params) to specify additional options for the match query.
|
||||
All [match query parameters](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-match-query#match-field-params) are supported.
|
||||
Match can use [function named parameters](https://www.elastic.co/docs/reference/query-languages/esql/esql-syntax#esql-function-named-params) to specify additional options for the match query.
|
||||
All [match query parameters](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-match-query#match-field-params) are supported.
|
||||
|
||||
For a simplified syntax, you can use the [match operator](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/operators#esql-match-operator) `:` operator instead of `MATCH`.
|
||||
For a simplified syntax, you can use the [match operator](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/operators#esql-match-operator) `:` operator instead of `MATCH`.
|
||||
|
||||
`MATCH` returns true if the provided query matches the row.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MEDIAN
|
||||
The value that is greater than half of all values and less than half of all values, also known as the 50% [`PERCENTILE`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-percentile).
|
||||
The value that is greater than half of all values and less than half of all values, also known as the 50% [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile).
|
||||
|
||||
```esql
|
||||
FROM employees
|
||||
| STATS MEDIAN(salary), PERCENTILE(salary, 50)
|
||||
```
|
||||
Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN` is [usually approximate](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate).
|
||||
Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate).
|
||||
|
|
|
|||
|
|
@ -9,4 +9,4 @@ It is calculated as the median of each data point’s deviation from the median
|
|||
FROM employees
|
||||
| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary)
|
||||
```
|
||||
Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN_ABSOLUTE_DEVIATION` is [usually approximate](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate).
|
||||
Note: Like [`PERCENTILE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile), `MEDIAN_ABSOLUTE_DEVIATION` is [usually approximate](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-percentile-approximate).
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
<!--
|
||||
This is generated by ESQL’s AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
-->
|
||||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MULTI_MATCH
|
||||
Use `MULTI_MATCH` to perform a [multi-match query](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-match-query#query-dsl-multi-match-query) on the specified field.
|
||||
### MULTI MATCH
|
||||
Use `MULTI_MATCH` to perform a [multi-match query](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-match-query#query-dsl-multi-match-query) on the specified field.
|
||||
The multi_match query builds on the match query to allow multi-field queries.
|
||||
|
||||
```esql
|
||||
FROM books
|
||||
| WHERE MULTI_MATCH("Faulkner", author, description, {"fuzziness": 1})
|
||||
| WHERE MULTI_MATCH("Faulkner", author, description)
|
||||
| KEEP book_no, author
|
||||
| SORT book_no
|
||||
| LIMIT 5
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
### MV FIRST
|
||||
Converts a multivalued expression into a single valued column containing the
|
||||
first value. This is most useful when reading from a function that emits
|
||||
multivalued columns in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/string-functions#esql-split).
|
||||
multivalued columns in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/string-functions#esql-split).
|
||||
|
||||
```esql
|
||||
ROW a="foo;bar;baz"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
### MV LAST
|
||||
Converts a multivalue expression into a single valued column containing the last
|
||||
value. This is most useful when reading from a function that emits multivalued
|
||||
columns in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/string-functions#esql-split).
|
||||
columns in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/string-functions#esql-split).
|
||||
|
||||
```esql
|
||||
ROW a="foo;bar;baz"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
### MV SLICE
|
||||
Returns a subset of the multivalued field using the start and end index values.
|
||||
This is most useful when reading from a function that emits multivalued columns
|
||||
in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/string-functions#esql-split) or [`MV_SORT`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/mv-functions#esql-mv_sort).
|
||||
in a known order like [`SPLIT`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/string-functions#esql-split) or [`MV_SORT`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/mv-functions#esql-mv_sort).
|
||||
|
||||
```esql
|
||||
row a = [1, 2, 2, 3]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### QSTR
|
||||
Performs a [query string query](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-query-string-query). Returns true if the provided query string matches the row.
|
||||
Performs a [query string query](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-query-string-query). Returns true if the provided query string matches the row.
|
||||
|
||||
```esql
|
||||
FROM books
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
### ST CONTAINS
|
||||
Returns whether the first geometry contains the second geometry.
|
||||
This is the inverse of the [ST_WITHIN](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/spatial-functions#esql-st_within) function.
|
||||
This is the inverse of the [ST_WITHIN](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/spatial-functions#esql-st_within) function.
|
||||
|
||||
```esql
|
||||
FROM airport_city_boundaries
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
### ST DISJOINT
|
||||
Returns whether the two geometries or geometry columns are disjoint.
|
||||
This is the inverse of the [ST_INTERSECTS](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/spatial-functions#esql-st_intersects) function.
|
||||
This is the inverse of the [ST_INTERSECTS](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/spatial-functions#esql-st_intersects) function.
|
||||
In mathematical terms: ST_Disjoint(A, B) ⇔ A ⋂ B = ∅
|
||||
|
||||
```esql
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
Returns true if two geometries intersect.
|
||||
They intersect if they have any point in common, including their interior points
|
||||
(points along lines or within polygons).
|
||||
This is the inverse of the [ST_DISJOINT](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/spatial-functions#esql-st_disjoint) function.
|
||||
This is the inverse of the [ST_DISJOINT](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/spatial-functions#esql-st_disjoint) function.
|
||||
In mathematical terms: ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅
|
||||
|
||||
```esql
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
### ST WITHIN
|
||||
Returns whether the first geometry is within the second geometry.
|
||||
This is the inverse of the [ST_CONTAINS](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/spatial-functions#esql-st_contains) function.
|
||||
This is the inverse of the [ST_CONTAINS](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/spatial-functions#esql-st_contains) function.
|
||||
|
||||
```esql
|
||||
FROM airport_city_boundaries
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
### TO DATETIME
|
||||
Converts an input value to a date value.
|
||||
A string will only be successfully converted if it’s respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.
|
||||
To convert dates in other formats, use [`DATE_PARSE`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/date-time-functions#esql-date_parse).
|
||||
To convert dates in other formats, use [`DATE_PARSE`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/date-time-functions#esql-date_parse).
|
||||
|
||||
```esql
|
||||
ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### VALUES
|
||||
Returns all values in a group as a multivalued field. The order of the returned values isn’t guaranteed. If you need the values returned in order use [`MV_SORT`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/mv-functions#esql-mv_sort).
|
||||
Returns all values in a group as a multivalued field. The order of the returned values isn’t guaranteed. If you need the values returned in order use [`MV_SORT`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/mv-functions#esql-mv_sort).
|
||||
|
||||
```esql
|
||||
FROM employees
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### ADD `+`
|
||||
Add two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Add two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### CAST `::`
|
||||
The `::` operator provides a convenient alternative syntax to the TO_<type> [conversion functions](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/type-conversion-functions).
|
||||
The `::` operator provides a convenient alternative syntax to the TO_<type> [conversion functions](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/type-conversion-functions).
|
||||
|
||||
```esql
|
||||
ROW ver = CONCAT(("0"::INT + 1)::STRING, ".2.3")::VERSION
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### DIVIDE `/`
|
||||
Divide one number by another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Divide one number by another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, [`Cast (::)`](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/operators#esql-cast-operator) one of the arguments to a `DOUBLE`.
|
||||
Note: Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, [`Cast (::)`](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/operators#esql-cast-operator) one of the arguments to a `DOUBLE`.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### EQUALS `==`
|
||||
Check if two fields are equal. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if two fields are equal. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### GREATER THAN `>`
|
||||
Check if one field is greater than another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if one field is greater than another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### GREATER THAN OR EQUAL `>=`
|
||||
Check if one field is greater than or equal to another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if one field is greater than or equal to another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### LESS THAN `<`
|
||||
Check if one field is less than another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if one field is less than another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### LESS THAN OR EQUAL `<=`
|
||||
Check if one field is less than or equal to another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if one field is less than or equal to another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MATCH OPERATOR `:`
|
||||
Use the match operator (`:`) to perform a [match query](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-match-query) on the specified field.
|
||||
Use the match operator (`:`) to perform a [match query](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-match-query) on the specified field.
|
||||
Using `:` is equivalent to using the `match` query in the Elasticsearch Query DSL.
|
||||
|
||||
The match operator is equivalent to the [match function](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-match).
|
||||
The match operator is equivalent to the [match function](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-match).
|
||||
|
||||
For using the function syntax, or adding [match query parameters](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/query-dsl-match-query#match-field-params), you can use the
|
||||
[match function](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/functions-operators/aggregation-functions#esql-match).
|
||||
For using the function syntax, or adding [match query parameters](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-match-query#match-field-params), you can use the
|
||||
[match function](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/aggregation-functions#esql-match).
|
||||
|
||||
`:` returns true if the provided query matches the row.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MODULO `%`
|
||||
Divide one number by another and return the remainder. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Divide one number by another and return the remainder. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### MULTIPLY `*`
|
||||
Multiply two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Multiply two numbers together. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
### NOT RLIKE
|
||||
Use `RLIKE` to filter data based on string patterns using using
|
||||
[regular expressions](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/regexp-syntax). `RLIKE` usually acts on a field placed on
|
||||
[regular expressions](https://www.elastic.co/docs/reference/query-languages/query-dsl/regexp-syntax). `RLIKE` usually acts on a field placed on
|
||||
the left-hand side of the operator, but it can also act on a constant (literal)
|
||||
expression. The right-hand side of the operator represents the pattern.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### NOT EQUALS `!=`
|
||||
Check if two fields are unequal. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Check if two fields are unequal. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/elasticsearch/mapping-reference/doc-values).
|
||||
Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an [mapping-index](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/mapping-index) and [doc-values](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/doc-values).
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
### RLIKE
|
||||
Use `RLIKE` to filter data based on string patterns using using
|
||||
[regular expressions](https://www.elastic.co/docs/reference/elasticsearch/query-languages/query-dsl/regexp-syntax). `RLIKE` usually acts on a field placed on
|
||||
[regular expressions](https://www.elastic.co/docs/reference/query-languages/query-dsl/regexp-syntax). `RLIKE` usually acts on a field placed on
|
||||
the left-hand side of the operator, but it can also act on a constant (literal)
|
||||
expression. The right-hand side of the operator represents the pattern.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
### SUBTRACT `-`
|
||||
Subtract one number from another. If either field is [multivalued](https://www.elastic.co/docs/reference/elasticsearch/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
Subtract one number from another. If either field is [multivalued](https://www.elastic.co/docs/reference/query-languages/esql/esql-multivalued-fields) then the result is `null`.
|
||||
|
||||
|
|
|
|||
|
|
@ -315,9 +315,6 @@ tests:
|
|||
- class: org.elasticsearch.search.CCSDuelIT
|
||||
method: testTerminateAfter
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126085
|
||||
- class: org.elasticsearch.search.sort.GeoDistanceIT
|
||||
method: testDistanceSortingWithUnmappedField
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126118
|
||||
- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
|
||||
method: testSearchWithRandomDisconnects
|
||||
issue: https://github.com/elastic/elasticsearch/issues/122707
|
||||
|
|
@ -387,9 +384,6 @@ tests:
|
|||
- class: org.elasticsearch.xpack.esql.action.EsqlActionIT
|
||||
method: testQueryOnEmptyDataIndex
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126580
|
||||
- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
|
||||
method: testShrinkActionInPolicyWithoutHotPhase
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126746
|
||||
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
|
||||
method: test {p0=transform/transforms_start_stop/Test start/stop/start continuous transform}
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126755
|
||||
|
|
@ -414,6 +408,12 @@ tests:
|
|||
- class: org.elasticsearch.cli.keystore.AddStringKeyStoreCommandTests
|
||||
method: testStdinWithMultipleValues
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126882
|
||||
- class: org.elasticsearch.packaging.test.DockerTests
|
||||
method: test024InstallPluginFromArchiveUsingConfigFile
|
||||
issue: https://github.com/elastic/elasticsearch/issues/126936
|
||||
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.RepositoryAnalysisFailureIT
|
||||
method: testFailsOnReadError
|
||||
issue: https://github.com/elastic/elasticsearch/issues/127029
|
||||
|
||||
# Examples:
|
||||
#
|
||||
|
|
|
|||
|
|
@ -225,6 +225,7 @@ public class TransportVersions {
|
|||
public static final TransportVersion ESQL_QUERY_PLANNING_DURATION = def(9_051_0_00);
|
||||
public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED = def(9_052_0_00);
|
||||
public static final TransportVersion BATCHED_QUERY_EXECUTION_DELAYABLE_WRITABLE = def(9_053_0_00);
|
||||
public static final TransportVersion SEARCH_INCREMENTAL_TOP_DOCS_NULL = def(9_054_0_00);
|
||||
|
||||
/*
|
||||
* STOP! READ THIS FIRST! No, really,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
package org.elasticsearch.action.admin.cluster.snapshots.status;
|
||||
|
||||
import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
|
|
@ -161,4 +162,9 @@ public class SnapshotIndexShardStatus extends BroadcastShardResponse implements
|
|||
result = 31 * result + (failure != null ? failure.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Strings.toString(this, true, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
package org.elasticsearch.action.admin.cluster.snapshots.status;
|
||||
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.xcontent.ToXContentFragment;
|
||||
import org.elasticsearch.xcontent.XContentBuilder;
|
||||
|
||||
|
|
@ -133,4 +134,9 @@ public class SnapshotIndexStatus implements Iterable<SnapshotIndexShardStatus>,
|
|||
result = 31 * result + (stats != null ? stats.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Strings.toString(this, true, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
package org.elasticsearch.action.admin.cluster.snapshots.status;
|
||||
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.xcontent.ToXContent;
|
||||
import org.elasticsearch.xcontent.ToXContentObject;
|
||||
import org.elasticsearch.xcontent.XContentBuilder;
|
||||
|
|
@ -150,4 +151,9 @@ public class SnapshotShardsStats implements ToXContentObject {
|
|||
result = 31 * result + totalShards;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Strings.toString(this, true, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -355,4 +355,9 @@ public class SnapshotStats implements Writeable, ToXContentObject {
|
|||
result = 31 * result + (int) (processedSize ^ (processedSize >>> 32));
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Strings.toString(this, true, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,9 +10,11 @@
|
|||
package org.elasticsearch.action.admin.cluster.snapshots.status;
|
||||
|
||||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.collect.Iterators;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.xcontent.ChunkedToXContent;
|
||||
import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
|
||||
import org.elasticsearch.xcontent.ToXContent;
|
||||
|
||||
|
|
@ -71,4 +73,9 @@ public class SnapshotsStatusResponse extends ActionResponse implements ChunkedTo
|
|||
Iterators.single((b, p) -> b.endArray().endObject())
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return Strings.toString(ChunkedToXContent.wrapAsToXContent(this), true, true);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -165,7 +165,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
|
|||
|
||||
}
|
||||
|
||||
private void buildResponse(
|
||||
// Package access for testing.
|
||||
void buildResponse(
|
||||
SnapshotsInProgress snapshotsInProgress,
|
||||
SnapshotsStatusRequest request,
|
||||
List<SnapshotsInProgress.Entry> currentSnapshotEntries,
|
||||
|
|
@ -190,6 +191,9 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
|
|||
for (Map.Entry<RepositoryShardId, SnapshotsInProgress.ShardSnapshotStatus> shardEntry : entry
|
||||
.shardSnapshotStatusByRepoShardId()
|
||||
.entrySet()) {
|
||||
if (task.notifyIfCancelled(listener)) {
|
||||
return;
|
||||
}
|
||||
SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.getValue();
|
||||
if (status.nodeId() != null) {
|
||||
// We should have information about this shard from the shard:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
|
|||
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
|
||||
import org.elasticsearch.core.Nullable;
|
||||
import org.elasticsearch.core.Releasable;
|
||||
import org.elasticsearch.core.Releasables;
|
||||
import org.elasticsearch.core.Tuple;
|
||||
import org.elasticsearch.search.SearchPhaseResult;
|
||||
import org.elasticsearch.search.SearchService;
|
||||
|
|
@ -162,7 +163,7 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
consume(querySearchResult, next);
|
||||
}
|
||||
|
||||
private final List<Tuple<TopDocsStats, MergeResult>> batchedResults = new ArrayList<>();
|
||||
private final ArrayDeque<Tuple<TopDocsStats, MergeResult>> batchedResults = new ArrayDeque<>();
|
||||
|
||||
/**
|
||||
* Unlinks partial merge results from this instance and returns them as a partial merge result to be sent to the coordinating node.
|
||||
|
|
@ -214,7 +215,7 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
buffer.sort(RESULT_COMPARATOR);
|
||||
final TopDocsStats topDocsStats = this.topDocsStats;
|
||||
var mergeResult = this.mergeResult;
|
||||
final List<Tuple<TopDocsStats, MergeResult>> batchedResults;
|
||||
final ArrayDeque<Tuple<TopDocsStats, MergeResult>> batchedResults;
|
||||
synchronized (this.batchedResults) {
|
||||
batchedResults = this.batchedResults;
|
||||
}
|
||||
|
|
@ -226,8 +227,8 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
if (mergeResult != null) {
|
||||
consumePartialMergeResult(mergeResult, topDocsList, aggsList);
|
||||
}
|
||||
for (int i = 0; i < batchedResults.size(); i++) {
|
||||
Tuple<TopDocsStats, MergeResult> batchedResult = batchedResults.set(i, null);
|
||||
Tuple<TopDocsStats, MergeResult> batchedResult;
|
||||
while ((batchedResult = batchedResults.poll()) != null) {
|
||||
topDocsStats.add(batchedResult.v1());
|
||||
consumePartialMergeResult(batchedResult.v2(), topDocsList, aggsList);
|
||||
}
|
||||
|
|
@ -303,13 +304,19 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
Collection<DelayableWriteable<InternalAggregations>> aggsList
|
||||
) {
|
||||
if (topDocsList != null) {
|
||||
topDocsList.add(partialResult.reducedTopDocs);
|
||||
addTopDocsToList(partialResult, topDocsList);
|
||||
}
|
||||
if (aggsList != null) {
|
||||
addAggsToList(partialResult, aggsList);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addTopDocsToList(MergeResult partialResult, List<TopDocs> topDocsList) {
|
||||
if (partialResult.reducedTopDocs != null) {
|
||||
topDocsList.add(partialResult.reducedTopDocs);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addAggsToList(MergeResult partialResult, Collection<DelayableWriteable<InternalAggregations>> aggsList) {
|
||||
var aggs = partialResult.reducedAggs;
|
||||
if (aggs != null) {
|
||||
|
|
@ -340,7 +347,7 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
if (hasTopDocs) {
|
||||
topDocsList = new ArrayList<>(resultSetSize);
|
||||
if (lastMerge != null) {
|
||||
topDocsList.add(lastMerge.reducedTopDocs);
|
||||
addTopDocsToList(lastMerge, topDocsList);
|
||||
}
|
||||
} else {
|
||||
topDocsList = null;
|
||||
|
|
@ -358,7 +365,7 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
}
|
||||
}
|
||||
// we have to merge here in the same way we collect on a shard
|
||||
newTopDocs = topDocsList == null ? Lucene.EMPTY_TOP_DOCS : mergeTopDocs(topDocsList, topNSize, 0);
|
||||
newTopDocs = topDocsList == null ? null : mergeTopDocs(topDocsList, topNSize, 0);
|
||||
newAggs = hasAggs
|
||||
? aggregate(
|
||||
toConsume.iterator(),
|
||||
|
|
@ -522,6 +529,12 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
querySearchResult.releaseAggs();
|
||||
}
|
||||
}
|
||||
synchronized (this.batchedResults) {
|
||||
Tuple<TopDocsStats, MergeResult> batchedResult;
|
||||
while ((batchedResult = batchedResults.poll()) != null) {
|
||||
Releasables.close(batchedResult.v2().reducedAggs());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void onMergeFailure(Exception exc) {
|
||||
|
|
@ -636,7 +649,7 @@ public class QueryPhaseResultConsumer extends ArraySearchPhaseResults<SearchPhas
|
|||
|
||||
record MergeResult(
|
||||
List<SearchShard> processedShards,
|
||||
TopDocs reducedTopDocs,
|
||||
@Nullable TopDocs reducedTopDocs,
|
||||
@Nullable DelayableWriteable<InternalAggregations> reducedAggs,
|
||||
long estimatedSize
|
||||
) implements Writeable {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ import java.util.Collections;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Consumer;
|
||||
|
|
@ -140,24 +141,26 @@ public final class SearchPhaseController {
|
|||
}
|
||||
|
||||
static TopDocs mergeTopDocs(List<TopDocs> results, int topN, int from) {
|
||||
if (results.isEmpty()) {
|
||||
List<TopDocs> topDocsList = results.stream().filter(Objects::nonNull).toList();
|
||||
if (topDocsList.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
final TopDocs topDocs = results.getFirst();
|
||||
final TopDocs mergedTopDocs;
|
||||
final int numShards = results.size();
|
||||
final TopDocs topDocs = topDocsList.getFirst();
|
||||
final int numShards = topDocsList.size();
|
||||
if (numShards == 1 && from == 0) { // only one shard and no pagination we can just return the topDocs as we got them.
|
||||
return topDocs;
|
||||
} else if (topDocs instanceof TopFieldGroups firstTopDocs) {
|
||||
}
|
||||
final TopDocs mergedTopDocs;
|
||||
if (topDocs instanceof TopFieldGroups firstTopDocs) {
|
||||
final Sort sort = new Sort(firstTopDocs.fields);
|
||||
final TopFieldGroups[] shardTopDocs = results.stream().filter(td -> td != Lucene.EMPTY_TOP_DOCS).toArray(TopFieldGroups[]::new);
|
||||
TopFieldGroups[] shardTopDocs = topDocsList.toArray(new TopFieldGroups[0]);
|
||||
mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false);
|
||||
} else if (topDocs instanceof TopFieldDocs firstTopDocs) {
|
||||
final Sort sort = checkSameSortTypes(results, firstTopDocs.fields);
|
||||
final TopFieldDocs[] shardTopDocs = results.stream().filter((td -> td != Lucene.EMPTY_TOP_DOCS)).toArray(TopFieldDocs[]::new);
|
||||
TopFieldDocs[] shardTopDocs = topDocsList.toArray(new TopFieldDocs[0]);
|
||||
final Sort sort = checkSameSortTypes(topDocsList, firstTopDocs.fields);
|
||||
mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
|
||||
} else {
|
||||
final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]);
|
||||
final TopDocs[] shardTopDocs = topDocsList.toArray(new TopDocs[0]);
|
||||
mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs);
|
||||
}
|
||||
return mergedTopDocs;
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
|
|||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.io.stream.Writeable;
|
||||
import org.elasticsearch.common.lucene.Lucene;
|
||||
import org.elasticsearch.common.util.concurrent.CountDown;
|
||||
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
||||
import org.elasticsearch.common.util.concurrent.ListenableFuture;
|
||||
|
|
@ -722,7 +721,7 @@ public class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<S
|
|||
|
||||
private static final QueryPhaseResultConsumer.MergeResult EMPTY_PARTIAL_MERGE_RESULT = new QueryPhaseResultConsumer.MergeResult(
|
||||
List.of(),
|
||||
Lucene.EMPTY_TOP_DOCS,
|
||||
null,
|
||||
null,
|
||||
0L
|
||||
);
|
||||
|
|
@ -782,10 +781,12 @@ public class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<S
|
|||
// also collect the set of indices that may be part of a subsequent fetch operation here so that we can release all other
|
||||
// indices without a roundtrip to the coordinating node
|
||||
final BitSet relevantShardIndices = new BitSet(searchRequest.shards.size());
|
||||
for (ScoreDoc scoreDoc : mergeResult.reducedTopDocs().scoreDocs) {
|
||||
final int localIndex = scoreDoc.shardIndex;
|
||||
scoreDoc.shardIndex = searchRequest.shards.get(localIndex).shardIndex;
|
||||
relevantShardIndices.set(localIndex);
|
||||
if (mergeResult.reducedTopDocs() != null) {
|
||||
for (ScoreDoc scoreDoc : mergeResult.reducedTopDocs().scoreDocs) {
|
||||
final int localIndex = scoreDoc.shardIndex;
|
||||
scoreDoc.shardIndex = searchRequest.shards.get(localIndex).shardIndex;
|
||||
relevantShardIndices.set(localIndex);
|
||||
}
|
||||
}
|
||||
final Object[] results = new Object[queryPhaseResultConsumer.getNumShards()];
|
||||
for (int i = 0; i < results.length; i++) {
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ import org.apache.lucene.util.Bits;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.elasticsearch.ExceptionsHelper;
|
||||
import org.elasticsearch.TransportVersions;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
|
|
@ -384,6 +385,14 @@ public class Lucene {
|
|||
* by shard for sorting purposes.
|
||||
*/
|
||||
public static void writeTopDocsIncludingShardIndex(StreamOutput out, TopDocs topDocs) throws IOException {
|
||||
if (topDocs == null) {
|
||||
if (out.getTransportVersion().onOrAfter(TransportVersions.SEARCH_INCREMENTAL_TOP_DOCS_NULL)) {
|
||||
out.writeByte((byte) -1);
|
||||
return;
|
||||
} else {
|
||||
topDocs = Lucene.EMPTY_TOP_DOCS;
|
||||
}
|
||||
}
|
||||
if (topDocs instanceof TopFieldGroups topFieldGroups) {
|
||||
out.writeByte((byte) 2);
|
||||
writeTotalHits(out, topDocs.totalHits);
|
||||
|
|
@ -424,7 +433,10 @@ public class Lucene {
|
|||
*/
|
||||
public static TopDocs readTopDocsIncludingShardIndex(StreamInput in) throws IOException {
|
||||
byte type = in.readByte();
|
||||
if (type == 0) {
|
||||
if (type == -1) {
|
||||
assert in.getTransportVersion().onOrAfter(TransportVersions.SEARCH_INCREMENTAL_TOP_DOCS_NULL);
|
||||
return null;
|
||||
} else if (type == 0) {
|
||||
TotalHits totalHits = readTotalHits(in);
|
||||
|
||||
final int scoreDocCount = in.readVInt();
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ public class IndexVersions {
|
|||
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0);
|
||||
public static final IndexVersion USE_LUCENE101_POSTINGS_FORMAT = def(9_021_0_00, Version.LUCENE_10_1_0);
|
||||
public static final IndexVersion UPGRADE_TO_LUCENE_10_2_0 = def(9_022_00_0, Version.LUCENE_10_2_0);
|
||||
public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_023_0_00, Version.LUCENE_10_2_0);
|
||||
/*
|
||||
* STOP! READ THIS FIRST! No, really,
|
||||
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.codec.tsdb.es819;
|
||||
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BitSetIterator;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.elasticsearch.core.SuppressForbidden;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Fork of {@link org.apache.lucene.codecs.lucene90.IndexedDISI#writeBitSet(DocIdSetIterator, IndexOutput)} but that allows
|
||||
* building jump list iteratively by one docid at a time instead of relying on docidset iterator.
|
||||
*/
|
||||
final class DISIAccumulator implements Closeable {
|
||||
|
||||
private static final int BLOCK_SIZE = 65536; // The number of docIDs that a single block represents
|
||||
|
||||
private static final int DENSE_BLOCK_LONGS = BLOCK_SIZE / Long.SIZE; // 1024
|
||||
public static final byte DEFAULT_DENSE_RANK_POWER = 9; // Every 512 docIDs / 8 longs
|
||||
|
||||
static final int MAX_ARRAY_LENGTH = (1 << 12) - 1;
|
||||
|
||||
final Directory dir;
|
||||
final IOContext context;
|
||||
final String skipListTempFileName;
|
||||
final IndexOutput disiTempOutput;
|
||||
final byte denseRankPower;
|
||||
final long origo;
|
||||
|
||||
int totalCardinality = 0;
|
||||
int blockCardinality = 0;
|
||||
final FixedBitSet buffer = new FixedBitSet(1 << 16);
|
||||
int[] jumps = new int[ArrayUtil.oversize(1, Integer.BYTES * 2)];
|
||||
int prevBlock = -1;
|
||||
int jumpBlockIndex = 0;
|
||||
|
||||
DISIAccumulator(Directory dir, IOContext context, IndexOutput data, byte denseRankPower) throws IOException {
|
||||
this.dir = dir;
|
||||
this.context = context;
|
||||
this.denseRankPower = denseRankPower;
|
||||
if ((denseRankPower < 7 || denseRankPower > 15) && denseRankPower != -1) {
|
||||
throw new IllegalArgumentException(
|
||||
"Acceptable values for denseRankPower are 7-15 (every 128-32768 docIDs). "
|
||||
+ "The provided power was "
|
||||
+ denseRankPower
|
||||
+ " (every "
|
||||
+ (int) Math.pow(2, denseRankPower)
|
||||
+ " docIDs)"
|
||||
);
|
||||
}
|
||||
this.disiTempOutput = dir.createTempOutput(data.getName(), "disi", context);
|
||||
this.skipListTempFileName = disiTempOutput.getName();
|
||||
this.origo = disiTempOutput.getFilePointer(); // All jumps are relative to the origo
|
||||
}
|
||||
|
||||
void addDocId(int doc) throws IOException {
|
||||
final int block = doc >>> 16;
|
||||
if (prevBlock != -1 && block != prevBlock) {
|
||||
// Track offset+index from previous block up to current
|
||||
jumps = addJumps(jumps, disiTempOutput.getFilePointer() - origo, totalCardinality, jumpBlockIndex, prevBlock + 1);
|
||||
jumpBlockIndex = prevBlock + 1;
|
||||
// Flush block
|
||||
flush(prevBlock, buffer, blockCardinality, denseRankPower, disiTempOutput);
|
||||
// Reset for next block
|
||||
buffer.clear();
|
||||
totalCardinality += blockCardinality;
|
||||
blockCardinality = 0;
|
||||
}
|
||||
buffer.set(doc & 0xFFFF);
|
||||
blockCardinality++;
|
||||
prevBlock = block;
|
||||
}
|
||||
|
||||
short build(IndexOutput data) throws IOException {
|
||||
if (blockCardinality > 0) {
|
||||
jumps = addJumps(jumps, disiTempOutput.getFilePointer() - origo, totalCardinality, jumpBlockIndex, prevBlock + 1);
|
||||
totalCardinality += blockCardinality;
|
||||
flush(prevBlock, buffer, blockCardinality, denseRankPower, disiTempOutput);
|
||||
buffer.clear();
|
||||
prevBlock++;
|
||||
}
|
||||
final int lastBlock = prevBlock == -1 ? 0 : prevBlock; // There will always be at least 1 block (NO_MORE_DOCS)
|
||||
// Last entry is a SPARSE with blockIndex == 32767 and the single entry 65535, which becomes the
|
||||
// docID NO_MORE_DOCS
|
||||
// To avoid creating 65K jump-table entries, only a single entry is created pointing to the
|
||||
// offset of the
|
||||
// NO_MORE_DOCS block, with the jumpBlockIndex set to the logical EMPTY block after all real
|
||||
// blocks.
|
||||
jumps = addJumps(jumps, disiTempOutput.getFilePointer() - origo, totalCardinality, lastBlock, lastBlock + 1);
|
||||
buffer.set(DocIdSetIterator.NO_MORE_DOCS & 0xFFFF);
|
||||
flush(DocIdSetIterator.NO_MORE_DOCS >>> 16, buffer, 1, denseRankPower, disiTempOutput);
|
||||
// offset+index jump-table stored at the end
|
||||
short blockCount = flushBlockJumps(jumps, lastBlock + 1, disiTempOutput);
|
||||
disiTempOutput.close();
|
||||
try (var addressDataInput = dir.openInput(skipListTempFileName, context)) {
|
||||
data.copyBytes(addressDataInput, addressDataInput.length());
|
||||
}
|
||||
return blockCount;
|
||||
}
|
||||
|
||||
// Adds entries to the offset & index jump-table for blocks
|
||||
private static int[] addJumps(int[] jumps, long offset, int index, int startBlock, int endBlock) {
|
||||
assert offset < Integer.MAX_VALUE : "Logically the offset should not exceed 2^30 but was >= Integer.MAX_VALUE";
|
||||
jumps = ArrayUtil.grow(jumps, (endBlock + 1) * 2);
|
||||
for (int b = startBlock; b < endBlock; b++) {
|
||||
jumps[b * 2] = index;
|
||||
jumps[b * 2 + 1] = (int) offset;
|
||||
}
|
||||
return jumps;
|
||||
}
|
||||
|
||||
private static void flush(int block, FixedBitSet buffer, int cardinality, byte denseRankPower, IndexOutput out) throws IOException {
|
||||
assert block >= 0 && block < BLOCK_SIZE;
|
||||
out.writeShort((short) block);
|
||||
assert cardinality > 0 && cardinality <= BLOCK_SIZE;
|
||||
out.writeShort((short) (cardinality - 1));
|
||||
if (cardinality > MAX_ARRAY_LENGTH) {
|
||||
if (cardinality != BLOCK_SIZE) { // all docs are set
|
||||
if (denseRankPower != -1) {
|
||||
final byte[] rank = createRank(buffer, denseRankPower);
|
||||
out.writeBytes(rank, rank.length);
|
||||
}
|
||||
for (long word : buffer.getBits()) {
|
||||
out.writeLong(word);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
BitSetIterator it = new BitSetIterator(buffer, cardinality);
|
||||
for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
|
||||
out.writeShort((short) doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flushes the offset & index jump-table for blocks. This should be the last data written to out
|
||||
// This method returns the blockCount for the blocks reachable for the jump_table or -1 for no
|
||||
// jump-table
|
||||
private static short flushBlockJumps(int[] jumps, int blockCount, IndexOutput out) throws IOException {
|
||||
if (blockCount == 2) { // Jumps with a single real entry + NO_MORE_DOCS is just wasted space so we ignore
|
||||
// that
|
||||
blockCount = 0;
|
||||
}
|
||||
for (int i = 0; i < blockCount; i++) {
|
||||
out.writeInt(jumps[i * 2]); // index
|
||||
out.writeInt(jumps[i * 2 + 1]); // offset
|
||||
}
|
||||
// As there are at most 32k blocks, the count is a short
|
||||
// The jumpTableOffset will be at lastPos - (blockCount * Long.BYTES)
|
||||
return (short) blockCount;
|
||||
}
|
||||
|
||||
// Creates a DENSE rank-entry (the number of set bits up to a given point) for the buffer.
|
||||
// One rank-entry for every {@code 2^denseRankPower} bits, with each rank-entry using 2 bytes.
|
||||
// Represented as a byte[] for fast flushing and mirroring of the retrieval representation.
|
||||
private static byte[] createRank(FixedBitSet buffer, byte denseRankPower) {
|
||||
final int longsPerRank = 1 << (denseRankPower - 6);
|
||||
final int rankMark = longsPerRank - 1;
|
||||
final int rankIndexShift = denseRankPower - 7; // 6 for the long (2^6) + 1 for 2 bytes/entry
|
||||
final byte[] rank = new byte[DENSE_BLOCK_LONGS >> rankIndexShift];
|
||||
final long[] bits = buffer.getBits();
|
||||
int bitCount = 0;
|
||||
for (int word = 0; word < DENSE_BLOCK_LONGS; word++) {
|
||||
if ((word & rankMark) == 0) { // Every longsPerRank longs
|
||||
rank[word >> rankIndexShift] = (byte) (bitCount >> 8);
|
||||
rank[(word >> rankIndexShift) + 1] = (byte) (bitCount & 0xFF);
|
||||
}
|
||||
bitCount += Long.bitCount(bits[word]);
|
||||
}
|
||||
return rank;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressForbidden(reason = "require usage of Lucene's IOUtils#deleteFilesIgnoringExceptions(...)")
|
||||
public void close() throws IOException {
|
||||
IOUtils.close(disiTempOutput);
|
||||
if (skipListTempFileName != null) {
|
||||
IOUtils.deleteFilesIgnoringExceptions(dir, skipListTempFileName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -29,6 +29,8 @@ import org.apache.lucene.search.SortedSetSelector;
|
|||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.ByteBuffersDataOutput;
|
||||
import org.apache.lucene.store.ByteBuffersIndexOutput;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
|
@ -54,6 +56,8 @@ import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.
|
|||
|
||||
final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
|
||||
|
||||
final Directory dir;
|
||||
final IOContext context;
|
||||
IndexOutput data, meta;
|
||||
final int maxDoc;
|
||||
private byte[] termsDictBuffer;
|
||||
|
|
@ -70,6 +74,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
|
|||
String metaExtension
|
||||
) throws IOException {
|
||||
this.termsDictBuffer = new byte[1 << 14];
|
||||
this.dir = state.directory;
|
||||
this.context = state.context;
|
||||
boolean success = false;
|
||||
try {
|
||||
final String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
|
||||
|
|
@ -138,84 +144,101 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
|
|||
meta.writeLong(numValues);
|
||||
meta.writeInt(numDocsWithValue);
|
||||
|
||||
if (numValues > 0) {
|
||||
// Special case for maxOrd of 1, signal -1 that no blocks will be written
|
||||
meta.writeInt(maxOrd != 1 ? ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT : -1);
|
||||
final ByteBuffersDataOutput indexOut = new ByteBuffersDataOutput();
|
||||
final DirectMonotonicWriter indexWriter = DirectMonotonicWriter.getInstance(
|
||||
meta,
|
||||
new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"),
|
||||
1L + ((numValues - 1) >>> ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT),
|
||||
ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT
|
||||
);
|
||||
DISIAccumulator disiAccumulator = null;
|
||||
try {
|
||||
if (numValues > 0) {
|
||||
assert numDocsWithValue > 0;
|
||||
// Special case for maxOrd of 1, signal -1 that no blocks will be written
|
||||
meta.writeInt(maxOrd != 1 ? ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT : -1);
|
||||
final ByteBuffersDataOutput indexOut = new ByteBuffersDataOutput();
|
||||
final DirectMonotonicWriter indexWriter = DirectMonotonicWriter.getInstance(
|
||||
meta,
|
||||
new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"),
|
||||
1L + ((numValues - 1) >>> ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT),
|
||||
ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT
|
||||
);
|
||||
|
||||
final long valuesDataOffset = data.getFilePointer();
|
||||
// Special case for maxOrd of 1, skip writing the blocks
|
||||
if (maxOrd != 1) {
|
||||
final long[] buffer = new long[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
|
||||
int bufferSize = 0;
|
||||
final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
|
||||
values = valuesProducer.getSortedNumeric(field);
|
||||
final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
|
||||
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
|
||||
final int count = values.docValueCount();
|
||||
for (int i = 0; i < count; ++i) {
|
||||
buffer[bufferSize++] = values.nextValue();
|
||||
if (bufferSize == ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE) {
|
||||
indexWriter.add(data.getFilePointer() - valuesDataOffset);
|
||||
if (maxOrd >= 0) {
|
||||
encoder.encodeOrdinals(buffer, data, bitsPerOrd);
|
||||
} else {
|
||||
encoder.encode(buffer, data);
|
||||
final long valuesDataOffset = data.getFilePointer();
|
||||
// Special case for maxOrd of 1, skip writing the blocks
|
||||
if (maxOrd != 1) {
|
||||
final long[] buffer = new long[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
|
||||
int bufferSize = 0;
|
||||
final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
|
||||
values = valuesProducer.getSortedNumeric(field);
|
||||
final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
|
||||
if (enableOptimizedMerge && numDocsWithValue < maxDoc) {
|
||||
disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
|
||||
}
|
||||
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
|
||||
if (disiAccumulator != null) {
|
||||
disiAccumulator.addDocId(doc);
|
||||
}
|
||||
final int count = values.docValueCount();
|
||||
for (int i = 0; i < count; ++i) {
|
||||
buffer[bufferSize++] = values.nextValue();
|
||||
if (bufferSize == ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE) {
|
||||
indexWriter.add(data.getFilePointer() - valuesDataOffset);
|
||||
if (maxOrd >= 0) {
|
||||
encoder.encodeOrdinals(buffer, data, bitsPerOrd);
|
||||
} else {
|
||||
encoder.encode(buffer, data);
|
||||
}
|
||||
bufferSize = 0;
|
||||
}
|
||||
bufferSize = 0;
|
||||
}
|
||||
}
|
||||
if (bufferSize > 0) {
|
||||
indexWriter.add(data.getFilePointer() - valuesDataOffset);
|
||||
// Fill unused slots in the block with zeroes rather than junk
|
||||
Arrays.fill(buffer, bufferSize, ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE, 0L);
|
||||
if (maxOrd >= 0) {
|
||||
encoder.encodeOrdinals(buffer, data, bitsPerOrd);
|
||||
} else {
|
||||
encoder.encode(buffer, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bufferSize > 0) {
|
||||
indexWriter.add(data.getFilePointer() - valuesDataOffset);
|
||||
// Fill unused slots in the block with zeroes rather than junk
|
||||
Arrays.fill(buffer, bufferSize, ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE, 0L);
|
||||
if (maxOrd >= 0) {
|
||||
encoder.encodeOrdinals(buffer, data, bitsPerOrd);
|
||||
} else {
|
||||
encoder.encode(buffer, data);
|
||||
}
|
||||
|
||||
final long valuesDataLength = data.getFilePointer() - valuesDataOffset;
|
||||
if (maxOrd != 1) {
|
||||
// Special case for maxOrd of 1, indexWriter isn't really used, so no need to invoke finish() method.
|
||||
indexWriter.finish();
|
||||
}
|
||||
final long indexDataOffset = data.getFilePointer();
|
||||
data.copyBytes(indexOut.toDataInput(), indexOut.size());
|
||||
meta.writeLong(indexDataOffset);
|
||||
meta.writeLong(data.getFilePointer() - indexDataOffset);
|
||||
|
||||
meta.writeLong(valuesDataOffset);
|
||||
meta.writeLong(valuesDataLength);
|
||||
}
|
||||
|
||||
final long valuesDataLength = data.getFilePointer() - valuesDataOffset;
|
||||
if (maxOrd != 1) {
|
||||
// Special case for maxOrd of 1, indexWriter isn't really used, so no need to invoke finish() method.
|
||||
indexWriter.finish();
|
||||
if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values
|
||||
meta.writeLong(-2); // docsWithFieldOffset
|
||||
meta.writeLong(0L); // docsWithFieldLength
|
||||
meta.writeShort((short) -1); // jumpTableEntryCount
|
||||
meta.writeByte((byte) -1); // denseRankPower
|
||||
} else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values
|
||||
meta.writeLong(-1); // docsWithFieldOffset
|
||||
meta.writeLong(0L); // docsWithFieldLength
|
||||
meta.writeShort((short) -1); // jumpTableEntryCount
|
||||
meta.writeByte((byte) -1); // denseRankPower
|
||||
} else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
|
||||
long offset = data.getFilePointer();
|
||||
meta.writeLong(offset); // docsWithFieldOffset
|
||||
final short jumpTableEntryCount;
|
||||
if (maxOrd != 1 && disiAccumulator != null) {
|
||||
jumpTableEntryCount = disiAccumulator.build(data);
|
||||
} else {
|
||||
values = valuesProducer.getSortedNumeric(field);
|
||||
jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
|
||||
}
|
||||
meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
|
||||
meta.writeShort(jumpTableEntryCount);
|
||||
meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
|
||||
}
|
||||
final long indexDataOffset = data.getFilePointer();
|
||||
data.copyBytes(indexOut.toDataInput(), indexOut.size());
|
||||
meta.writeLong(indexDataOffset);
|
||||
meta.writeLong(data.getFilePointer() - indexDataOffset);
|
||||
|
||||
meta.writeLong(valuesDataOffset);
|
||||
meta.writeLong(valuesDataLength);
|
||||
}
|
||||
|
||||
if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values
|
||||
meta.writeLong(-2); // docsWithFieldOffset
|
||||
meta.writeLong(0L); // docsWithFieldLength
|
||||
meta.writeShort((short) -1); // jumpTableEntryCount
|
||||
meta.writeByte((byte) -1); // denseRankPower
|
||||
} else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values
|
||||
meta.writeLong(-1); // docsWithFieldOffset
|
||||
meta.writeLong(0L); // docsWithFieldLength
|
||||
meta.writeShort((short) -1); // jumpTableEntryCount
|
||||
meta.writeByte((byte) -1); // denseRankPower
|
||||
} else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
|
||||
long offset = data.getFilePointer();
|
||||
meta.writeLong(offset); // docsWithFieldOffset
|
||||
values = valuesProducer.getSortedNumeric(field);
|
||||
final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
|
||||
meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
|
||||
meta.writeShort(jumpTableEntryCount);
|
||||
meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
|
||||
} finally {
|
||||
IOUtils.close(disiAccumulator);
|
||||
}
|
||||
|
||||
return new long[] { numDocsWithValue, numValues };
|
||||
|
|
|
|||
|
|
@ -290,6 +290,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder indexOptions(IndexOptions indexOptions) {
|
||||
this.indexOptions.setValue(indexOptions);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DenseVectorFieldMapper build(MapperBuilderContext context) {
|
||||
// Validate again here because the dimensions or element type could have been set programmatically,
|
||||
|
|
@ -1221,7 +1226,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType);
|
||||
}
|
||||
|
||||
abstract static class IndexOptions implements ToXContent {
|
||||
public abstract static class IndexOptions implements ToXContent {
|
||||
final VectorIndexType type;
|
||||
|
||||
IndexOptions(VectorIndexType type) {
|
||||
|
|
@ -1230,21 +1235,36 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
|
||||
abstract KnnVectorsFormat getVectorsFormat(ElementType elementType);
|
||||
|
||||
final void validateElementType(ElementType elementType) {
|
||||
if (type.supportsElementType(elementType) == false) {
|
||||
public boolean validate(ElementType elementType, int dim, boolean throwOnError) {
|
||||
return validateElementType(elementType, throwOnError) && validateDimension(dim, throwOnError);
|
||||
}
|
||||
|
||||
public boolean validateElementType(ElementType elementType) {
|
||||
return validateElementType(elementType, true);
|
||||
}
|
||||
|
||||
final boolean validateElementType(ElementType elementType, boolean throwOnError) {
|
||||
boolean validElementType = type.supportsElementType(elementType);
|
||||
if (throwOnError && validElementType == false) {
|
||||
throw new IllegalArgumentException(
|
||||
"[element_type] cannot be [" + elementType.toString() + "] when using index type [" + type + "]"
|
||||
);
|
||||
}
|
||||
return validElementType;
|
||||
}
|
||||
|
||||
abstract boolean updatableTo(IndexOptions update);
|
||||
|
||||
public void validateDimension(int dim) {
|
||||
if (type.supportsDimension(dim)) {
|
||||
return;
|
||||
public boolean validateDimension(int dim) {
|
||||
return validateDimension(dim, true);
|
||||
}
|
||||
|
||||
public boolean validateDimension(int dim, boolean throwOnError) {
|
||||
boolean supportsDimension = type.supportsDimension(dim);
|
||||
if (throwOnError && supportsDimension == false) {
|
||||
throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
|
||||
}
|
||||
throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
|
||||
return supportsDimension;
|
||||
}
|
||||
|
||||
abstract boolean doEquals(IndexOptions other);
|
||||
|
|
@ -1747,12 +1767,12 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
|
||||
}
|
||||
|
||||
static class Int8HnswIndexOptions extends QuantizedIndexOptions {
|
||||
public static class Int8HnswIndexOptions extends QuantizedIndexOptions {
|
||||
private final int m;
|
||||
private final int efConstruction;
|
||||
private final Float confidenceInterval;
|
||||
|
||||
Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) {
|
||||
public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) {
|
||||
super(VectorIndexType.INT8_HNSW, rescoreVector);
|
||||
this.m = m;
|
||||
this.efConstruction = efConstruction;
|
||||
|
|
@ -1890,11 +1910,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
}
|
||||
}
|
||||
|
||||
static class BBQHnswIndexOptions extends QuantizedIndexOptions {
|
||||
public static class BBQHnswIndexOptions extends QuantizedIndexOptions {
|
||||
private final int m;
|
||||
private final int efConstruction;
|
||||
|
||||
BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) {
|
||||
public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) {
|
||||
super(VectorIndexType.BBQ_HNSW, rescoreVector);
|
||||
this.m = m;
|
||||
this.efConstruction = efConstruction;
|
||||
|
|
@ -1936,11 +1956,14 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void validateDimension(int dim) {
|
||||
if (type.supportsDimension(dim)) {
|
||||
return;
|
||||
public boolean validateDimension(int dim, boolean throwOnError) {
|
||||
boolean supportsDimension = type.supportsDimension(dim);
|
||||
if (throwOnError && supportsDimension == false) {
|
||||
throw new IllegalArgumentException(
|
||||
type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
|
||||
);
|
||||
}
|
||||
throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
|
||||
return supportsDimension;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1984,15 +2007,19 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void validateDimension(int dim) {
|
||||
if (type.supportsDimension(dim)) {
|
||||
return;
|
||||
public boolean validateDimension(int dim, boolean throwOnError) {
|
||||
boolean supportsDimension = type.supportsDimension(dim);
|
||||
if (throwOnError && supportsDimension == false) {
|
||||
throw new IllegalArgumentException(
|
||||
type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
|
||||
);
|
||||
}
|
||||
throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
|
||||
return supportsDimension;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
record RescoreVector(float oversample) implements ToXContentObject {
|
||||
public record RescoreVector(float oversample) implements ToXContentObject {
|
||||
static final String NAME = "rescore_vector";
|
||||
static final String OVERSAMPLE = "oversample";
|
||||
|
||||
|
|
@ -2311,6 +2338,10 @@ public class DenseVectorFieldMapper extends FieldMapper {
|
|||
ElementType getElementType() {
|
||||
return elementType;
|
||||
}
|
||||
|
||||
public IndexOptions getIndexOptions() {
|
||||
return indexOptions;
|
||||
}
|
||||
}
|
||||
|
||||
private final IndexOptions indexOptions;
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ public class InboundAggregator implements Releasable {
|
|||
checkBreaker(aggregated.getHeader(), aggregated.getContentLength(), breakerControl);
|
||||
}
|
||||
if (isShortCircuited()) {
|
||||
aggregated.decRef();
|
||||
aggregated.close();
|
||||
success = true;
|
||||
return new InboundMessage(aggregated.getHeader(), aggregationException);
|
||||
} else {
|
||||
|
|
@ -131,7 +131,7 @@ public class InboundAggregator implements Releasable {
|
|||
} finally {
|
||||
resetCurrentAggregation();
|
||||
if (success == false) {
|
||||
aggregated.decRef();
|
||||
aggregated.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import org.elasticsearch.common.network.HandlingTimeTracker;
|
|||
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
|
||||
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
||||
import org.elasticsearch.common.util.concurrent.ThreadContext;
|
||||
import org.elasticsearch.core.Releasable;
|
||||
import org.elasticsearch.core.Releasables;
|
||||
import org.elasticsearch.core.TimeValue;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
|
|
@ -87,21 +86,31 @@ public class InboundHandler {
|
|||
this.slowLogThresholdMs = slowLogThreshold.getMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message the transport message received, guaranteed to be closed by this method if it returns without exception.
|
||||
* Callers must ensure that {@code message} is closed if this method throws an exception but must not release
|
||||
* the message themselves otherwise
|
||||
*/
|
||||
void inboundMessage(TcpChannel channel, InboundMessage message) throws Exception {
|
||||
final long startTime = threadPool.rawRelativeTimeInMillis();
|
||||
channel.getChannelStats().markAccessed(startTime);
|
||||
TransportLogger.logInboundMessage(channel, message);
|
||||
|
||||
if (message.isPing()) {
|
||||
keepAlive.receiveKeepAlive(channel);
|
||||
keepAlive.receiveKeepAlive(channel); // pings hold no resources, no need to close
|
||||
} else {
|
||||
messageReceived(channel, message, startTime);
|
||||
messageReceived(channel, /* autocloses absent exception */ message, startTime);
|
||||
}
|
||||
}
|
||||
|
||||
// Empty stream constant to avoid instantiating a new stream for empty messages.
|
||||
private static final StreamInput EMPTY_STREAM_INPUT = new ByteBufferStreamInput(ByteBuffer.wrap(BytesRef.EMPTY_BYTES));
|
||||
|
||||
/**
|
||||
* @param message the transport message received, guaranteed to be closed by this method if it returns without exception.
|
||||
* Callers must ensure that {@code message} is closed if this method throws an exception but must not release
|
||||
* the message themselves otherwise
|
||||
*/
|
||||
private void messageReceived(TcpChannel channel, InboundMessage message, long startTime) throws IOException {
|
||||
final InetSocketAddress remoteAddress = channel.getRemoteAddress();
|
||||
final Header header = message.getHeader();
|
||||
|
|
@ -115,14 +124,16 @@ public class InboundHandler {
|
|||
threadContext.setHeaders(header.getHeaders());
|
||||
threadContext.putTransient("_remote_address", remoteAddress);
|
||||
if (header.isRequest()) {
|
||||
handleRequest(channel, message);
|
||||
handleRequest(channel, /* autocloses absent exception */ message);
|
||||
} else {
|
||||
// Responses do not support short circuiting currently
|
||||
assert message.isShortCircuit() == false;
|
||||
responseHandler = findResponseHandler(header);
|
||||
// ignore if its null, the service logs it
|
||||
if (responseHandler != null) {
|
||||
executeResponseHandler(message, responseHandler, remoteAddress);
|
||||
executeResponseHandler( /* autocloses absent exception */ message, responseHandler, remoteAddress);
|
||||
} else {
|
||||
message.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -135,6 +146,11 @@ public class InboundHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message the transport message received, guaranteed to be closed by this method if it returns without exception.
|
||||
* Callers must ensure that {@code message} is closed if this method throws an exception but must not release
|
||||
* the message themselves otherwise
|
||||
*/
|
||||
private void executeResponseHandler(
|
||||
InboundMessage message,
|
||||
TransportResponseHandler<?> responseHandler,
|
||||
|
|
@ -145,13 +161,13 @@ public class InboundHandler {
|
|||
final StreamInput streamInput = namedWriteableStream(message.openOrGetStreamInput());
|
||||
assert assertRemoteVersion(streamInput, header.getVersion());
|
||||
if (header.isError()) {
|
||||
handlerResponseError(streamInput, message, responseHandler);
|
||||
handlerResponseError(streamInput, /* autocloses */ message, responseHandler);
|
||||
} else {
|
||||
handleResponse(remoteAddress, streamInput, responseHandler, message);
|
||||
handleResponse(remoteAddress, streamInput, responseHandler, /* autocloses */ message);
|
||||
}
|
||||
} else {
|
||||
assert header.isError() == false;
|
||||
handleResponse(remoteAddress, EMPTY_STREAM_INPUT, responseHandler, message);
|
||||
handleResponse(remoteAddress, EMPTY_STREAM_INPUT, responseHandler, /* autocloses */ message);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -220,10 +236,15 @@ public class InboundHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message the transport message received, guaranteed to be closed by this method if it returns without exception.
|
||||
* Callers must ensure that {@code message} is closed if this method throws an exception but must not release
|
||||
* the message themselves otherwise
|
||||
*/
|
||||
private <T extends TransportRequest> void handleRequest(TcpChannel channel, InboundMessage message) throws IOException {
|
||||
final Header header = message.getHeader();
|
||||
if (header.isHandshake()) {
|
||||
handleHandshakeRequest(channel, message);
|
||||
handleHandshakeRequest(channel, /* autocloses */ message);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -257,9 +278,11 @@ public class InboundHandler {
|
|||
assert reg != null;
|
||||
final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput());
|
||||
assert assertRemoteVersion(stream, header.getVersion());
|
||||
final T request;
|
||||
T request;
|
||||
try {
|
||||
request = reg.newRequest(stream);
|
||||
message.close();
|
||||
message = null;
|
||||
} catch (Exception e) {
|
||||
assert ignoreDeserializationErrors : e;
|
||||
throw e;
|
||||
|
|
@ -274,13 +297,20 @@ public class InboundHandler {
|
|||
doHandleRequest(reg, request, transportChannel);
|
||||
}
|
||||
} else {
|
||||
handleRequestForking(request, reg, transportChannel);
|
||||
handleRequestForking(/* autocloses */ request, reg, transportChannel);
|
||||
request = null; // now owned by the thread we forked to
|
||||
}
|
||||
} finally {
|
||||
request.decRef();
|
||||
if (request != null) {
|
||||
request.decRef();
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
sendErrorResponse(action, transportChannel, e);
|
||||
} finally {
|
||||
if (message != null) {
|
||||
message.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -294,7 +324,6 @@ public class InboundHandler {
|
|||
|
||||
private <T extends TransportRequest> void handleRequestForking(T request, RequestHandlerRegistry<T> reg, TransportChannel channel) {
|
||||
boolean success = false;
|
||||
request.mustIncRef();
|
||||
try {
|
||||
reg.getExecutor().execute(threadPool.getThreadContext().preserveContextWithTracing(new AbstractRunnable() {
|
||||
@Override
|
||||
|
|
@ -331,6 +360,9 @@ public class InboundHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message guaranteed to get closed by this method
|
||||
*/
|
||||
private void handleHandshakeRequest(TcpChannel channel, InboundMessage message) throws IOException {
|
||||
var header = message.getHeader();
|
||||
assert header.actionName.equals(TransportHandshaker.HANDSHAKE_ACTION_NAME);
|
||||
|
|
@ -351,7 +383,7 @@ public class InboundHandler {
|
|||
true,
|
||||
Releasables.assertOnce(message.takeBreakerReleaseControl())
|
||||
);
|
||||
try {
|
||||
try (message) {
|
||||
handshaker.handleHandshake(transportChannel, requestId, stream);
|
||||
} catch (Exception e) {
|
||||
logger.warn(
|
||||
|
|
@ -371,29 +403,30 @@ public class InboundHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message guaranteed to get closed by this method
|
||||
*/
|
||||
private <T extends TransportResponse> void handleResponse(
|
||||
InetSocketAddress remoteAddress,
|
||||
final StreamInput stream,
|
||||
final TransportResponseHandler<T> handler,
|
||||
final InboundMessage inboundMessage
|
||||
final InboundMessage message
|
||||
) {
|
||||
final var executor = handler.executor();
|
||||
if (executor == EsExecutors.DIRECT_EXECUTOR_SERVICE) {
|
||||
// no need to provide a buffer release here, we never escape the buffer when handling directly
|
||||
doHandleResponse(handler, remoteAddress, stream, inboundMessage.getHeader(), () -> {});
|
||||
doHandleResponse(handler, remoteAddress, stream, /* autocloses */ message);
|
||||
} else {
|
||||
inboundMessage.mustIncRef();
|
||||
// release buffer once we deserialize the message, but have a fail-safe in #onAfter below in case that didn't work out
|
||||
final Releasable releaseBuffer = Releasables.releaseOnce(inboundMessage::decRef);
|
||||
executor.execute(new ForkingResponseHandlerRunnable(handler, null) {
|
||||
@Override
|
||||
protected void doRun() {
|
||||
doHandleResponse(handler, remoteAddress, stream, inboundMessage.getHeader(), releaseBuffer);
|
||||
doHandleResponse(handler, remoteAddress, stream, /* autocloses */ message);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onAfter() {
|
||||
Releasables.closeExpectNoException(releaseBuffer);
|
||||
message.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -404,20 +437,19 @@ public class InboundHandler {
|
|||
* @param handler response handler
|
||||
* @param remoteAddress remote address that the message was sent from
|
||||
* @param stream bytes stream for reading the message
|
||||
* @param header message header
|
||||
* @param releaseResponseBuffer releasable that will be released once the message has been read from the {@code stream}
|
||||
* @param inboundMessage inbound message, guaranteed to get closed by this method
|
||||
* @param <T> response message type
|
||||
*/
|
||||
private <T extends TransportResponse> void doHandleResponse(
|
||||
TransportResponseHandler<T> handler,
|
||||
InetSocketAddress remoteAddress,
|
||||
final StreamInput stream,
|
||||
final Header header,
|
||||
Releasable releaseResponseBuffer
|
||||
InboundMessage inboundMessage
|
||||
) {
|
||||
final T response;
|
||||
try (releaseResponseBuffer) {
|
||||
try (inboundMessage) {
|
||||
response = handler.read(stream);
|
||||
verifyResponseReadFully(inboundMessage.getHeader(), handler, stream);
|
||||
} catch (Exception e) {
|
||||
final TransportException serializationException = new TransportSerializationException(
|
||||
"Failed to deserialize response from handler [" + handler + "]",
|
||||
|
|
@ -429,7 +461,6 @@ public class InboundHandler {
|
|||
return;
|
||||
}
|
||||
try {
|
||||
verifyResponseReadFully(header, handler, stream);
|
||||
handler.handleResponse(response);
|
||||
} catch (Exception e) {
|
||||
doHandleException(handler, new ResponseHandlerFailureTransportException(e));
|
||||
|
|
@ -438,9 +469,12 @@ public class InboundHandler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message guaranteed to get closed by this method
|
||||
*/
|
||||
private void handlerResponseError(StreamInput stream, InboundMessage message, final TransportResponseHandler<?> handler) {
|
||||
Exception error;
|
||||
try {
|
||||
try (message) {
|
||||
error = stream.readException();
|
||||
verifyResponseReadFully(message.getHeader(), handler, stream);
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -12,14 +12,15 @@ package org.elasticsearch.transport;
|
|||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.common.bytes.ReleasableBytesReference;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.core.AbstractRefCounted;
|
||||
import org.elasticsearch.core.IOUtils;
|
||||
import org.elasticsearch.core.Releasable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.lang.invoke.VarHandle;
|
||||
import java.util.Objects;
|
||||
|
||||
public class InboundMessage extends AbstractRefCounted {
|
||||
public class InboundMessage implements Releasable {
|
||||
|
||||
private final Header header;
|
||||
private final ReleasableBytesReference content;
|
||||
|
|
@ -28,6 +29,19 @@ public class InboundMessage extends AbstractRefCounted {
|
|||
private Releasable breakerRelease;
|
||||
private StreamInput streamInput;
|
||||
|
||||
@SuppressWarnings("unused") // updated via CLOSED (and _only_ via CLOSED)
|
||||
private boolean closed;
|
||||
|
||||
private static final VarHandle CLOSED;
|
||||
|
||||
static {
|
||||
try {
|
||||
CLOSED = MethodHandles.lookup().findVarHandle(InboundMessage.class, "closed", boolean.class);
|
||||
} catch (Exception e) {
|
||||
throw new ExceptionInInitializerError(e);
|
||||
}
|
||||
}
|
||||
|
||||
public InboundMessage(Header header, ReleasableBytesReference content, Releasable breakerRelease) {
|
||||
this.header = header;
|
||||
this.content = content;
|
||||
|
|
@ -84,7 +98,7 @@ public class InboundMessage extends AbstractRefCounted {
|
|||
|
||||
public StreamInput openOrGetStreamInput() throws IOException {
|
||||
assert isPing == false && content != null;
|
||||
assert hasReferences();
|
||||
assert (boolean) CLOSED.getAcquire(this) == false;
|
||||
if (streamInput == null) {
|
||||
streamInput = content.streamInput();
|
||||
streamInput.setTransportVersion(header.getVersion());
|
||||
|
|
@ -98,7 +112,10 @@ public class InboundMessage extends AbstractRefCounted {
|
|||
}
|
||||
|
||||
@Override
|
||||
protected void closeInternal() {
|
||||
public void close() {
|
||||
if (CLOSED.compareAndSet(this, false, true) == false) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
IOUtils.close(streamInput, content, breakerRelease);
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -112,13 +112,8 @@ public class InboundPipeline implements Releasable {
|
|||
messageHandler.accept(channel, PING_MESSAGE);
|
||||
} else if (fragment == InboundDecoder.END_CONTENT) {
|
||||
assert aggregator.isAggregating();
|
||||
InboundMessage aggregated = aggregator.finishAggregation();
|
||||
try {
|
||||
statsTracker.markMessageReceived();
|
||||
messageHandler.accept(channel, aggregated);
|
||||
} finally {
|
||||
aggregated.decRef();
|
||||
}
|
||||
statsTracker.markMessageReceived();
|
||||
messageHandler.accept(channel, /* autocloses */ aggregator.finishAggregation());
|
||||
} else {
|
||||
assert aggregator.isAggregating();
|
||||
assert fragment instanceof ReleasableBytesReference;
|
||||
|
|
|
|||
|
|
@ -813,9 +813,14 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
|
|||
*/
|
||||
public void inboundMessage(TcpChannel channel, InboundMessage message) {
|
||||
try {
|
||||
inboundHandler.inboundMessage(channel, message);
|
||||
inboundHandler.inboundMessage(channel, /* autocloses absent exception */ message);
|
||||
message = null;
|
||||
} catch (Exception e) {
|
||||
onException(channel, e);
|
||||
} finally {
|
||||
if (message != null) {
|
||||
message.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
package org.elasticsearch.action.admin.cluster.snapshots.status;
|
||||
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.support.ActionFilters;
|
||||
import org.elasticsearch.client.internal.node.NodeClient;
|
||||
import org.elasticsearch.cluster.SnapshotsInProgress;
|
||||
import org.elasticsearch.cluster.metadata.ProjectId;
|
||||
import org.elasticsearch.cluster.service.ClusterService;
|
||||
import org.elasticsearch.index.IndexVersion;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
import org.elasticsearch.repositories.IndexId;
|
||||
import org.elasticsearch.repositories.RepositoriesService;
|
||||
import org.elasticsearch.repositories.ShardGeneration;
|
||||
import org.elasticsearch.snapshots.Snapshot;
|
||||
import org.elasticsearch.snapshots.SnapshotId;
|
||||
import org.elasticsearch.tasks.CancellableTask;
|
||||
import org.elasticsearch.tasks.TaskCancelHelper;
|
||||
import org.elasticsearch.tasks.TaskCancelledException;
|
||||
import org.elasticsearch.test.ClusterServiceUtils;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.test.transport.CapturingTransport;
|
||||
import org.elasticsearch.threadpool.TestThreadPool;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.elasticsearch.transport.TransportService;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class TransportSnapshotsStatusActionTests extends ESTestCase {
|
||||
|
||||
private ThreadPool threadPool;
|
||||
private ClusterService clusterService;
|
||||
private TransportService transportService;
|
||||
private RepositoriesService repositoriesService;
|
||||
private TransportSnapshotsStatusAction action;
|
||||
|
||||
@Before
|
||||
public void initializeComponents() throws Exception {
|
||||
threadPool = new TestThreadPool(TransportSnapshotsStatusActionTests.class.getName());
|
||||
clusterService = ClusterServiceUtils.createClusterService(threadPool);
|
||||
transportService = new CapturingTransport().createTransportService(
|
||||
clusterService.getSettings(),
|
||||
threadPool,
|
||||
TransportService.NOOP_TRANSPORT_INTERCEPTOR,
|
||||
address -> clusterService.localNode(),
|
||||
clusterService.getClusterSettings(),
|
||||
Set.of()
|
||||
);
|
||||
final var nodeClient = new NodeClient(clusterService.getSettings(), threadPool);
|
||||
repositoriesService = new RepositoriesService(
|
||||
clusterService.getSettings(),
|
||||
clusterService,
|
||||
Map.of(),
|
||||
Map.of(),
|
||||
threadPool,
|
||||
nodeClient,
|
||||
List.of()
|
||||
);
|
||||
action = new TransportSnapshotsStatusAction(
|
||||
transportService,
|
||||
clusterService,
|
||||
threadPool,
|
||||
repositoriesService,
|
||||
nodeClient,
|
||||
new ActionFilters(Set.of())
|
||||
);
|
||||
}
|
||||
|
||||
@After
|
||||
public void shutdownComponents() throws Exception {
|
||||
threadPool.shutdown();
|
||||
repositoriesService.close();
|
||||
transportService.close();
|
||||
clusterService.close();
|
||||
}
|
||||
|
||||
public void testBuildResponseDetectsTaskIsCancelledWhileProcessingCurrentSnapshotEntries() throws Exception {
|
||||
runBasicBuildResponseTest(true);
|
||||
}
|
||||
|
||||
public void testBuildResponseInvokesListenerWithResponseWhenTaskIsNotCancelled() throws Exception {
|
||||
runBasicBuildResponseTest(false);
|
||||
}
|
||||
|
||||
private void runBasicBuildResponseTest(boolean shouldCancelTask) {
|
||||
final var expectedSnapshot = new Snapshot(ProjectId.DEFAULT, "test-repo", new SnapshotId("snapshot", "uuid"));
|
||||
final var expectedState = SnapshotsInProgress.State.STARTED;
|
||||
final var indexName = "test-index-name";
|
||||
final var indexUuid = "test-index-uuid";
|
||||
final var currentSnapshotEntries = List.of(
|
||||
SnapshotsInProgress.Entry.snapshot(
|
||||
expectedSnapshot,
|
||||
randomBoolean(),
|
||||
randomBoolean(),
|
||||
SnapshotsInProgress.State.STARTED,
|
||||
Map.of(indexName, new IndexId(indexName, indexUuid)),
|
||||
List.of(),
|
||||
List.of(),
|
||||
randomNonNegativeLong(),
|
||||
randomNonNegativeLong(),
|
||||
Map.of(
|
||||
new ShardId(indexName, indexUuid, 0),
|
||||
new SnapshotsInProgress.ShardSnapshotStatus("node", new ShardGeneration("gen"))
|
||||
),
|
||||
null,
|
||||
Map.of(),
|
||||
IndexVersion.current()
|
||||
)
|
||||
);
|
||||
final var nodeSnapshotStatuses = new TransportNodesSnapshotsStatus.NodesSnapshotStatus(
|
||||
clusterService.getClusterName(),
|
||||
List.of(),
|
||||
List.of()
|
||||
);
|
||||
|
||||
// Run some sanity checks for when the task is not cancelled and we get back a response object.
|
||||
// Note that thorough verification of the SnapshotsStatusResponse is done in the higher level SnapshotStatus API integration tests.
|
||||
final Consumer<SnapshotsStatusResponse> verifyResponse = rsp -> {
|
||||
assertNotNull(rsp);
|
||||
final var snapshotStatuses = rsp.getSnapshots();
|
||||
assertNotNull(snapshotStatuses);
|
||||
assertEquals(
|
||||
"expected 1 snapshot status, got " + snapshotStatuses.size() + ": " + snapshotStatuses,
|
||||
1,
|
||||
snapshotStatuses.size()
|
||||
);
|
||||
final var snapshotStatus = snapshotStatuses.getFirst();
|
||||
assertNotNull(snapshotStatus.getSnapshot());
|
||||
assertEquals(expectedSnapshot, snapshotStatus.getSnapshot());
|
||||
assertEquals(expectedState, snapshotStatus.getState());
|
||||
final var snapshotStatusShards = snapshotStatus.getShards();
|
||||
assertNotNull(snapshotStatusShards);
|
||||
assertEquals(
|
||||
"expected 1 index shard status, got " + snapshotStatusShards.size() + ": " + snapshotStatusShards,
|
||||
1,
|
||||
snapshotStatusShards.size()
|
||||
);
|
||||
final var snapshotStatusIndices = snapshotStatus.getIndices();
|
||||
assertNotNull(snapshotStatusIndices);
|
||||
assertEquals(
|
||||
"expected 1 entry in snapshotStatusIndices, got " + snapshotStatusIndices.size() + ": " + snapshotStatusIndices,
|
||||
1,
|
||||
snapshotStatusIndices.size()
|
||||
);
|
||||
assertTrue(
|
||||
"no entry for indexName [" + indexName + "] found in snapshotStatusIndices keyset " + snapshotStatusIndices.keySet(),
|
||||
snapshotStatusIndices.containsKey(indexName)
|
||||
);
|
||||
assertNotNull(snapshotStatus.getShardsStats());
|
||||
};
|
||||
|
||||
final var listener = new ActionListener<SnapshotsStatusResponse>() {
|
||||
@Override
|
||||
public void onResponse(SnapshotsStatusResponse rsp) {
|
||||
if (shouldCancelTask) {
|
||||
fail("expected detection of task cancellation and onFailure() instead of onResponse(" + rsp + ")");
|
||||
} else {
|
||||
verifyResponse.accept(rsp);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Exception e) {
|
||||
if (shouldCancelTask) {
|
||||
assertTrue(e instanceof TaskCancelledException);
|
||||
} else {
|
||||
fail("expected onResponse() instead of onFailure(" + e + ")");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
final var listenerInvoked = new AtomicBoolean(false);
|
||||
final var cancellableTask = new CancellableTask(randomLong(), "type", "action", "desc", null, Map.of());
|
||||
|
||||
if (shouldCancelTask) {
|
||||
TaskCancelHelper.cancel(cancellableTask, "simulated cancellation");
|
||||
}
|
||||
|
||||
action.buildResponse(
|
||||
SnapshotsInProgress.EMPTY,
|
||||
new SnapshotsStatusRequest(TEST_REQUEST_TIMEOUT),
|
||||
currentSnapshotEntries,
|
||||
nodeSnapshotStatuses,
|
||||
cancellableTask,
|
||||
ActionListener.runAfter(listener, () -> listenerInvoked.set(true))
|
||||
);
|
||||
assertTrue("Expected listener to be invoked", listenerInvoked.get());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,518 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
package org.elasticsearch.index.codec.tsdb.es819;
|
||||
|
||||
import org.apache.lucene.codecs.lucene90.IndexedDISI;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||
import org.apache.lucene.tests.util.TestUtil;
|
||||
import org.apache.lucene.util.BitSet;
|
||||
import org.apache.lucene.util.BitSetIterator;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.SparseFixedBitSet;
|
||||
import org.elasticsearch.core.SuppressForbidden;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Random;
|
||||
|
||||
// Copied from org.apache.lucene.codecs.lucene90.TestIndexedDISI and kept tests that we can run.
|
||||
// The test suite has been modified to write jump table using writeJumpTable(...) in this class.
|
||||
// (some original tests require access to package protected constructor of IndexedDISI and was removed)
|
||||
public class DISIAccumulatorTests extends LuceneTestCase {
|
||||
|
||||
public void testEmpty() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
// EMPTY blocks are special with regard to jumps as they have size 0
|
||||
public void testEmptyBlocks() throws IOException {
|
||||
final int B = 65536;
|
||||
int maxDoc = B * 11;
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
// block 0: EMPTY
|
||||
set.set(B + 5); // block 1: SPARSE
|
||||
// block 2: EMPTY
|
||||
// block 3: EMPTY
|
||||
set.set(B * 4 + 5); // block 4: SPARSE
|
||||
|
||||
for (int i = 0; i < B; i++) {
|
||||
set.set(B * 6 + i); // block 6: ALL
|
||||
}
|
||||
for (int i = 0; i < B; i += 3) {
|
||||
set.set(B * 7 + i); // block 7: DENSE
|
||||
}
|
||||
for (int i = 0; i < B; i++) {
|
||||
if (i != 32768) {
|
||||
set.set(B * 8 + i); // block 8: DENSE (all-1)
|
||||
}
|
||||
}
|
||||
// block 9-11: EMPTY
|
||||
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTestAllSingleJump(set, dir);
|
||||
}
|
||||
|
||||
// Change the first block to DENSE to see if jump-tables sets to position 0
|
||||
set.set(0);
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTestAllSingleJump(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
// EMPTY blocks are special with regard to jumps as they have size 0
|
||||
public void testLastEmptyBlocks() throws IOException {
|
||||
final int B = 65536;
|
||||
int maxDoc = B * 3;
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
for (int docID = 0; docID < B * 2; docID++) { // first 2 blocks are ALL
|
||||
set.set(docID);
|
||||
}
|
||||
// Last block is EMPTY
|
||||
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTestAllSingleJump(set, dir);
|
||||
assertAdvanceBeyondEnd(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
// Checks that advance after the end of the blocks has been reached has the correct behaviour
|
||||
private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
|
||||
final int cardinality = set.cardinality();
|
||||
final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
|
||||
int jumpTableentryCount;
|
||||
try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
|
||||
jumpTableentryCount = writeJumpTable(set, dir, out, denseRankPower);
|
||||
}
|
||||
|
||||
try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
|
||||
BitSetIterator disi2 = new BitSetIterator(set, cardinality);
|
||||
int doc = disi2.docID();
|
||||
int index = 0;
|
||||
while (doc < cardinality) {
|
||||
doc = disi2.nextDoc();
|
||||
index++;
|
||||
}
|
||||
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
|
||||
// Advance 1 docID beyond end
|
||||
assertFalse("There should be no set bit beyond the valid docID range", disi.advanceExact(set.length()));
|
||||
disi.advance(doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
|
||||
// disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
|
||||
assertEquals("The index when advancing beyond the last defined docID should be correct", index, disi.index() + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: can this be toned down?
|
||||
public void testRandomBlocks() throws IOException {
|
||||
final int BLOCKS = 5;
|
||||
BitSet set = createSetWithRandomBlocks(BLOCKS);
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTestAllSingleJump(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
private BitSet createSetWithRandomBlocks(int blockCount) {
|
||||
final int B = 65536;
|
||||
BitSet set = new SparseFixedBitSet(blockCount * B);
|
||||
for (int block = 0; block < blockCount; block++) {
|
||||
switch (random().nextInt(4)) {
|
||||
case 0: { // EMPTY
|
||||
break;
|
||||
}
|
||||
case 1: { // ALL
|
||||
for (int docID = block * B; docID < (block + 1) * B; docID++) {
|
||||
set.set(docID);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: { // SPARSE ( < 4096 )
|
||||
for (int docID = block * B; docID < (block + 1) * B; docID += 101) {
|
||||
set.set(docID);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: { // DENSE ( >= 4096 )
|
||||
for (int docID = block * B; docID < (block + 1) * B; docID += 3) {
|
||||
set.set(docID);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new IllegalStateException("Modulo logic error: there should only be 4 possibilities");
|
||||
}
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {
|
||||
final int cardinality = set.cardinality();
|
||||
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
|
||||
long length;
|
||||
int jumpTableentryCount;
|
||||
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
|
||||
jumpTableentryCount = writeJumpTable(set, dir, out, denseRankPower);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
for (int i = 0; i < set.length(); i++) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
assertEquals("The bit at " + i + " should be correct with advanceExact", set.get(i), disi.advanceExact(i));
|
||||
|
||||
IndexedDISI disi2 = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
disi2.advance(i);
|
||||
// Proper sanity check with jump tables as an error could make them seek backwards
|
||||
assertTrue("The docID should at least be " + i + " after advance(" + i + ") but was " + disi2.docID(), i <= disi2.docID());
|
||||
if (set.get(i)) {
|
||||
assertEquals("The docID should be present with advance", i, disi2.docID());
|
||||
} else {
|
||||
assertNotSame("The docID should not be present with advance", i, disi2.docID());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testOneDoc() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
set.set(random().nextInt(maxDoc));
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testTwoDocs() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
set.set(random().nextInt(maxDoc));
|
||||
set.set(random().nextInt(maxDoc));
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testAllDocs() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
set.set(1, maxDoc);
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testHalfFull() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
BitSet set = new SparseFixedBitSet(maxDoc);
|
||||
for (int i = random().nextInt(2); i < maxDoc; i += TestUtil.nextInt(random(), 1, 3)) {
|
||||
set.set(i);
|
||||
}
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testDocRange() throws IOException {
|
||||
try (Directory dir = newDirectory()) {
|
||||
for (int iter = 0; iter < 10; ++iter) {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 1000000);
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
final int start = random().nextInt(maxDoc);
|
||||
final int end = TestUtil.nextInt(random(), start + 1, maxDoc);
|
||||
set.set(start, end);
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testSparseDenseBoundary() throws IOException, NoSuchFieldException, IllegalAccessException {
|
||||
try (Directory dir = newDirectory()) {
|
||||
FixedBitSet set = new FixedBitSet(200000);
|
||||
int start = 65536 + random().nextInt(100);
|
||||
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
|
||||
|
||||
// we set MAX_ARRAY_LENGTH bits so the encoding will be sparse
|
||||
set.set(start, start + DISIAccumulator.MAX_ARRAY_LENGTH);
|
||||
long length;
|
||||
int jumpTableEntryCount;
|
||||
try (IndexOutput out = dir.createOutput("sparse", IOContext.DEFAULT)) {
|
||||
jumpTableEntryCount = writeJumpTable(set, DISIAccumulator.MAX_ARRAY_LENGTH, dir, out, denseRankPower);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
try (IndexInput in = dir.openInput("sparse", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPower, DISIAccumulator.MAX_ARRAY_LENGTH);
|
||||
assertEquals(start, disi.nextDoc());
|
||||
if (System.getSecurityManager() == null) {
|
||||
assertEquals("SPARSE", getMethodFromDISI(disi));
|
||||
}
|
||||
}
|
||||
doTest(set, dir);
|
||||
|
||||
// now we set one more bit so the encoding will be dense
|
||||
set.set(start + DISIAccumulator.MAX_ARRAY_LENGTH + random().nextInt(100));
|
||||
try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
|
||||
writeJumpTable(set, DISIAccumulator.MAX_ARRAY_LENGTH, dir, out, denseRankPower);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(
|
||||
in,
|
||||
0L,
|
||||
length,
|
||||
jumpTableEntryCount,
|
||||
denseRankPower,
|
||||
DISIAccumulator.MAX_ARRAY_LENGTH + 1
|
||||
);
|
||||
assertEquals(start, disi.nextDoc());
|
||||
if (System.getSecurityManager() == null) {
|
||||
assertEquals("DENSE", getMethodFromDISI(disi));
|
||||
}
|
||||
}
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressForbidden(reason = "access violation required in order to read private field for this test")
|
||||
private static String getMethodFromDISI(Object o) throws NoSuchFieldException, IllegalAccessException {
|
||||
var field = IndexedDISI.class.getDeclaredField("method");
|
||||
field.setAccessible(true);
|
||||
return field.get(o).toString();
|
||||
}
|
||||
|
||||
public void testOneDocMissing() throws IOException {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 1000000);
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
set.set(0, maxDoc);
|
||||
set.clear(random().nextInt(maxDoc));
|
||||
try (Directory dir = newDirectory()) {
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testFewMissingDocs() throws IOException {
|
||||
try (Directory dir = newDirectory()) {
|
||||
int numIters = atLeast(10);
|
||||
for (int iter = 0; iter < numIters; ++iter) {
|
||||
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
set.set(0, maxDoc);
|
||||
final int numMissingDocs = TestUtil.nextInt(random(), 2, 1000);
|
||||
for (int i = 0; i < numMissingDocs; ++i) {
|
||||
set.clear(random().nextInt(maxDoc));
|
||||
}
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testDenseMultiBlock() throws IOException {
|
||||
try (Directory dir = newDirectory()) {
|
||||
int maxDoc = 10 * 65536; // 10 blocks
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense
|
||||
set.set(i);
|
||||
}
|
||||
doTest(set, dir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testIllegalDenseRankPower() throws IOException {
|
||||
|
||||
// Legal values
|
||||
for (byte denseRankPower : new byte[] { -1, 7, 8, 9, 10, 11, 12, 13, 14, 15 }) {
|
||||
createAndOpenDISI(denseRankPower, denseRankPower);
|
||||
}
|
||||
|
||||
// Illegal values
|
||||
for (byte denseRankPower : new byte[] { -2, 0, 1, 6, 16 }) {
|
||||
expectThrows(IllegalArgumentException.class, () -> {
|
||||
createAndOpenDISI(denseRankPower, (byte) 8); // Illegal write, legal read (should not reach read)
|
||||
});
|
||||
|
||||
expectThrows(IllegalArgumentException.class, () -> {
|
||||
createAndOpenDISI((byte) 8, denseRankPower); // Legal write, illegal read (should reach read)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void createAndOpenDISI(byte denseRankPowerWrite, byte denseRankPowerRead) throws IOException {
|
||||
BitSet set = new FixedBitSet(10);
|
||||
set.set(set.length() - 1);
|
||||
try (Directory dir = newDirectory()) {
|
||||
long length;
|
||||
int jumpTableEntryCount = -1;
|
||||
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
|
||||
jumpTableEntryCount = writeJumpTable(set, dir, out, denseRankPowerWrite);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPowerRead, set.cardinality());
|
||||
}
|
||||
// This tests the legality of the denseRankPower only, so we don't do anything with the disi
|
||||
}
|
||||
}
|
||||
|
||||
public void testOneDocMissingFixed() throws IOException {
|
||||
int maxDoc = 9699;
|
||||
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
|
||||
FixedBitSet set = new FixedBitSet(maxDoc);
|
||||
set.set(0, maxDoc);
|
||||
set.clear(1345);
|
||||
try (Directory dir = newDirectory()) {
|
||||
|
||||
final int cardinality = set.cardinality();
|
||||
long length;
|
||||
int jumpTableentryCount;
|
||||
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
|
||||
jumpTableentryCount = writeJumpTable(set, dir, out, denseRankPower);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
|
||||
int step = 16000;
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
BitSetIterator disi2 = new BitSetIterator(set, cardinality);
|
||||
assertAdvanceEquality(disi, disi2, step);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testRandom() throws IOException {
|
||||
try (Directory dir = newDirectory()) {
|
||||
int numIters = atLeast(3);
|
||||
for (int i = 0; i < numIters; ++i) {
|
||||
doTestRandom(dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void doTestRandom(Directory dir) throws IOException {
|
||||
Random random = random();
|
||||
final int maxStep = TestUtil.nextInt(random, 1, 1 << TestUtil.nextInt(random, 2, 20));
|
||||
final int numDocs = TestUtil.nextInt(random, 1, Math.min(100000, (Integer.MAX_VALUE - 1) / maxStep));
|
||||
BitSet docs = new SparseFixedBitSet(numDocs * maxStep + 1);
|
||||
int lastDoc = -1;
|
||||
for (int doc = -1, i = 0; i < numDocs; ++i) {
|
||||
doc += TestUtil.nextInt(random, 1, maxStep);
|
||||
docs.set(doc);
|
||||
lastDoc = doc;
|
||||
}
|
||||
final int maxDoc = lastDoc + TestUtil.nextInt(random, 1, 100);
|
||||
|
||||
BitSet set = BitSet.of(new BitSetIterator(docs, docs.approximateCardinality()), maxDoc);
|
||||
doTest(set, dir);
|
||||
}
|
||||
|
||||
private void doTest(BitSet set, Directory dir) throws IOException {
|
||||
final int cardinality = set.cardinality();
|
||||
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
|
||||
long length;
|
||||
int jumpTableentryCount;
|
||||
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
|
||||
jumpTableentryCount = writeJumpTable(set, dir, out, denseRankPower);
|
||||
length = out.getFilePointer();
|
||||
}
|
||||
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
BitSetIterator disi2 = new BitSetIterator(set, cardinality);
|
||||
assertSingleStepEquality(disi, disi2);
|
||||
}
|
||||
|
||||
for (int step : new int[] { 1, 10, 100, 1000, 10000, 100000 }) {
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
BitSetIterator disi2 = new BitSetIterator(set, cardinality);
|
||||
assertAdvanceEquality(disi, disi2, step);
|
||||
}
|
||||
}
|
||||
|
||||
for (int step : new int[] { 10, 100, 1000, 10000, 100000 }) {
|
||||
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
|
||||
IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
|
||||
BitSetIterator disi2 = new BitSetIterator(set, cardinality);
|
||||
int disi2length = set.length();
|
||||
assertAdvanceExactRandomized(disi, disi2, disi2length, step);
|
||||
}
|
||||
}
|
||||
|
||||
dir.deleteFile("foo");
|
||||
}
|
||||
|
||||
private void assertAdvanceExactRandomized(IndexedDISI disi, BitSetIterator disi2, int disi2length, int step) throws IOException {
|
||||
int index = -1;
|
||||
Random random = random();
|
||||
for (int target = 0; target < disi2length;) {
|
||||
target += TestUtil.nextInt(random, 0, step);
|
||||
int doc = disi2.docID();
|
||||
while (doc < target) {
|
||||
doc = disi2.nextDoc();
|
||||
index++;
|
||||
}
|
||||
|
||||
boolean exists = disi.advanceExact(target);
|
||||
assertEquals(doc == target, exists);
|
||||
if (exists) {
|
||||
assertEquals(index, disi.index());
|
||||
} else if (random.nextBoolean()) {
|
||||
assertEquals(doc, disi.nextDoc());
|
||||
// This is a bit strange when doc == NO_MORE_DOCS as the index overcounts in the disi2
|
||||
// while-loop
|
||||
assertEquals(index, disi.index());
|
||||
target = doc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void assertSingleStepEquality(IndexedDISI disi, BitSetIterator disi2) throws IOException {
|
||||
int i = 0;
|
||||
for (int doc = disi2.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = disi2.nextDoc()) {
|
||||
assertEquals(doc, disi.nextDoc());
|
||||
assertEquals(i++, disi.index());
|
||||
}
|
||||
assertEquals(DocIdSetIterator.NO_MORE_DOCS, disi.nextDoc());
|
||||
}
|
||||
|
||||
private void assertAdvanceEquality(IndexedDISI disi, BitSetIterator disi2, int step) throws IOException {
|
||||
int index = -1;
|
||||
while (true) {
|
||||
int target = disi2.docID() + step;
|
||||
int doc;
|
||||
do {
|
||||
doc = disi2.nextDoc();
|
||||
index++;
|
||||
} while (doc < target);
|
||||
assertEquals(doc, disi.advance(target));
|
||||
if (doc == DocIdSetIterator.NO_MORE_DOCS) {
|
||||
break;
|
||||
}
|
||||
assertEquals("Expected equality using step " + step + " at docID " + doc, index, disi.index());
|
||||
}
|
||||
}
|
||||
|
||||
private static short writeJumpTable(BitSet set, Directory dir, IndexOutput out, byte denseRankPower) throws IOException {
|
||||
return writeJumpTable(set, set.cardinality(), dir, out, denseRankPower);
|
||||
}
|
||||
|
||||
private static short writeJumpTable(BitSet set, long cost, Directory dir, IndexOutput out, byte denseRankPower) throws IOException {
|
||||
var disiAccumulator = new DISIAccumulator(dir, IOContext.DEFAULT, out, denseRankPower);
|
||||
var iterator = new BitSetIterator(set, cost);
|
||||
for (int docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
|
||||
disiAccumulator.addDocId(docId);
|
||||
}
|
||||
return disiAccumulator.build(out);
|
||||
}
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ public class InboundAggregatorTests extends ESTestCase {
|
|||
for (ReleasableBytesReference reference : references) {
|
||||
assertTrue(reference.hasReferences());
|
||||
}
|
||||
aggregated.decRef();
|
||||
aggregated.close();
|
||||
for (ReleasableBytesReference reference : references) {
|
||||
assertFalse(reference.hasReferences());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ public class InboundPipelineTests extends ESTestCase {
|
|||
final List<Tuple<MessageData, Exception>> actual = new ArrayList<>();
|
||||
final List<ReleasableBytesReference> toRelease = new ArrayList<>();
|
||||
final BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
|
||||
try {
|
||||
try (m) {
|
||||
final Header header = m.getHeader();
|
||||
final MessageData actualData;
|
||||
final TransportVersion version = header.getVersion();
|
||||
|
|
@ -204,7 +204,7 @@ public class InboundPipelineTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testDecodeExceptionIsPropagated() throws IOException {
|
||||
BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
|
||||
BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> m.close();
|
||||
final StatsTracker statsTracker = new StatsTracker();
|
||||
final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
|
||||
final InboundDecoder decoder = new InboundDecoder(recycler);
|
||||
|
|
@ -245,7 +245,7 @@ public class InboundPipelineTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
|
||||
BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
|
||||
BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> m.close();
|
||||
final StatsTracker statsTracker = new StatsTracker();
|
||||
final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
|
||||
final InboundDecoder decoder = new InboundDecoder(recycler);
|
||||
|
|
|
|||
|
|
@ -32,9 +32,12 @@ dependencies {
|
|||
|
||||
api "org.elasticsearch:mocksocket:${versions.mocksocket}"
|
||||
|
||||
testImplementation project(":modules:mapper-extras")
|
||||
testImplementation project(':x-pack:plugin:core')
|
||||
testImplementation project(':x-pack:plugin:mapper-unsigned-long')
|
||||
testImplementation project(':x-pack:plugin:mapper-counted-keyword')
|
||||
testImplementation project(":modules:mapper-extras")
|
||||
testImplementation project(':x-pack:plugin:mapper-constant-keyword')
|
||||
testImplementation project(':x-pack:plugin:wildcard')
|
||||
}
|
||||
|
||||
sourceSets {
|
||||
|
|
|
|||
|
|
@ -207,6 +207,13 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
|
|||
return mapperService;
|
||||
}
|
||||
|
||||
protected final MapperService createMapperService(IndexVersion indexVersion, Settings settings, XContentBuilder mappings)
|
||||
throws IOException {
|
||||
MapperService mapperService = createMapperService(indexVersion, settings, () -> true, mappings);
|
||||
merge(mapperService, mappings);
|
||||
return mapperService;
|
||||
}
|
||||
|
||||
protected final MapperService createMapperService(IndexVersion version, XContentBuilder mapping) throws IOException {
|
||||
return createMapperService(version, getIndexSettings(), () -> true, mapping);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ package org.elasticsearch.logsdb.datageneration;
|
|||
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.ConstantKeywordFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
|
||||
|
|
@ -26,6 +27,7 @@ import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataG
|
|||
import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.TextFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.fields.leaf.WildcardFieldDataGenerator;
|
||||
|
||||
/**
|
||||
* Lists all leaf field types that are supported for data generation by default.
|
||||
|
|
@ -46,7 +48,9 @@ public enum FieldType {
|
|||
DATE("date"),
|
||||
GEO_POINT("geo_point"),
|
||||
TEXT("text"),
|
||||
IP("ip");
|
||||
IP("ip"),
|
||||
CONSTANT_KEYWORD("constant_keyword"),
|
||||
WILDCARD("wildcard");
|
||||
|
||||
private final String name;
|
||||
|
||||
|
|
@ -56,7 +60,7 @@ public enum FieldType {
|
|||
|
||||
public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
|
||||
return switch (this) {
|
||||
case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
|
||||
case KEYWORD -> new KeywordFieldDataGenerator(dataSource);
|
||||
case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
|
||||
case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
|
||||
case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
|
||||
|
|
@ -72,6 +76,8 @@ public enum FieldType {
|
|||
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
|
||||
case TEXT -> new TextFieldDataGenerator(dataSource);
|
||||
case IP -> new IpFieldDataGenerator(dataSource);
|
||||
case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator();
|
||||
case WILDCARD -> new WildcardFieldDataGenerator(dataSource);
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -93,6 +99,8 @@ public enum FieldType {
|
|||
case "geo_point" -> FieldType.GEO_POINT;
|
||||
case "text" -> FieldType.TEXT;
|
||||
case "ip" -> FieldType.IP;
|
||||
case "constant_keyword" -> FieldType.CONSTANT_KEYWORD;
|
||||
case "wildcard" -> FieldType.WILDCARD;
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,10 @@ public interface DataSourceHandler {
|
|||
return null;
|
||||
}
|
||||
|
||||
default DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
|
||||
return null;
|
||||
}
|
||||
|
||||
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -126,6 +126,12 @@ public interface DataSourceRequest<TResponse extends DataSourceResponse> {
|
|||
}
|
||||
}
|
||||
|
||||
record VersionStringGenerator() implements DataSourceRequest<DataSourceResponse.VersionStringGenerator> {
|
||||
public DataSourceResponse.VersionStringGenerator accept(DataSourceHandler handler) {
|
||||
return handler.handle(this);
|
||||
}
|
||||
}
|
||||
|
||||
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
|
||||
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
|
||||
return handler.handle(this);
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ public interface DataSourceResponse {
|
|||
|
||||
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
|
||||
|
||||
record VersionStringGenerator(Supplier<String> generator) implements DataSourceResponse {}
|
||||
|
||||
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
|
||||
|
||||
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ public class DefaultMappingParametersHandler implements DataSourceHandler {
|
|||
case GEO_POINT -> geoPointMapping(map);
|
||||
case TEXT -> textMapping(request, new HashMap<>());
|
||||
case IP -> ipMapping(map);
|
||||
case CONSTANT_KEYWORD -> constantKeywordMapping(new HashMap<>());
|
||||
case WILDCARD -> wildcardMapping(new HashMap<>());
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -225,6 +227,29 @@ public class DefaultMappingParametersHandler implements DataSourceHandler {
|
|||
};
|
||||
}
|
||||
|
||||
private Supplier<Map<String, Object>> constantKeywordMapping(Map<String, Object> injected) {
|
||||
return () -> {
|
||||
// value is optional and can be set from the first document
|
||||
// we don't cover this case here
|
||||
injected.put("value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
|
||||
|
||||
return injected;
|
||||
};
|
||||
}
|
||||
|
||||
private Supplier<Map<String, Object>> wildcardMapping(Map<String, Object> injected) {
|
||||
return () -> {
|
||||
if (ESTestCase.randomDouble() <= 0.2) {
|
||||
injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
|
||||
}
|
||||
if (ESTestCase.randomDouble() <= 0.2) {
|
||||
injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
|
||||
}
|
||||
|
||||
return injected;
|
||||
};
|
||||
}
|
||||
|
||||
private static HashMap<String, Object> commonMappingParameters() {
|
||||
var map = new HashMap<String, Object>();
|
||||
map.put("store", ESTestCase.randomBoolean());
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
package org.elasticsearch.logsdb.datageneration.fields.leaf;
|
||||
|
||||
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class ConstantKeywordFieldDataGenerator implements FieldDataGenerator {
|
||||
@Override
|
||||
public Object generateValue(Map<String, Object> fieldMapping) {
|
||||
if (fieldMapping == null) {
|
||||
// Dynamically mapped, skip it because it will be mapped as text, and we cover this case already
|
||||
return null;
|
||||
}
|
||||
|
||||
var value = fieldMapping.get("value");
|
||||
assert value != null;
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
|
@ -13,14 +13,11 @@ import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
|
|||
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class CountedKeywordFieldDataGenerator implements FieldDataGenerator {
|
||||
private final Supplier<Object> valueGenerator;
|
||||
private final Set<String> previousStrings = new HashSet<>();
|
||||
|
||||
public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
|
||||
var strings = dataSource.get(new DataSourceRequest.StringGenerator());
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import java.util.function.Supplier;
|
|||
public class KeywordFieldDataGenerator implements FieldDataGenerator {
|
||||
private final Supplier<Object> valueGenerator;
|
||||
|
||||
public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
|
||||
public KeywordFieldDataGenerator(DataSource dataSource) {
|
||||
var strings = dataSource.get(new DataSourceRequest.StringGenerator());
|
||||
var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
|
||||
var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
package org.elasticsearch.logsdb.datageneration.fields.leaf;
|
||||
|
||||
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class WildcardFieldDataGenerator implements FieldDataGenerator {
|
||||
private final FieldDataGenerator keywordGenerator;
|
||||
|
||||
public WildcardFieldDataGenerator(DataSource dataSource) {
|
||||
this.keywordGenerator = new KeywordFieldDataGenerator(dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object generateValue(Map<String, Object> fieldMapping) {
|
||||
return keywordGenerator.generateValue(fieldMapping);
|
||||
}
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@ interface FieldSpecificMatcher {
|
|||
return new HashMap<>() {
|
||||
{
|
||||
put("keyword", new KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("long", new NumberMatcher("long", actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("unsigned_long", new UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("integer", new NumberMatcher("integer", actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
|
|
@ -58,11 +57,14 @@ interface FieldSpecificMatcher {
|
|||
put("scaled_float", new ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("counted_keyword", new CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("boolean", new BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("date", new DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("geo_shape", new ExactMatcher("geo_shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("shape", new ExactMatcher("shape", actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("geo_point", new GeoPointMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("text", new TextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("ip", new IpMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("constant_keyword", new ConstantKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
put("wildcard", new WildcardMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -691,6 +693,46 @@ interface FieldSpecificMatcher {
|
|||
}
|
||||
}
|
||||
|
||||
class ConstantKeywordMatcher extends GenericMappingAwareMatcher {
|
||||
ConstantKeywordMatcher(
|
||||
XContentBuilder actualMappings,
|
||||
Settings.Builder actualSettings,
|
||||
XContentBuilder expectedMappings,
|
||||
Settings.Builder expectedSettings
|
||||
) {
|
||||
super("constant_keyword", actualMappings, actualSettings, expectedMappings, expectedSettings);
|
||||
}
|
||||
|
||||
@Override
|
||||
Object convert(Object value, Object nullValue) {
|
||||
// We just need to get rid of literal `null`s which is done in the caller.
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
class WildcardMatcher extends GenericMappingAwareMatcher {
|
||||
WildcardMatcher(
|
||||
XContentBuilder actualMappings,
|
||||
Settings.Builder actualSettings,
|
||||
XContentBuilder expectedMappings,
|
||||
Settings.Builder expectedSettings
|
||||
) {
|
||||
super("wildcard", actualMappings, actualSettings, expectedMappings, expectedSettings);
|
||||
}
|
||||
|
||||
@Override
|
||||
Object convert(Object value, Object nullValue) {
|
||||
if (value == null) {
|
||||
if (nullValue != null) {
|
||||
return nullValue;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic matcher that supports common matching logic like null values.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -2057,12 +2057,34 @@ public abstract class ESRestTestCase extends ESTestCase {
|
|||
}
|
||||
|
||||
protected static boolean indexExists(String index) throws IOException {
|
||||
return indexExists(client(), index);
|
||||
// We use the /_cluster/health/{index} API to ensure the index exists on the master node - which means all nodes see the index.
|
||||
Request request = new Request("GET", "/_cluster/health/" + index);
|
||||
request.addParameter("timeout", "0");
|
||||
request.addParameter("level", "indices");
|
||||
try {
|
||||
final var response = client().performRequest(request);
|
||||
@SuppressWarnings("unchecked")
|
||||
final var indices = (Map<String, Object>) entityAsMap(response).get("indices");
|
||||
return indices.containsKey(index);
|
||||
} catch (ResponseException e) {
|
||||
if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_REQUEST_TIMEOUT) {
|
||||
return false;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
protected static boolean indexExists(RestClient client, String index) throws IOException {
|
||||
Response response = client.performRequest(new Request("HEAD", "/" + index));
|
||||
return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode();
|
||||
protected static void awaitIndexExists(String index) throws IOException {
|
||||
awaitIndexExists(index, SAFE_AWAIT_TIMEOUT);
|
||||
}
|
||||
|
||||
protected static void awaitIndexExists(String index, TimeValue timeout) throws IOException {
|
||||
// We use the /_cluster/health/{index} API to ensure the index exists on the master node - which means all nodes see the index.
|
||||
ensureHealth(client(), index, request -> request.addParameter("timeout", timeout.toString()));
|
||||
}
|
||||
|
||||
protected static void awaitIndexDoesNotExist(String index, TimeValue timeout) throws Exception {
|
||||
assertBusy(() -> assertFalse(indexExists(index)), timeout.millis(), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -20,8 +20,10 @@ import org.elasticsearch.plugins.Plugin;
|
|||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.xcontent.XContentType;
|
||||
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
|
||||
import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin;
|
||||
import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin;
|
||||
import org.elasticsearch.xpack.wildcard.Wildcard;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
|
|
@ -111,7 +113,13 @@ public class DataGenerationTests extends ESTestCase {
|
|||
var mappingService = new MapperServiceTestCase() {
|
||||
@Override
|
||||
protected Collection<? extends Plugin> getPlugins() {
|
||||
return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin(), new CountedKeywordMapperPlugin());
|
||||
return List.of(
|
||||
new UnsignedLongMapperPlugin(),
|
||||
new MapperExtrasPlugin(),
|
||||
new CountedKeywordMapperPlugin(),
|
||||
new ConstantKeywordMapperPlugin(),
|
||||
new Wildcard()
|
||||
);
|
||||
}
|
||||
}.createMapperService(mappingXContent);
|
||||
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ tasks.named("test").configure {
|
|||
// The kibana docs are not deployed to the normal docs location, so need absolute paths for internal references
|
||||
line.replaceAll(
|
||||
/\]\(\/reference\/([^)\s]+)\.md(#\S+)?\)/,
|
||||
'](https://www.elastic.co/docs/reference/elasticsearch/$1$2)'
|
||||
'](https://www.elastic.co/docs/reference/$1$2)'
|
||||
)
|
||||
}
|
||||
if (countKibana == 0) {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import org.elasticsearch.client.WarningFailureException;
|
|||
import org.elasticsearch.cluster.metadata.IndexMetadata;
|
||||
import org.elasticsearch.cluster.metadata.Template;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.core.TimeValue;
|
||||
import org.elasticsearch.index.engine.EngineConfig;
|
||||
import org.elasticsearch.test.rest.ESRestTestCase;
|
||||
import org.elasticsearch.xcontent.XContentType;
|
||||
|
|
@ -153,7 +154,7 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
|
|||
);
|
||||
|
||||
String shrunkenIndex = waitAndGetShrinkIndexName(client(), backingIndexName);
|
||||
assertBusy(() -> assertTrue(indexExists(shrunkenIndex)), 30, TimeUnit.SECONDS);
|
||||
awaitIndexExists(shrunkenIndex, TimeValue.timeValueSeconds(30));
|
||||
assertBusy(() -> assertThat(getStepKeyForIndex(client(), shrunkenIndex), equalTo(PhaseCompleteStep.finalStep("warm").getKey())));
|
||||
assertBusy(() -> assertThat("the original index must've been deleted", indexExists(backingIndexName), is(false)));
|
||||
}
|
||||
|
|
@ -182,8 +183,8 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
|
|||
// Manual rollover the original index such that it's not the write index in the data stream anymore
|
||||
rolloverMaxOneDocCondition(client(), dataStream);
|
||||
|
||||
assertBusy(() -> assertThat(indexExists(restoredIndexName), is(true)));
|
||||
assertBusy(() -> assertFalse(indexExists(backingIndexName)), 60, TimeUnit.SECONDS);
|
||||
awaitIndexExists(restoredIndexName);
|
||||
awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60));
|
||||
assertBusy(
|
||||
() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)),
|
||||
30,
|
||||
|
|
@ -211,15 +212,13 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
|
|||
// Manual rollover the original index such that it's not the write index in the data stream anymore
|
||||
rolloverMaxOneDocCondition(client(), dataStream);
|
||||
|
||||
assertBusy(
|
||||
() -> assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME)),
|
||||
30,
|
||||
TimeUnit.SECONDS
|
||||
);
|
||||
assertThat(
|
||||
getOnlyIndexSettings(client(), backingIndexName).get(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()),
|
||||
equalTo("true")
|
||||
);
|
||||
assertBusy(() -> {
|
||||
assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME));
|
||||
assertThat(
|
||||
getOnlyIndexSettings(client(), backingIndexName).get(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()),
|
||||
equalTo("true")
|
||||
);
|
||||
}, 30, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public void testFreezeAction() throws Exception {
|
||||
|
|
@ -243,10 +242,10 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
|
|||
containsString("The freeze action in ILM is deprecated and will be removed in a future version")
|
||||
);
|
||||
}
|
||||
Map<String, Object> settings = getOnlyIndexSettings(client(), backingIndexName);
|
||||
assertNull(settings.get("index.frozen"));
|
||||
}, 30, TimeUnit.SECONDS);
|
||||
|
||||
Map<String, Object> settings = getOnlyIndexSettings(client(), backingIndexName);
|
||||
assertNull(settings.get("index.frozen"));
|
||||
}
|
||||
|
||||
public void checkForceMergeAction(String codec) throws Exception {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ package org.elasticsearch.xpack.inference.mapper;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
|
|
@ -95,6 +96,7 @@ import java.util.function.BiConsumer;
|
|||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ;
|
||||
import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING;
|
||||
import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING;
|
||||
import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
|
||||
|
|
@ -133,6 +135,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
|
|||
public static final String CONTENT_TYPE = "semantic_text";
|
||||
public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
|
||||
|
||||
public static final float DEFAULT_RESCORE_OVERSAMPLE = 3.0f;
|
||||
|
||||
public static final TypeParser parser(Supplier<ModelRegistry> modelRegistry) {
|
||||
return new TypeParser(
|
||||
(n, c) -> new Builder(n, c::bitSetProducer, c.getIndexSettings(), modelRegistry.get()),
|
||||
|
|
@ -1054,12 +1058,30 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
|
|||
denseVectorMapperBuilder.dimensions(modelSettings.dimensions());
|
||||
denseVectorMapperBuilder.elementType(modelSettings.elementType());
|
||||
|
||||
DenseVectorFieldMapper.IndexOptions defaultIndexOptions = null;
|
||||
if (indexVersionCreated.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BBQ)) {
|
||||
defaultIndexOptions = defaultSemanticDenseIndexOptions();
|
||||
}
|
||||
if (defaultIndexOptions != null
|
||||
&& defaultIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false)) {
|
||||
denseVectorMapperBuilder.indexOptions(defaultIndexOptions);
|
||||
}
|
||||
|
||||
yield denseVectorMapperBuilder;
|
||||
}
|
||||
default -> throw new IllegalArgumentException("Invalid task_type in model_settings [" + modelSettings.taskType().name() + "]");
|
||||
};
|
||||
}
|
||||
|
||||
static DenseVectorFieldMapper.IndexOptions defaultSemanticDenseIndexOptions() {
|
||||
// As embedding models for text perform better with BBQ, we aggressively default semantic_text fields to use optimized index
|
||||
// options outside of dense_vector defaults
|
||||
int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
|
||||
int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
||||
DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE);
|
||||
return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector);
|
||||
}
|
||||
|
||||
private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) {
|
||||
if (previous != null && current != null && previous.canMergeWith(current)) {
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -37,7 +37,10 @@ public class SemanticInferenceMetadataFieldsMapperTests extends MapperServiceTes
|
|||
assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings));
|
||||
|
||||
settings = Settings.builder()
|
||||
.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true))
|
||||
.put(
|
||||
IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(),
|
||||
getRandomCompatibleIndexVersion(true, IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS))
|
||||
)
|
||||
.put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false)
|
||||
.build();
|
||||
assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings));
|
||||
|
|
@ -114,18 +117,18 @@ public class SemanticInferenceMetadataFieldsMapperTests extends MapperServiceTes
|
|||
}
|
||||
|
||||
static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) {
|
||||
return getRandomCompatibleIndexVersion(useLegacyFormat, IndexVersion.current());
|
||||
}
|
||||
|
||||
static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat, IndexVersion maxVersion) {
|
||||
if (useLegacyFormat) {
|
||||
if (randomBoolean()) {
|
||||
return IndexVersionUtils.randomVersionBetween(
|
||||
random(),
|
||||
IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
|
||||
IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS)
|
||||
);
|
||||
return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.UPGRADE_TO_LUCENE_10_0_0, maxVersion);
|
||||
}
|
||||
return IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT);
|
||||
} else {
|
||||
if (randomBoolean()) {
|
||||
return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersion.current());
|
||||
return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, maxVersion);
|
||||
}
|
||||
return IndexVersionUtils.randomVersionBetween(
|
||||
random(),
|
||||
|
|
@ -134,4 +137,5 @@ public class SemanticInferenceMetadataFieldsMapperTests extends MapperServiceTes
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ package org.elasticsearch.xpack.inference.mapper;
|
|||
|
||||
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
|
||||
|
||||
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
|
||||
import org.apache.lucene.index.FieldInfo;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
|
|
@ -35,6 +36,7 @@ import org.elasticsearch.common.lucene.search.Queries;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.core.CheckedConsumer;
|
||||
import org.elasticsearch.index.IndexVersion;
|
||||
import org.elasticsearch.index.IndexVersions;
|
||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||
import org.elasticsearch.index.mapper.DocumentParsingException;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
|
|
@ -66,6 +68,7 @@ import org.elasticsearch.search.NestedDocuments;
|
|||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.test.ClusterServiceUtils;
|
||||
import org.elasticsearch.test.client.NoOpClient;
|
||||
import org.elasticsearch.test.index.IndexVersionUtils;
|
||||
import org.elasticsearch.threadpool.TestThreadPool;
|
||||
import org.elasticsearch.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.xcontent.XContentType;
|
||||
|
|
@ -149,14 +152,44 @@ public class SemanticTextFieldMapperTests extends MapperTestCase {
|
|||
}
|
||||
|
||||
private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException {
|
||||
IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat);
|
||||
return createMapperService(mappings, useLegacyFormat, indexVersion, indexVersion, false);
|
||||
}
|
||||
|
||||
private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat, IndexVersion minIndexVersion)
|
||||
throws IOException {
|
||||
return createMapperService(mappings, useLegacyFormat, minIndexVersion, IndexVersion.current(), false);
|
||||
}
|
||||
|
||||
private MapperService createMapperService(
|
||||
XContentBuilder mappings,
|
||||
boolean useLegacyFormat,
|
||||
IndexVersion minIndexVersion,
|
||||
IndexVersion maxIndexVersion,
|
||||
boolean propagateIndexVersion
|
||||
) throws IOException {
|
||||
validateIndexVersion(minIndexVersion, useLegacyFormat);
|
||||
IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(random(), minIndexVersion, maxIndexVersion);
|
||||
var settings = Settings.builder()
|
||||
.put(
|
||||
IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(),
|
||||
SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat)
|
||||
)
|
||||
.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion)
|
||||
.put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
|
||||
.build();
|
||||
return createMapperService(settings, mappings);
|
||||
// TODO - This is added, because we discovered a bug where the index version was not being correctly propagated
|
||||
// in our mappings even though we were specifying the index version in settings. We will fix this in a followup and
|
||||
// remove the boolean flag accordingly.
|
||||
if (propagateIndexVersion) {
|
||||
return createMapperService(indexVersion, settings, mappings);
|
||||
} else {
|
||||
return createMapperService(settings, mappings);
|
||||
}
|
||||
}
|
||||
|
||||
private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) {
|
||||
if (useLegacyFormat == false
|
||||
&& indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS)
|
||||
&& indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) {
|
||||
throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -602,14 +635,15 @@ public class SemanticTextFieldMapperTests extends MapperTestCase {
|
|||
}
|
||||
|
||||
private static void assertSemanticTextField(MapperService mapperService, String fieldName, boolean expectedModelSettings) {
|
||||
assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null);
|
||||
assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null, null);
|
||||
}
|
||||
|
||||
private static void assertSemanticTextField(
|
||||
MapperService mapperService,
|
||||
String fieldName,
|
||||
boolean expectedModelSettings,
|
||||
ChunkingSettings expectedChunkingSettings
|
||||
ChunkingSettings expectedChunkingSettings,
|
||||
DenseVectorFieldMapper.IndexOptions expectedIndexOptions
|
||||
) {
|
||||
Mapper mapper = mapperService.mappingLookup().getMapper(fieldName);
|
||||
assertNotNull(mapper);
|
||||
|
|
@ -655,8 +689,17 @@ public class SemanticTextFieldMapperTests extends MapperTestCase {
|
|||
assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
|
||||
SparseVectorFieldMapper sparseMapper = (SparseVectorFieldMapper) embeddingsMapper;
|
||||
assertEquals(sparseMapper.fieldType().isStored(), semanticTextFieldType.useLegacyFormat() == false);
|
||||
assertNull(expectedIndexOptions);
|
||||
}
|
||||
case TEXT_EMBEDDING -> {
|
||||
assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
|
||||
DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper;
|
||||
if (expectedIndexOptions != null) {
|
||||
assertEquals(expectedIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions());
|
||||
} else {
|
||||
assertNull(denseVectorFieldMapper.fieldType().getIndexOptions());
|
||||
}
|
||||
}
|
||||
case TEXT_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
|
||||
default -> throw new AssertionError("Invalid task type");
|
||||
}
|
||||
} else {
|
||||
|
|
@ -951,11 +994,11 @@ public class SemanticTextFieldMapperTests extends MapperTestCase {
|
|||
mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings)),
|
||||
useLegacyFormat
|
||||
);
|
||||
assertSemanticTextField(mapperService, fieldName, false, chunkingSettings);
|
||||
assertSemanticTextField(mapperService, fieldName, false, chunkingSettings, null);
|
||||
|
||||
ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings);
|
||||
merge(mapperService, mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings)));
|
||||
assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings);
|
||||
assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, null);
|
||||
}
|
||||
|
||||
public void testModelSettingsRequiredWithChunks() throws IOException {
|
||||
|
|
@ -1085,6 +1128,74 @@ public class SemanticTextFieldMapperTests extends MapperTestCase {
|
|||
assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class));
|
||||
}
|
||||
|
||||
private static DenseVectorFieldMapper.IndexOptions defaultDenseVectorIndexOptions() {
|
||||
// These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ.
|
||||
int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
|
||||
int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
|
||||
return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null);
|
||||
}
|
||||
|
||||
public void testDefaultIndexOptions() throws IOException {
|
||||
|
||||
// We default to BBQ for eligible dense vectors
|
||||
var mapperService = createMapperService(fieldMapping(b -> {
|
||||
b.field("type", "semantic_text");
|
||||
b.field("inference_id", "another_inference_id");
|
||||
b.startObject("model_settings");
|
||||
b.field("task_type", "text_embedding");
|
||||
b.field("dimensions", 100);
|
||||
b.field("similarity", "cosine");
|
||||
b.field("element_type", "float");
|
||||
b.endObject();
|
||||
}), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
|
||||
assertSemanticTextField(mapperService, "field", true, null, SemanticTextFieldMapper.defaultSemanticDenseIndexOptions());
|
||||
|
||||
// Element types that are incompatible with BBQ will continue to use dense_vector defaults
|
||||
mapperService = createMapperService(fieldMapping(b -> {
|
||||
b.field("type", "semantic_text");
|
||||
b.field("inference_id", "another_inference_id");
|
||||
b.startObject("model_settings");
|
||||
b.field("task_type", "text_embedding");
|
||||
b.field("dimensions", 100);
|
||||
b.field("similarity", "cosine");
|
||||
b.field("element_type", "byte");
|
||||
b.endObject();
|
||||
}), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
|
||||
assertSemanticTextField(mapperService, "field", true, null, null);
|
||||
|
||||
// A dim count of 10 is too small to support BBQ, so we continue to use dense_vector defaults
|
||||
mapperService = createMapperService(fieldMapping(b -> {
|
||||
b.field("type", "semantic_text");
|
||||
b.field("inference_id", "another_inference_id");
|
||||
b.startObject("model_settings");
|
||||
b.field("task_type", "text_embedding");
|
||||
b.field("dimensions", 10);
|
||||
b.field("similarity", "cosine");
|
||||
b.field("element_type", "float");
|
||||
b.endObject();
|
||||
}), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
|
||||
assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions());
|
||||
|
||||
// Previous index versions do not set BBQ index options
|
||||
mapperService = createMapperService(fieldMapping(b -> {
|
||||
b.field("type", "semantic_text");
|
||||
b.field("inference_id", "another_inference_id");
|
||||
b.startObject("model_settings");
|
||||
b.field("task_type", "text_embedding");
|
||||
b.field("dimensions", 100);
|
||||
b.field("similarity", "cosine");
|
||||
b.field("element_type", "float");
|
||||
b.endObject();
|
||||
}),
|
||||
useLegacyFormat,
|
||||
IndexVersions.INFERENCE_METADATA_FIELDS,
|
||||
IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ),
|
||||
true
|
||||
);
|
||||
assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions());
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) {
|
||||
// Until a doc is indexed, the query is rewritten as match no docs
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.constantkeyword.mapper;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
|
||||
import org.elasticsearch.logsdb.datageneration.FieldType;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ConstantKeywordFieldBlockLoaderTests extends BlockLoaderTestCase {
|
||||
public ConstantKeywordFieldBlockLoaderTests(Params params) {
|
||||
super(FieldType.CONSTANT_KEYWORD.toString(), params);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
|
||||
return new BytesRef((String) fieldMapping.get("value"));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Plugin> getPlugins() {
|
||||
return List.of(new ConstantKeywordMapperPlugin());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.xpack.versionfield.datageneration.VersionStringDataSourceHandler;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public class VersionStringFieldBlockLoaderTests extends BlockLoaderTestCase {
|
||||
public VersionStringFieldBlockLoaderTests(Params params) {
|
||||
super("version", List.of(new VersionStringDataSourceHandler()), params);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof String s) {
|
||||
return convert(s);
|
||||
}
|
||||
|
||||
var resultList = ((List<String>) value).stream().map(this::convert).filter(Objects::nonNull).distinct().sorted().toList();
|
||||
return maybeFoldList(resultList);
|
||||
}
|
||||
|
||||
private BytesRef convert(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return VersionEncoder.encodeVersion(value).bytesRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<? extends Plugin> getPlugins() {
|
||||
return Collections.singletonList(new VersionFieldPlugin(getIndexSettings()));
|
||||
}
|
||||
}
|
||||
|
|
@ -148,27 +148,7 @@ public class VersionStringFieldMapperTests extends MapperTestCase {
|
|||
|
||||
@Override
|
||||
protected String generateRandomInputValue(MappedFieldType ft) {
|
||||
return randomValue();
|
||||
}
|
||||
|
||||
protected static String randomValue() {
|
||||
return randomVersionNumber() + (randomBoolean() ? "" : randomPrerelease());
|
||||
}
|
||||
|
||||
private static String randomVersionNumber() {
|
||||
int numbers = between(1, 3);
|
||||
String v = Integer.toString(between(0, 100));
|
||||
for (int i = 1; i < numbers; i++) {
|
||||
v += "." + between(0, 100);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
private static String randomPrerelease() {
|
||||
if (rarely()) {
|
||||
return randomFrom("alpha", "beta", "prerelease", "whatever");
|
||||
}
|
||||
return randomFrom("alpha", "beta", "") + randomVersionNumber();
|
||||
return VersionStringTestUtils.randomVersionString();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -212,7 +192,7 @@ public class VersionStringFieldMapperTests extends MapperTestCase {
|
|||
}
|
||||
|
||||
private Tuple<String, String> generateValue() {
|
||||
String v = randomValue();
|
||||
String v = VersionStringTestUtils.randomVersionString();
|
||||
return Tuple.tuple(v, v);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield;
|
||||
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
public class VersionStringTestUtils {
|
||||
public static String randomVersionString() {
|
||||
return randomVersionNumber() + (ESTestCase.randomBoolean() ? "" : randomPrerelease());
|
||||
}
|
||||
|
||||
private static String randomVersionNumber() {
|
||||
int numbers = ESTestCase.between(1, 3);
|
||||
String v = Integer.toString(ESTestCase.between(0, 100));
|
||||
for (int i = 1; i < numbers; i++) {
|
||||
v += "." + ESTestCase.between(0, 100);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
private static String randomPrerelease() {
|
||||
if (ESTestCase.rarely()) {
|
||||
return ESTestCase.randomFrom("alpha", "beta", "prerelease", "whatever");
|
||||
}
|
||||
return ESTestCase.randomFrom("alpha", "beta", "") + randomVersionNumber();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield.datageneration;
|
||||
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
|
||||
import org.elasticsearch.xpack.versionfield.VersionStringTestUtils;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
public class VersionStringDataSourceHandler implements DataSourceHandler {
|
||||
@Override
|
||||
public DataSourceResponse.VersionStringGenerator handle(DataSourceRequest.VersionStringGenerator request) {
|
||||
return new DataSourceResponse.VersionStringGenerator(VersionStringTestUtils::randomVersionString);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
|
||||
if (request.fieldType().equals("version") == false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new DataSourceResponse.LeafMappingParametersGenerator(HashMap::new);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) {
|
||||
if (request.fieldType().equals("version") == false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new DataSourceResponse.FieldDataGenerator(new VersionStringFieldDataGenerator(request.dataSource()));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.versionfield.datageneration;
|
||||
|
||||
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
|
||||
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class VersionStringFieldDataGenerator implements FieldDataGenerator {
|
||||
private final Supplier<Object> values;
|
||||
|
||||
public VersionStringFieldDataGenerator(DataSource dataSource) {
|
||||
var nullWrapper = dataSource.get(new DataSourceRequest.NullWrapper());
|
||||
var versionStrings = dataSource.get(new DataSourceRequest.VersionStringGenerator());
|
||||
|
||||
this.values = nullWrapper.wrapper().apply(versionStrings.generator()::get);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object generateValue(Map<String, Object> fieldMapping) {
|
||||
return this.values.get();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.wildcard.mapper;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
|
||||
import org.elasticsearch.logsdb.datageneration.FieldType;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.xpack.wildcard.Wildcard;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public class WildcardFieldBlockLoaderTests extends BlockLoaderTestCase {
|
||||
public WildcardFieldBlockLoaderTests(Params params) {
|
||||
super(FieldType.WILDCARD.toString(), params);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
|
||||
var nullValue = (String) fieldMapping.get("null_value");
|
||||
|
||||
var ignoreAbove = fieldMapping.get("ignore_above") == null
|
||||
? Integer.MAX_VALUE
|
||||
: ((Number) fieldMapping.get("ignore_above")).intValue();
|
||||
|
||||
if (value == null) {
|
||||
return convert(null, nullValue, ignoreAbove);
|
||||
}
|
||||
|
||||
if (value instanceof String s) {
|
||||
return convert(s, nullValue, ignoreAbove);
|
||||
}
|
||||
|
||||
var resultList = ((List<String>) value).stream()
|
||||
.map(s -> convert(s, nullValue, ignoreAbove))
|
||||
.filter(Objects::nonNull)
|
||||
.distinct()
|
||||
.sorted()
|
||||
.toList();
|
||||
return maybeFoldList(resultList);
|
||||
}
|
||||
|
||||
private static BytesRef convert(String value, String nullValue, int ignoreAbove) {
|
||||
if (value == null) {
|
||||
if (nullValue != null) {
|
||||
value = nullValue;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return value.length() <= ignoreAbove ? new BytesRef(value) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<? extends Plugin> getPlugins() {
|
||||
return Collections.singleton(new Wildcard());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue