github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-25 07:37:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge main into multi-project

Browse source
This commit is contained in:
Tim Vernum 2025-02-19 16:40:34 +11:00
commit 838d8389de
144 changed files with 1643 additions and 2426 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.buildkite/scripts/dra-update-staging.sh
View file

@ -37,7 +37,7 @@ for BRANCH in "${BRANCHES[@]}"; do
if [[ "$SHOULD_TRIGGER" == "true" ]]; then if [[ "$SHOULD_TRIGGER" == "true" ]]; then
if [[ "$BRANCH" == "9.0" ]]; then if [[ "$BRANCH" == "9.0" ]]; then
export VERSION_QUALIFIER="beta1" export VERSION_QUALIFIER="rc1"
fi fi
echo "Triggering DRA staging workflow for $BRANCH" echo "Triggering DRA staging workflow for $BRANCH"
cat << EOF | buildkite-agent pipeline upload cat << EOF | buildkite-agent pipeline upload

2
.buildkite/scripts/dra-workflow.trigger.sh
View file

@ -8,7 +8,7 @@ source .buildkite/scripts/branches.sh
for BRANCH in "${BRANCHES[@]}"; do for BRANCH in "${BRANCHES[@]}"; do
if [[ "$BRANCH" == "9.0" ]]; then if [[ "$BRANCH" == "9.0" ]]; then
export VERSION_QUALIFIER="beta1" export VERSION_QUALIFIER="rc1"
fi fi
INTAKE_PIPELINE_SLUG="elasticsearch-intake" INTAKE_PIPELINE_SLUG="elasticsearch-intake"

2
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
View file

@ -22,7 +22,7 @@ public enum DockerBase {
// Chainguard based wolfi image with latest jdk // Chainguard based wolfi image with latest jdk
// This is usually updated via renovatebot // This is usually updated via renovatebot
// spotless:off // spotless:off
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:ecd940be9f342ee6173397c48f3df5bb410e95000f8726fd01759b6c39b0beda", WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b",
"-wolfi", "-wolfi",
"apk" "apk"
), ),

5
docs/changelog/120952.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 120952
summary: Add `_metric_names_hash` field to OTel metric mappings
area: Data streams
type: bug
issues: []

6
docs/changelog/122409.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 122409
summary: Allow setting the `type` in the reroute processor
area: Ingest Node
type: enhancement
issues:
- 121553

5
docs/changelog/122538.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 122538
summary: Fix `ArrayIndexOutOfBoundsException` in `ShardBulkInferenceActionFilter`
area: Ingest
type: bug
issues: []

6
docs/changelog/122637.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 122637
summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float`
fields
area: Mapping
type: enhancement
issues: []

5
docs/changelog/122737.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 122737
summary: Bump json-smart and oauth2-oidc-sdk
area: Authentication
type: upgrade
issues: []

42
docs/internal/DistributedArchitectureGuide.md
View file

@ -229,19 +229,45 @@ works in parallel with the storage engine.)
# Allocation # Allocation
(AllocationService runs on the master node) ### Core Components
(Discuss different deciders that limit allocation. Sketch / list the different deciders that we have.) The `DesiredBalanceShardsAllocator` is what runs shard allocation decisions. It leverages the `DesiredBalanceComputer` to produce
`DesiredBalance` instances for the cluster based on the latest cluster changes (add/remove nodes, create/remove indices, load, etc.). Then
the `DesiredBalanceReconciler` is invoked to choose the next steps to take to move the cluster from the current shard allocation to the
latest computed `DesiredBalance` shard allocation. The `DesiredBalanceReconciler` will apply changes to a copy of the `RoutingNodes`, which
is then published in a cluster state update that will reach the data nodes to start the individual shard recovery/deletion/move work.
### APIs for Balancing Operations The `DesiredBalanceReconciler` is throttled by cluster settings, like the max number of concurrent shard moves and recoveries per cluster
and node: this is why the `DesiredBalanceReconciler` will make, and publish via cluster state updates, incremental changes to the cluster
shard allocation. The `DesiredBalanceShardsAllocator` is the endpoint for reroute requests, which may trigger immediate requests to the
`DesiredBalanceReconciler`, but asynchronous requests to the `DesiredBalanceComputer` via the `ContinuousComputation` component. Cluster
state changes that affect shard balancing (for example index deletion) all call some reroute method interface that reaches the
`DesiredBalanceShardsAllocator` to run reconciliation and queue a request for the `DesiredBalancerComputer`, leading to desired balance
computation and reconciliation actions. Asynchronous completion of a new `DesiredBalance` will also invoke a reconciliation action, as will
cluster state updates completing shard moves/recoveries (unthrottling the next shard move/recovery).
(Significant internal APIs for balancing a cluster) The `ContinuousComputation` saves the latest desired balance computation request, which holds the cluster information at the time of that
request, and a thread that runs the `DesiredBalanceComputer`. The `ContinuousComputation` thread takes the latest request, with the
associated cluster information, feeds it into the `DesiredBalanceComputer` and publishes a `DesiredBalance` back to the
`DesiredBalanceShardsAllocator` to use for reconciliation actions. Sometimes the `ContinuousComputation` thread's desired balance
computation will be signalled to exit early and publish the initial `DesiredBalance` improvements it has made, when newer rebalancing
requests (due to cluster state changes) have arrived, or in order to begin recovery of unassigned shards as quickly as possible.
### Heuristics for Allocation ### Rebalancing Process
### Cluster Reroute Command There are different priorities in shard allocation, reflected in which moves the `DesiredBalancerReconciler` selects to do first given that
it can only move, recover, or remove a limited number of shards at once. The first priority is assigning unassigned shards, primaries being
(How does this command behave with the desired auto balancer.) more important than replicas. The second is to move shards that violate any rule (such as node resource limits) as defined by an
`AllocationDecider`. The `AllocationDeciders` holds a group of `AllocationDecider` implementations that place hard constraints on shard
allocation. There is a decider, `DiskThresholdDecider`, that manages disk memory usage thresholds, such that further shards may not be
allowed assignment to a node, or shards may be required to move off because they grew to exceed the disk space; or another,
`FilterAllocationDecider`, that excludes a configurable list of indices from certain nodes; or `MaxRetryAllocationDecider` that will not
attempt to recover a shard on a certain node after so many failed retries. The third priority is to rebalance shards to even out the
relative weight of shards on each node: the intention is to avoid, or ease, future hot-spotting on data nodes due to too many shards being
placed on the same data node. Node shard weight is based on a sum of factors: disk memory usage, projected shard write load, total number
of shards, and an incentive to distribute shards within the same index across different nodes. See the `WeightFunction` and
`NodeAllocationStatsAndWeightsCalculator` classes for more details on the weight calculations that support the `DesiredBalanceComputer`
decisions.
# Autoscaling # Autoscaling

3
docs/reference/ingest/processors/reroute.asciidoc
View file

@ -45,6 +45,9 @@ Otherwise, the document will be rejected with a security exception which looks l
|====== |======
| Name | Required | Default | Description | Name | Required | Default | Description
| `destination` | no | - | A static value for the target. Can't be set when the `dataset` or `namespace` option is set. | `destination` | no | - | A static value for the target. Can't be set when the `dataset` or `namespace` option is set.
| `type` | no | `{{data_stream.type}}` a| Field references or a static value for the type part of the data stream name. In addition to the criteria for <<indices-create-api-path-params, index names>>, cannot contain `-` and must be no longer than 100 characters. Example values are `logs` and `metrics`.
Supports field references with a mustache-like syntax (denoted as `{{double}}` or `{{{triple}}}` curly braces). When resolving field references, the processor replaces invalid characters with `_`. Uses the `<type>` part of the index name as a fallback if all field references resolve to a `null`, missing, or non-string value.
| `dataset` | no | `{{data_stream.dataset}}` a| Field references or a static value for the dataset part of the data stream name. In addition to the criteria for <<indices-create-api-path-params, index names>>, cannot contain `-` and must be no longer than 100 characters. Example values are `nginx.access` and `nginx.error`. | `dataset` | no | `{{data_stream.dataset}}` a| Field references or a static value for the dataset part of the data stream name. In addition to the criteria for <<indices-create-api-path-params, index names>>, cannot contain `-` and must be no longer than 100 characters. Example values are `nginx.access` and `nginx.error`.
Supports field references with a mustache-like syntax (denoted as `{{double}}` or `{{{triple}}}` curly braces). When resolving field references, the processor replaces invalid characters with `_`. Uses the `<dataset>` part of the index name as a fallback if all field references resolve to a `null`, missing, or non-string value. Supports field references with a mustache-like syntax (denoted as `{{double}}` or `{{{triple}}}` curly braces). When resolving field references, the processor replaces invalid characters with `_`. Uses the `<dataset>` part of the index name as a fallback if all field references resolve to a `null`, missing, or non-string value.

80
gradle/verification-metadata.xml
View file

@ -984,36 +984,19 @@
<sha256 value="e8c1c594e2425bdbea2d860de55c69b69fc5d59454452449a0f0913c2a5b8a31" origin="Generated by Gradle"/> <sha256 value="e8c1c594e2425bdbea2d860de55c69b69fc5d59454452449a0f0913c2a5b8a31" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="com.nimbusds" name="nimbus-jose-jwt" version="10.0.1">
<artifact name="nimbus-jose-jwt-10.0.1.jar">
<sha256 value="f28dbd9ab128324f05050d76b78469d3a9cd83e0319aabc68d1c276e3923e13a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.nimbusds" name="nimbus-jose-jwt" version="4.41.1"> <component group="com.nimbusds" name="nimbus-jose-jwt" version="4.41.1">
<artifact name="nimbus-jose-jwt-4.41.1.jar"> <artifact name="nimbus-jose-jwt-4.41.1.jar">
<sha256 value="fbfd0d5f2b2f86758b821daa5e79b5d7c965edd9dc1b2cc80b515df1c6ddc22d" origin="Generated by Gradle"/> <sha256 value="fbfd0d5f2b2f86758b821daa5e79b5d7c965edd9dc1b2cc80b515df1c6ddc22d" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="com.nimbusds" name="nimbus-jose-jwt" version="9.37.3"> <component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.22.2">
<artifact name="nimbus-jose-jwt-9.37.3.jar"> <artifact name="oauth2-oidc-sdk-11.22.2.jar">
<sha256 value="12ae4a3a260095d7aeba2adea7ae396e8b9570db8b7b409e09a824c219cc0444" origin="Generated by Gradle"> <sha256 value="64fab42f17bf8e0efb193dd34da716ef7abb7515234036119df1776b808dc066" origin="Generated by Gradle"/>
<also-trust value="afc63b689d881439b95f343b1dca750391edac63b87392be4d90d19c94ccafbe"/>
</sha256>
</artifact>
</component>
<component group="com.nimbusds" name="nimbus-jose-jwt" version="9.8.1">
<artifact name="nimbus-jose-jwt-9.8.1.jar">
<sha256 value="7664cf8c6f2adadf600287812b32878277beda54912eab9d4c2932cd50cb704a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.10.1">
<artifact name="oauth2-oidc-sdk-11.10.1.jar">
<sha256 value="9e51b2c17503cdd3eb97f41491c712aff7783bb3c67185d789f44ccf2a603b26" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.9.1">
<artifact name="oauth2-oidc-sdk-11.9.1.jar">
<sha256 value="0820c9690966304d075347b88e81ae490213440fc4d2c84f3d370d41941b2b9c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.nimbusds" name="oauth2-oidc-sdk" version="9.37">
<artifact name="oauth2-oidc-sdk-9.37.jar">
<sha256 value="44a04bbed5ae3f6d198aa73ee6b545c476e528ec1a267ef3e9f7033f886dd6fe" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="com.perforce" name="p4java" version="2015.2.1365273"> <component group="com.perforce" name="p4java" version="2015.2.1365273">
@ -1779,9 +1762,9 @@
<sha256 value="0972bbc99437c4163acd09b630e6c77eab4cfab8a9594621c95466c0c6645396" origin="Generated by Gradle"/> <sha256 value="0972bbc99437c4163acd09b630e6c77eab4cfab8a9594621c95466c0c6645396" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="net.minidev" name="accessors-smart" version="2.5.0"> <component group="net.minidev" name="accessors-smart" version="2.5.2">
<artifact name="accessors-smart-2.5.0.jar"> <artifact name="accessors-smart-2.5.2.jar">
<sha256 value="12314fc6881d66a413fd66370787adba16e504fbf7e138690b0f3952e3fbd321" origin="Generated by Gradle"/> <sha256 value="9b8a7bc43861d6156c021166d941fb7dddbe4463e2fa5ee88077e4b01452a836" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="net.minidev" name="json-smart" version="2.3"> <component group="net.minidev" name="json-smart" version="2.3">
@ -1789,24 +1772,14 @@
<sha256 value="903f48c8aa4c3f6426440b8d32de89fa1dc23b1169abde25e4e1d068aa67708b" origin="Generated by Gradle"/> <sha256 value="903f48c8aa4c3f6426440b8d32de89fa1dc23b1169abde25e4e1d068aa67708b" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="net.minidev" name="json-smart" version="2.4.10">
<artifact name="json-smart-2.4.10.jar">
<sha256 value="70cab5e9488630dc631b1fc6e7fa550d95cddd19ba14db39ceca7cabfbd4e5ae" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="net.minidev" name="json-smart" version="2.4.2"> <component group="net.minidev" name="json-smart" version="2.4.2">
<artifact name="json-smart-2.4.2.jar"> <artifact name="json-smart-2.4.2.jar">
<sha256 value="64072f56d9dff5040b2acec477c5d5e6bcebfc88c508f12acb26072d07942146" origin="Generated by Gradle"/> <sha256 value="64072f56d9dff5040b2acec477c5d5e6bcebfc88c508f12acb26072d07942146" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="net.minidev" name="json-smart" version="2.5.0"> <component group="net.minidev" name="json-smart" version="2.5.2">
<artifact name="json-smart-2.5.0.jar"> <artifact name="json-smart-2.5.2.jar">
<sha256 value="432b9e545848c4141b80717b26e367f83bf33f19250a228ce75da6e967da2bc7" origin="Generated by Gradle"/> <sha256 value="4fbdedb0105cedc7f766b95c297d2e88fb6a560da48f3bbaa0cc538ea8b7bf71" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="net.minidev" name="json-smart" version="2.5.1">
<artifact name="json-smart-2.5.1.jar">
<sha256 value="86c0c189581b79b57b0719f443a724e9f628ffbb9eef645cf79194f5973a1001" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="net.nextencia" name="rrdiagram" version="0.9.4"> <component group="net.nextencia" name="rrdiagram" version="0.9.4">
@ -4408,31 +4381,6 @@
<sha256 value="ca5b8d11569e53921b0e3486469e7c674361c79845dad3d514f38ab6e0c8c10a" origin="Generated by Gradle"/> <sha256 value="ca5b8d11569e53921b0e3486469e7c674361c79845dad3d514f38ab6e0c8c10a" origin="Generated by Gradle"/>
</artifact> </artifact>
</component> </component>
<component group="org.ow2.asm" name="asm" version="9.2">
<artifact name="asm-9.2.jar">
<sha256 value="b9d4fe4d71938df38839f0eca42aaaa64cf8b313d678da036f0cb3ca199b47f5" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.ow2.asm" name="asm" version="9.3">
<artifact name="asm-9.3.jar">
<sha256 value="1263369b59e29c943918de11d6d6152e2ec6085ce63e5710516f8c67d368e4bc" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.ow2.asm" name="asm" version="9.4">
<artifact name="asm-9.4.jar">
<sha256 value="39d0e2b3dc45af65a09b097945750a94a126e052e124f93468443a1d0e15f381" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.ow2.asm" name="asm" version="9.5">
<artifact name="asm-9.5.jar">
<sha256 value="b62e84b5980729751b0458c534cf1366f727542bb8d158621335682a460f0353" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.ow2.asm" name="asm" version="9.6">
<artifact name="asm-9.6.jar">
<sha256 value="3c6fac2424db3d4a853b669f4e3d1d9c3c552235e19a319673f887083c2303a1" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.ow2.asm" name="asm" version="9.7.1"> <component group="org.ow2.asm" name="asm" version="9.7.1">
<artifact name="asm-9.7.1.jar"> <artifact name="asm-9.7.1.jar">
<sha256 value="8cadd43ac5eb6d09de05faecca38b917a040bb9139c7edeb4cc81c740b713281" origin="Generated by Gradle"/> <sha256 value="8cadd43ac5eb6d09de05faecca38b917a040bb9139c7edeb4cc81c740b713281" origin="Generated by Gradle"/>

41
libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.entitlement.bridge; package org.elasticsearch.entitlement.bridge;
import java.io.File; import java.io.File;
import java.io.FileDescriptor;
import java.io.FileFilter; import java.io.FileFilter;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.io.InputStream; import java.io.InputStream;
@ -572,14 +573,54 @@ public interface EntitlementChecker {
void check$java_io_File$setWritable(Class<?> callerClass, File file, boolean writable, boolean ownerOnly); void check$java_io_File$setWritable(Class<?> callerClass, File file, boolean writable, boolean ownerOnly);
void check$java_io_FileInputStream$(Class<?> callerClass, File file);
void check$java_io_FileInputStream$(Class<?> callerClass, FileDescriptor fd);
void check$java_io_FileInputStream$(Class<?> callerClass, String name);
void check$java_io_FileOutputStream$(Class<?> callerClass, File file); void check$java_io_FileOutputStream$(Class<?> callerClass, File file);
void check$java_io_FileOutputStream$(Class<?> callerClass, File file, boolean append); void check$java_io_FileOutputStream$(Class<?> callerClass, File file, boolean append);
void check$java_io_FileOutputStream$(Class<?> callerClass, FileDescriptor fd);
void check$java_io_FileOutputStream$(Class<?> callerClass, String name); void check$java_io_FileOutputStream$(Class<?> callerClass, String name);
void check$java_io_FileOutputStream$(Class<?> callerClass, String name, boolean append); void check$java_io_FileOutputStream$(Class<?> callerClass, String name, boolean append);
void check$java_io_FileReader$(Class<?> callerClass, File file);
void check$java_io_FileReader$(Class<?> callerClass, File file, Charset charset);
void check$java_io_FileReader$(Class<?> callerClass, FileDescriptor fd);
void check$java_io_FileReader$(Class<?> callerClass, String name);
void check$java_io_FileReader$(Class<?> callerClass, String name, Charset charset);
void check$java_io_FileWriter$(Class<?> callerClass, File file);
void check$java_io_FileWriter$(Class<?> callerClass, File file, boolean append);
void check$java_io_FileWriter$(Class<?> callerClass, File file, Charset charset);
void check$java_io_FileWriter$(Class<?> callerClass, File file, Charset charset, boolean append);
void check$java_io_FileWriter$(Class<?> callerClass, FileDescriptor fd);
void check$java_io_FileWriter$(Class<?> callerClass, String name);
void check$java_io_FileWriter$(Class<?> callerClass, String name, boolean append);
void check$java_io_FileWriter$(Class<?> callerClass, String name, Charset charset);
void check$java_io_FileWriter$(Class<?> callerClass, String name, Charset charset, boolean append);
void check$java_io_RandomAccessFile$(Class<?> callerClass, String name, String mode);
void check$java_io_RandomAccessFile$(Class<?> callerClass, File file, String mode);
void check$java_util_Scanner$(Class<?> callerClass, File source); void check$java_util_Scanner$(Class<?> callerClass, File source);
void check$java_util_Scanner$(Class<?> callerClass, File source, String charsetName); void check$java_util_Scanner$(Class<?> callerClass, File source, String charsetName);

111
libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java
View file

@ -13,9 +13,14 @@ import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.entitlement.qa.entitled.EntitledActions; import org.elasticsearch.entitlement.qa.entitled.EntitledActions;
import java.io.File; import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -23,6 +28,7 @@ import java.nio.file.Paths;
import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipal;
import java.util.Scanner; import java.util.Scanner;
import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED;
import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS; import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS;
@SuppressForbidden(reason = "Explicitly checking APIs that are forbidden") @SuppressForbidden(reason = "Explicitly checking APIs that are forbidden")
@ -216,6 +222,21 @@ class FileCheckActions {
new Scanner(readFile().toFile(), "UTF-8"); new Scanner(readFile().toFile(), "UTF-8");
} }
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileInputStreamFile() throws IOException {
new FileInputStream(readFile().toFile()).close();
}
@EntitlementTest(expectedAccess = ALWAYS_DENIED)
static void createFileInputStreamFileDescriptor() throws IOException {
new FileInputStream(FileDescriptor.in).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileInputStreamString() throws IOException {
new FileInputStream(readFile().toString()).close();
}
@EntitlementTest(expectedAccess = PLUGINS) @EntitlementTest(expectedAccess = PLUGINS)
static void createFileOutputStreamString() throws IOException { static void createFileOutputStreamString() throws IOException {
new FileOutputStream(readWriteFile().toString()).close(); new FileOutputStream(readWriteFile().toString()).close();
@ -236,6 +257,96 @@ class FileCheckActions {
new FileOutputStream(readWriteFile().toFile(), false).close(); new FileOutputStream(readWriteFile().toFile(), false).close();
} }
@EntitlementTest(expectedAccess = ALWAYS_DENIED)
static void createFileOutputStreamFileDescriptor() throws IOException {
new FileOutputStream(FileDescriptor.out).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileReaderFile() throws IOException {
new FileReader(readFile().toFile()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileReaderFileCharset() throws IOException {
new FileReader(readFile().toFile(), StandardCharsets.UTF_8).close();
}
@EntitlementTest(expectedAccess = ALWAYS_DENIED)
static void createFileReaderFileDescriptor() throws IOException {
new FileReader(FileDescriptor.in).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileReaderString() throws IOException {
new FileReader(readFile().toString()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileReaderStringCharset() throws IOException {
new FileReader(readFile().toString(), StandardCharsets.UTF_8).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterFile() throws IOException {
new FileWriter(readWriteFile().toFile()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterFileWithAppend() throws IOException {
new FileWriter(readWriteFile().toFile(), false).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterFileCharsetWithAppend() throws IOException {
new FileWriter(readWriteFile().toFile(), StandardCharsets.UTF_8, false).close();
}
@EntitlementTest(expectedAccess = ALWAYS_DENIED)
static void createFileWriterFileDescriptor() throws IOException {
new FileWriter(FileDescriptor.out).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterString() throws IOException {
new FileWriter(readWriteFile().toString()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterStringWithAppend() throws IOException {
new FileWriter(readWriteFile().toString(), false).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterStringCharset() throws IOException {
new FileWriter(readWriteFile().toString(), StandardCharsets.UTF_8).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileWriterStringCharsetWithAppend() throws IOException {
new FileWriter(readWriteFile().toString(), StandardCharsets.UTF_8, false).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createRandomAccessFileStringRead() throws IOException {
new RandomAccessFile(readFile().toString(), "r").close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createRandomAccessFileStringReadWrite() throws IOException {
new RandomAccessFile(readWriteFile().toString(), "rw").close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createRandomAccessFileRead() throws IOException {
new RandomAccessFile(readFile().toFile(), "r").close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createRandomAccessFileReadWrite() throws IOException {
new RandomAccessFile(readWriteFile().toFile(), "rw").close();
}
@EntitlementTest(expectedAccess = PLUGINS) @EntitlementTest(expectedAccess = PLUGINS)
static void filesGetOwner() throws IOException { static void filesGetOwner() throws IOException {
Files.getOwner(readFile()); Files.getOwner(readFile());

27
libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
View file

@ -28,6 +28,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Map; import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Stream;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;
@ -36,19 +37,24 @@ public class EntitlementBootstrap {
public record BootstrapArgs( public record BootstrapArgs(
Map<String, Policy> pluginPolicies, Map<String, Policy> pluginPolicies,
Function<Class<?>, String> pluginResolver, Function<Class<?>, String> pluginResolver,
Function<String, String> settingResolver,
Function<String, Stream<String>> settingGlobResolver,
Path[] dataDirs, Path[] dataDirs,
Path configDir, Path configDir,
Path tempDir, Path logsDir,
Path logsDir Path tempDir
) { ) {
public BootstrapArgs { public BootstrapArgs {
requireNonNull(pluginPolicies); requireNonNull(pluginPolicies);
requireNonNull(pluginResolver); requireNonNull(pluginResolver);
requireNonNull(settingResolver);
requireNonNull(settingGlobResolver);
requireNonNull(dataDirs); requireNonNull(dataDirs);
if (dataDirs.length == 0) { if (dataDirs.length == 0) {
throw new IllegalArgumentException("must provide at least one data directory"); throw new IllegalArgumentException("must provide at least one data directory");
} }
requireNonNull(configDir); requireNonNull(configDir);
requireNonNull(logsDir);
requireNonNull(tempDir); requireNonNull(tempDir);
} }
} }
@ -73,16 +79,27 @@ public class EntitlementBootstrap {
public static void bootstrap( public static void bootstrap(
Map<String, Policy> pluginPolicies, Map<String, Policy> pluginPolicies,
Function<Class<?>, String> pluginResolver, Function<Class<?>, String> pluginResolver,
Function<String, String> settingResolver,
Function<String, Stream<String>> settingGlobResolver,
Path[] dataDirs, Path[] dataDirs,
Path configDir, Path configDir,
Path tempDir, Path logsDir,
Path logsDir Path tempDir
) { ) {
logger.debug("Loading entitlement agent"); logger.debug("Loading entitlement agent");
if (EntitlementBootstrap.bootstrapArgs != null) { if (EntitlementBootstrap.bootstrapArgs != null) {
throw new IllegalStateException("plugin data is already set"); throw new IllegalStateException("plugin data is already set");
} }
EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir, logsDir); EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(
pluginPolicies,
pluginResolver,
settingResolver,
settingGlobResolver,
dataDirs,
configDir,
logsDir,
tempDir
);
exportInitializationToAgent(); exportInitializationToAgent();
loadAgent(findAgentJar()); loadAgent(findAgentJar());
selfTest(); selfTest();

154
libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
View file

@ -53,6 +53,7 @@ import java.nio.file.attribute.FileAttribute;
import java.nio.file.spi.FileSystemProvider; import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -134,79 +135,93 @@ public class EntitlementInitialization {
private static PolicyManager createPolicyManager() { private static PolicyManager createPolicyManager() {
EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs(); EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies(); Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
var pathLookup = new PathLookup(getUserHome(), bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir()); var pathLookup = new PathLookup(
Path logsDir = EntitlementBootstrap.bootstrapArgs().logsDir(); getUserHome(),
bootstrapArgs.configDir(),
bootstrapArgs.dataDirs(),
bootstrapArgs.tempDir(),
bootstrapArgs.settingResolver(),
bootstrapArgs.settingGlobResolver()
);
// TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it List<Scope> serverScopes = new ArrayList<>();
var serverPolicy = new Policy( Collections.addAll(
"server", serverScopes,
List.of( new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())),
new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())), new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())),
new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())), new Scope(
new Scope( "org.elasticsearch.server",
"org.elasticsearch.server", List.of(
List.of( new ExitVMEntitlement(),
new ExitVMEntitlement(), new ReadStoreAttributesEntitlement(),
new ReadStoreAttributesEntitlement(), new CreateClassLoaderEntitlement(),
new CreateClassLoaderEntitlement(), new InboundNetworkEntitlement(),
new InboundNetworkEntitlement(), new OutboundNetworkEntitlement(),
new OutboundNetworkEntitlement(), new LoadNativeLibrariesEntitlement(),
new LoadNativeLibrariesEntitlement(), new ManageThreadsEntitlement(),
new ManageThreadsEntitlement(), new FilesEntitlement(
new FilesEntitlement( Stream.concat(
Stream.concat( Stream.of(
Stream.of( FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE),
FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE), FileData.ofPath(bootstrapArgs.configDir(), READ),
FileData.ofPath(bootstrapArgs.configDir(), READ), FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), // OS release on Linux
// OS release on Linux FileData.ofPath(Path.of("/etc/os-release"), READ),
FileData.ofPath(Path.of("/etc/os-release"), READ), FileData.ofPath(Path.of("/etc/system-release"), READ),
FileData.ofPath(Path.of("/etc/system-release"), READ), FileData.ofPath(Path.of("/usr/lib/os-release"), READ),
FileData.ofPath(Path.of("/usr/lib/os-release"), READ), // read max virtual memory areas
// read max virtual memory areas FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ),
FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ), FileData.ofPath(Path.of("/proc/meminfo"), READ),
FileData.ofPath(Path.of("/proc/meminfo"), READ), // load averages on Linux
// load averages on Linux FileData.ofPath(Path.of("/proc/loadavg"), READ),
FileData.ofPath(Path.of("/proc/loadavg"), READ), // control group stats on Linux. cgroup v2 stats are in an unpredicable
// control group stats on Linux. cgroup v2 stats are in an unpredicable // location under `/sys/fs/cgroup`, so unfortunately we have to allow
// location under `/sys/fs/cgroup`, so unfortunately we have to allow // read access to the entire directory hierarchy.
// read access to the entire directory hierarchy. FileData.ofPath(Path.of("/proc/self/cgroup"), READ),
FileData.ofPath(Path.of("/proc/self/cgroup"), READ), FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ),
FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ), // // io stats on Linux
// // io stats on Linux FileData.ofPath(Path.of("/proc/self/mountinfo"), READ),
FileData.ofPath(Path.of("/proc/self/mountinfo"), READ), FileData.ofPath(Path.of("/proc/diskstats"), READ)
FileData.ofPath(Path.of("/proc/diskstats"), READ) ),
), Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ))
Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ)) ).toList()
).toList()
)
) )
), )
new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), ),
new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())), new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())),
new Scope( new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())),
"org.apache.lucene.core", new Scope(
List.of( "org.apache.lucene.core",
new LoadNativeLibrariesEntitlement(), List.of(
new ManageThreadsEntitlement(), new LoadNativeLibrariesEntitlement(),
new FilesEntitlement( new ManageThreadsEntitlement(),
Stream.concat( new FilesEntitlement(
Stream.of(FileData.ofPath(bootstrapArgs.configDir(), READ)), Stream.concat(
Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ_WRITE)) Stream.of(FileData.ofPath(bootstrapArgs.configDir(), READ)),
).toList() Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ_WRITE))
) ).toList()
)
),
new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())),
new Scope(
"org.elasticsearch.nativeaccess",
List.of(
new LoadNativeLibrariesEntitlement(),
new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
) )
) )
),
new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())),
new Scope(
"org.elasticsearch.nativeaccess",
List.of(
new LoadNativeLibrariesEntitlement(),
new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
)
) )
); );
Path trustStorePath = trustStorePath();
if (trustStorePath != null) {
serverScopes.add(
new Scope("org.bouncycastle.fips.tls", List.of(new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ)))))
);
}
// TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
var serverPolicy = new Policy("server", serverScopes);
// agents run without a module, so this is a special hack for the apm agent // agents run without a module, so this is a special hack for the apm agent
// this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed
List<Entitlement> agentEntitlements = List.of(new CreateClassLoaderEntitlement(), new ManageThreadsEntitlement()); List<Entitlement> agentEntitlements = List.of(new CreateClassLoaderEntitlement(), new ManageThreadsEntitlement());
@ -230,6 +245,11 @@ public class EntitlementInitialization {
return PathUtils.get(userHome); return PathUtils.get(userHome);
} }
private static Path trustStorePath() {
String trustStore = System.getProperty("javax.net.ssl.trustStore");
return trustStore != null ? Path.of(trustStore) : null;
}
private static Stream<InstrumentationService.InstrumentationInfo> fileSystemProviderChecks() throws ClassNotFoundException, private static Stream<InstrumentationService.InstrumentationInfo> fileSystemProviderChecks() throws ClassNotFoundException,
NoSuchMethodException { NoSuchMethodException {
var fileSystemProviderClass = FileSystems.getDefault().provider().getClass(); var fileSystemProviderClass = FileSystems.getDefault().provider().getClass();

109
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java
View file

@ -14,6 +14,7 @@ import org.elasticsearch.entitlement.bridge.EntitlementChecker;
import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
import java.io.File; import java.io.File;
import java.io.FileDescriptor;
import java.io.FileFilter; import java.io.FileFilter;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.io.IOException; import java.io.IOException;
@ -1103,6 +1104,21 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker {
policyManager.checkFileWrite(callerClass, file); policyManager.checkFileWrite(callerClass, file);
} }
@Override
public void check$java_io_FileInputStream$(Class<?> callerClass, File file) {
policyManager.checkFileRead(callerClass, file);
}
@Override
public void check$java_io_FileInputStream$(Class<?> callerClass, FileDescriptor fd) {
policyManager.checkFileDescriptorRead(callerClass);
}
@Override
public void check$java_io_FileInputStream$(Class<?> callerClass, String name) {
policyManager.checkFileRead(callerClass, new File(name));
}
@Override @Override
public void check$java_io_FileOutputStream$(Class<?> callerClass, String name) { public void check$java_io_FileOutputStream$(Class<?> callerClass, String name) {
policyManager.checkFileWrite(callerClass, new File(name)); policyManager.checkFileWrite(callerClass, new File(name));
@ -1123,6 +1139,99 @@ public class ElasticsearchEntitlementChecker implements EntitlementChecker {
policyManager.checkFileWrite(callerClass, file); policyManager.checkFileWrite(callerClass, file);
} }
@Override
public void check$java_io_FileOutputStream$(Class<?> callerClass, FileDescriptor fd) {
policyManager.checkFileDescriptorWrite(callerClass);
}
@Override
public void check$java_io_FileReader$(Class<?> callerClass, File file) {
policyManager.checkFileRead(callerClass, file);
}
@Override
public void check$java_io_FileReader$(Class<?> callerClass, File file, Charset charset) {
policyManager.checkFileRead(callerClass, file);
}
@Override
public void check$java_io_FileReader$(Class<?> callerClass, FileDescriptor fd) {
policyManager.checkFileDescriptorRead(callerClass);
}
@Override
public void check$java_io_FileReader$(Class<?> callerClass, String name) {
policyManager.checkFileRead(callerClass, new File(name));
}
@Override
public void check$java_io_FileReader$(Class<?> callerClass, String name, Charset charset) {
policyManager.checkFileRead(callerClass, new File(name));
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, File file) {
policyManager.checkFileWrite(callerClass, file);
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, File file, boolean append) {
policyManager.checkFileWrite(callerClass, file);
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, File file, Charset charset) {
policyManager.checkFileWrite(callerClass, file);
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, File file, Charset charset, boolean append) {
policyManager.checkFileWrite(callerClass, file);
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, FileDescriptor fd) {
policyManager.checkFileDescriptorWrite(callerClass);
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, String name) {
policyManager.checkFileWrite(callerClass, new File(name));
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, String name, boolean append) {
policyManager.checkFileWrite(callerClass, new File(name));
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, String name, Charset charset) {
policyManager.checkFileWrite(callerClass, new File(name));
}
@Override
public void check$java_io_FileWriter$(Class<?> callerClass, String name, Charset charset, boolean append) {
policyManager.checkFileWrite(callerClass, new File(name));
}
@Override
public void check$java_io_RandomAccessFile$(Class<?> callerClass, String name, String mode) {
if (mode.equals("r")) {
policyManager.checkFileRead(callerClass, new File(name));
} else {
policyManager.checkFileWrite(callerClass, new File(name));
}
}
@Override
public void check$java_io_RandomAccessFile$(Class<?> callerClass, File file, String mode) {
if (mode.equals("r")) {
policyManager.checkFileRead(callerClass, file);
} else {
policyManager.checkFileWrite(callerClass, file);
}
}
@Override @Override
public void check$java_util_Scanner$(Class<?> callerClass, File source) { public void check$java_util_Scanner$(Class<?> callerClass, File source) {
policyManager.checkFileRead(callerClass, source); policyManager.checkFileRead(callerClass, source);

5
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java
View file

@ -42,8 +42,9 @@ public final class FileAccessTree {
} }
// everything has access to the temp dir // everything has access to the temp dir
readPaths.add(pathLookup.tempDir().toString()); String tempDir = normalizePath(pathLookup.tempDir());
writePaths.add(pathLookup.tempDir().toString()); readPaths.add(tempDir);
writePaths.add(tempDir);
readPaths.sort(String::compareTo); readPaths.sort(String::compareTo);
writePaths.sort(String::compareTo); writePaths.sort(String::compareTo);

11
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java
View file

@ -10,5 +10,14 @@
package org.elasticsearch.entitlement.runtime.policy; package org.elasticsearch.entitlement.runtime.policy;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.function.Function;
import java.util.stream.Stream;
public record PathLookup(Path homeDir, Path configDir, Path[] dataDirs, Path tempDir) {} public record PathLookup(
Path homeDir,
Path configDir,
Path[] dataDirs,
Path tempDir,
Function<String, String> settingResolver,
Function<String, Stream<String>> settingGlobResolver
) {}

8
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
View file

@ -304,6 +304,14 @@ public class PolicyManager {
} }
} }
public void checkFileDescriptorRead(Class<?> callerClass) {
neverEntitled(callerClass, () -> "read file descriptor");
}
public void checkFileDescriptorWrite(Class<?> callerClass) {
neverEntitled(callerClass, () -> "write file descriptor");
}
/** /**
* Invoked when we try to get an arbitrary {@code FileAttributeView} class. Such a class can modify attributes, like owner etc.; * Invoked when we try to get an arbitrary {@code FileAttributeView} class. Such a class can modify attributes, like owner etc.;
* we could think about introducing checks for each of the operations, but for now we over-approximate this and simply deny when it is * we could think about introducing checks for each of the operations, but for now we over-approximate this and simply deny when it is

110
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java
View file

@ -15,7 +15,6 @@ import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -53,6 +52,43 @@ public record FilesEntitlement(List<FileData> filesData) implements Entitlement
static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) { static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) {
return new RelativePathFileData(relativePath, baseDir, mode); return new RelativePathFileData(relativePath, baseDir, mode);
} }
static FileData ofPathSetting(String setting, Mode mode) {
return new PathSettingFileData(setting, mode);
}
static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode) {
return new RelativePathSettingFileData(setting, baseDir, mode);
}
}
private sealed interface RelativeFileData extends FileData {
BaseDir baseDir();
Stream<Path> resolveRelativePaths(PathLookup pathLookup);
@Override
default Stream<Path> resolvePaths(PathLookup pathLookup) {
Objects.requireNonNull(pathLookup);
var relativePaths = resolveRelativePaths(pathLookup);
switch (baseDir()) {
case CONFIG:
return relativePaths.map(relativePath -> pathLookup.configDir().resolve(relativePath));
case DATA:
// multiple data dirs are a pain...we need the combination of relative paths and data dirs
List<Path> paths = new ArrayList<>();
for (var relativePath : relativePaths.toList()) {
for (var dataDir : pathLookup.dataDirs()) {
paths.add(dataDir.resolve(relativePath));
}
}
return paths.stream();
case HOME:
return relativePaths.map(relativePath -> pathLookup.homeDir().resolve(relativePath));
default:
throw new IllegalArgumentException();
}
}
} }
private record AbsolutePathFileData(Path path, Mode mode) implements FileData { private record AbsolutePathFileData(Path path, Mode mode) implements FileData {
@ -62,24 +98,35 @@ public record FilesEntitlement(List<FileData> filesData) implements Entitlement
} }
} }
private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode) implements FileData { private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode) implements FileData, RelativeFileData {
@Override
public Stream<Path> resolveRelativePaths(PathLookup pathLookup) {
return Stream.of(relativePath);
}
}
private record PathSettingFileData(String setting, Mode mode) implements FileData {
@Override @Override
public Stream<Path> resolvePaths(PathLookup pathLookup) { public Stream<Path> resolvePaths(PathLookup pathLookup) {
Objects.requireNonNull(pathLookup); return resolvePathSettings(pathLookup, setting);
switch (baseDir) {
case CONFIG:
return Stream.of(pathLookup.configDir().resolve(relativePath));
case DATA:
return Arrays.stream(pathLookup.dataDirs()).map(d -> d.resolve(relativePath));
case HOME:
return Stream.of(pathLookup.homeDir().resolve(relativePath));
default:
throw new IllegalArgumentException();
}
} }
} }
private record RelativePathSettingFileData(String setting, BaseDir baseDir, Mode mode) implements FileData, RelativeFileData {
@Override
public Stream<Path> resolveRelativePaths(PathLookup pathLookup) {
return resolvePathSettings(pathLookup, setting);
}
}
private static Stream<Path> resolvePathSettings(PathLookup pathLookup, String setting) {
if (setting.contains("*")) {
return pathLookup.settingGlobResolver().apply(setting).map(Path::of);
}
String path = pathLookup.settingResolver().apply(setting);
return path == null ? Stream.of() : Stream.of(Path.of(path));
}
private static Mode parseMode(String mode) { private static Mode parseMode(String mode) {
if (mode.equals("read")) { if (mode.equals("read")) {
return Mode.READ; return Mode.READ;
@ -113,37 +160,56 @@ public record FilesEntitlement(List<FileData> filesData) implements Entitlement
String pathAsString = file.remove("path"); String pathAsString = file.remove("path");
String relativePathAsString = file.remove("relative_path"); String relativePathAsString = file.remove("relative_path");
String relativeTo = file.remove("relative_to"); String relativeTo = file.remove("relative_to");
String mode = file.remove("mode"); String pathSetting = file.remove("path_setting");
String relativePathSetting = file.remove("relative_path_setting");
String modeAsString = file.remove("mode");
if (file.isEmpty() == false) { if (file.isEmpty() == false) {
throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement"); throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement");
} }
if (mode == null) { int foundKeys = (pathAsString != null ? 1 : 0) + (relativePathAsString != null ? 1 : 0) + (pathSetting != null ? 1 : 0)
+ (relativePathSetting != null ? 1 : 0);
if (foundKeys != 1) {
throw new PolicyValidationException(
"a files entitlement entry must contain one of " + "[path, relative_path, path_setting, relative_path_setting]"
);
}
if (modeAsString == null) {
throw new PolicyValidationException("files entitlement must contain 'mode' for every listed file"); throw new PolicyValidationException("files entitlement must contain 'mode' for every listed file");
} }
if (pathAsString != null && relativePathAsString != null) { Mode mode = parseMode(modeAsString);
throw new PolicyValidationException("a files entitlement entry cannot contain both 'path' and 'relative_path'");
BaseDir baseDir = null;
if (relativeTo != null) {
baseDir = parseBaseDir(relativeTo);
} }
if (relativePathAsString != null) { if (relativePathAsString != null) {
if (relativeTo == null) { if (baseDir == null) {
throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'"); throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'");
} }
final BaseDir baseDir = parseBaseDir(relativeTo);
Path relativePath = Path.of(relativePathAsString); Path relativePath = Path.of(relativePathAsString);
if (relativePath.isAbsolute()) { if (relativePath.isAbsolute()) {
throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative"); throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative");
} }
filesData.add(FileData.ofRelativePath(relativePath, baseDir, parseMode(mode))); filesData.add(FileData.ofRelativePath(relativePath, baseDir, mode));
} else if (pathAsString != null) { } else if (pathAsString != null) {
Path path = Path.of(pathAsString); Path path = Path.of(pathAsString);
if (path.isAbsolute() == false) { if (path.isAbsolute() == false) {
throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute"); throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute");
} }
filesData.add(FileData.ofPath(path, parseMode(mode))); filesData.add(FileData.ofPath(path, mode));
} else if (pathSetting != null) {
filesData.add(FileData.ofPathSetting(pathSetting, mode));
} else if (relativePathSetting != null) {
if (baseDir == null) {
throw new PolicyValidationException("files entitlement with a 'relative_path_setting' must specify 'relative_to'");
}
filesData.add(FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode));
} else { } else {
throw new PolicyValidationException("files entitlement must contain either 'path' or 'relative_path' for every entry"); throw new AssertionError("File entry validation error");
} }
} }
return new FilesEntitlement(filesData); return new FilesEntitlement(filesData);

17
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.entitlement.runtime.policy; package org.elasticsearch.entitlement.runtime.policy;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -25,10 +26,12 @@ import static org.hamcrest.Matchers.is;
public class FileAccessTreeTests extends ESTestCase { public class FileAccessTreeTests extends ESTestCase {
static Path root; static Path root;
static Settings settings;
@BeforeClass @BeforeClass
public static void setupRoot() { public static void setupRoot() {
root = createTempDir(); root = createTempDir();
settings = Settings.EMPTY;
} }
private static Path path(String s) { private static Path path(String s) {
@ -39,7 +42,9 @@ public class FileAccessTreeTests extends ESTestCase {
Path.of("/home"), Path.of("/home"),
Path.of("/config"), Path.of("/config"),
new Path[] { Path.of("/data1"), Path.of("/data2") }, new Path[] { Path.of("/data1"), Path.of("/data2") },
Path.of("/tmp") Path.of("/tmp"),
setting -> settings.get(setting),
glob -> settings.getGlobValues(glob)
); );
public void testEmpty() { public void testEmpty() {
@ -163,13 +168,9 @@ public class FileAccessTreeTests extends ESTestCase {
} }
public void testTempDirAccess() { public void testTempDirAccess() {
Path tempDir = createTempDir(); var tree = FileAccessTree.of(FilesEntitlement.EMPTY, TEST_PATH_LOOKUP);
var tree = FileAccessTree.of( assertThat(tree.canRead(TEST_PATH_LOOKUP.tempDir()), is(true));
FilesEntitlement.EMPTY, assertThat(tree.canWrite(TEST_PATH_LOOKUP.tempDir()), is(true));
new PathLookup(Path.of("/home"), Path.of("/config"), new Path[] { Path.of("/data1"), Path.of("/data2") }, tempDir)
);
assertThat(tree.canRead(tempDir), is(true));
assertThat(tree.canWrite(tempDir), is(true));
} }
FileAccessTree accessTree(FilesEntitlement entitlement) { FileAccessTree accessTree(FilesEntitlement entitlement) {

5
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.entitlement.runtime.policy; package org.elasticsearch.entitlement.runtime.policy;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements; import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements;
import org.elasticsearch.entitlement.runtime.policy.agent.TestAgent; import org.elasticsearch.entitlement.runtime.policy.agent.TestAgent;
import org.elasticsearch.entitlement.runtime.policy.agent.inner.TestInnerAgent; import org.elasticsearch.entitlement.runtime.policy.agent.inner.TestInnerAgent;
@ -68,7 +69,9 @@ public class PolicyManagerTests extends ESTestCase {
TEST_BASE_DIR.resolve("/user/home"), TEST_BASE_DIR.resolve("/user/home"),
TEST_BASE_DIR.resolve("/config"), TEST_BASE_DIR.resolve("/config"),
new Path[] { TEST_BASE_DIR.resolve("/data1/"), TEST_BASE_DIR.resolve("/data2") }, new Path[] { TEST_BASE_DIR.resolve("/data1/"), TEST_BASE_DIR.resolve("/data2") },
TEST_BASE_DIR.resolve("/temp") TEST_BASE_DIR.resolve("/temp"),
Settings.EMPTY::get,
Settings.EMPTY::getGlobValues
); );
} catch (Exception e) { } catch (Exception e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);

6
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java
View file

@ -74,7 +74,8 @@ public class PolicyParserFailureTests extends ESTestCase {
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals( assertEquals(
"[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+ "for entitlement type [files]: a files entitlement entry cannot contain both 'path' and 'relative_path'", + "for entitlement type [files]: a files entitlement entry must contain one of "
+ "[path, relative_path, path_setting, relative_path_setting]",
ppe.getMessage() ppe.getMessage()
); );
} }
@ -87,7 +88,8 @@ public class PolicyParserFailureTests extends ESTestCase {
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
assertEquals( assertEquals(
"[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
+ "for entitlement type [files]: files entitlement must contain either 'path' or 'relative_path' for every entry", + "for entitlement type [files]: a files entitlement entry must contain one of "
+ "[path, relative_path, path_setting, relative_path_setting]",
ppe.getMessage() ppe.getMessage()
); );
} }

9
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
View file

@ -182,6 +182,11 @@ public class PolicyParserTests extends ESTestCase {
mode: "read" mode: "read"
- path: '%s' - path: '%s'
mode: "read_write" mode: "read_write"
- path_setting: foo.bar
mode: read
- relative_path_setting: foo.bar
relative_to: config
mode: read
""", relativePathToFile, relativePathToDir, TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8)), """, relativePathToFile, relativePathToDir, TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8)),
"test-policy.yaml", "test-policy.yaml",
false false
@ -196,7 +201,9 @@ public class PolicyParserTests extends ESTestCase {
List.of( List.of(
Map.of("relative_path", relativePathToFile, "mode", "read_write", "relative_to", "data"), Map.of("relative_path", relativePathToFile, "mode", "read_write", "relative_to", "data"),
Map.of("relative_path", relativePathToDir, "mode", "read", "relative_to", "config"), Map.of("relative_path", relativePathToDir, "mode", "read", "relative_to", "config"),
Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write") Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"),
Map.of("path_setting", "foo.bar", "mode", "read"),
Map.of("relative_path_setting", "foo.bar", "relative_to", "config", "mode", "read")
) )
) )
) )

50
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlementTests.java
View file

@ -9,20 +9,42 @@
package org.elasticsearch.entitlement.runtime.policy.entitlements; package org.elasticsearch.entitlement.runtime.policy.entitlements;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.entitlement.runtime.policy.PathLookup; import org.elasticsearch.entitlement.runtime.policy.PathLookup;
import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException; import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException;
import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.BeforeClass;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE; import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
public class FilesEntitlementTests extends ESTestCase { public class FilesEntitlementTests extends ESTestCase {
static Settings settings;
@BeforeClass
public static void setupRoot() {
settings = Settings.EMPTY;
}
private static final PathLookup TEST_PATH_LOOKUP = new PathLookup(
Path.of("home"),
Path.of("/config"),
new Path[] { Path.of("/data1"), Path.of("/data2") },
Path.of("/tmp"),
setting -> settings.get(setting),
glob -> settings.getGlobValues(glob)
);
public void testEmptyBuild() { public void testEmptyBuild() {
PolicyValidationException pve = expectThrows(PolicyValidationException.class, () -> FilesEntitlement.build(List.of())); PolicyValidationException pve = expectThrows(PolicyValidationException.class, () -> FilesEntitlement.build(List.of()));
assertEquals("must specify at least one path", pve.getMessage()); assertEquals("must specify at least one path", pve.getMessage());
@ -39,10 +61,30 @@ public class FilesEntitlementTests extends ESTestCase {
} }
public void testFileDataRelativeWithEmptyDirectory() { public void testFileDataRelativeWithEmptyDirectory() {
var fileData = FilesEntitlement.FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE); var fileData = FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE);
var dataDirs = fileData.resolvePaths( var dataDirs = fileData.resolvePaths(TEST_PATH_LOOKUP);
new PathLookup(Path.of("/home"), Path.of("/config"), new Path[] { Path.of("/data1/"), Path.of("/data2") }, Path.of("/temp"))
);
assertThat(dataDirs.toList(), contains(Path.of("/data1/"), Path.of("/data2"))); assertThat(dataDirs.toList(), contains(Path.of("/data1/"), Path.of("/data2")));
} }
public void testPathSettingResolve() {
var entitlement = FilesEntitlement.build(List.of(Map.of("path_setting", "foo.bar", "mode", "read")));
var filesData = entitlement.filesData();
assertThat(filesData, contains(FileData.ofPathSetting("foo.bar", READ)));
var fileData = FileData.ofPathSetting("foo.bar", READ);
// empty settings
assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), empty());
fileData = FileData.ofPathSetting("foo.bar", READ);
settings = Settings.builder().put("foo.bar", "/setting/path").build();
assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path")));
fileData = FileData.ofPathSetting("foo.*.bar", READ);
settings = Settings.builder().put("foo.baz.bar", "/setting/path").build();
assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), contains(Path.of("/setting/path")));
fileData = FileData.ofPathSetting("foo.*.bar", READ);
settings = Settings.builder().put("foo.baz.bar", "/setting/path").put("foo.baz2.bar", "/other/path").build();
assertThat(fileData.resolvePaths(TEST_PATH_LOOKUP).toList(), containsInAnyOrder(Path.of("/setting/path"), Path.of("/other/path")));
}
} }

34
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java
View file

@ -26,6 +26,7 @@ import static org.elasticsearch.core.Strings.format;
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
import static org.elasticsearch.ingest.common.RerouteProcessor.DataStreamValueSource.DATASET_VALUE_SOURCE; import static org.elasticsearch.ingest.common.RerouteProcessor.DataStreamValueSource.DATASET_VALUE_SOURCE;
import static org.elasticsearch.ingest.common.RerouteProcessor.DataStreamValueSource.NAMESPACE_VALUE_SOURCE; import static org.elasticsearch.ingest.common.RerouteProcessor.DataStreamValueSource.NAMESPACE_VALUE_SOURCE;
import static org.elasticsearch.ingest.common.RerouteProcessor.DataStreamValueSource.TYPE_VALUE_SOURCE;
public final class RerouteProcessor extends AbstractProcessor { public final class RerouteProcessor extends AbstractProcessor {
@ -39,6 +40,7 @@ public final class RerouteProcessor extends AbstractProcessor {
private static final String DATA_STREAM_DATASET = DATA_STREAM_PREFIX + "dataset"; private static final String DATA_STREAM_DATASET = DATA_STREAM_PREFIX + "dataset";
private static final String DATA_STREAM_NAMESPACE = DATA_STREAM_PREFIX + "namespace"; private static final String DATA_STREAM_NAMESPACE = DATA_STREAM_PREFIX + "namespace";
private static final String EVENT_DATASET = "event.dataset"; private static final String EVENT_DATASET = "event.dataset";
private final List<DataStreamValueSource> type;
private final List<DataStreamValueSource> dataset; private final List<DataStreamValueSource> dataset;
private final List<DataStreamValueSource> namespace; private final List<DataStreamValueSource> namespace;
private final String destination; private final String destination;
@ -46,11 +48,17 @@ public final class RerouteProcessor extends AbstractProcessor {
RerouteProcessor( RerouteProcessor(
String tag, String tag,
String description, String description,
List<DataStreamValueSource> type,
List<DataStreamValueSource> dataset, List<DataStreamValueSource> dataset,
List<DataStreamValueSource> namespace, List<DataStreamValueSource> namespace,
String destination String destination
) { ) {
super(tag, description); super(tag, description);
if (type.isEmpty()) {
this.type = List.of(TYPE_VALUE_SOURCE);
} else {
this.type = type;
}
if (dataset.isEmpty()) { if (dataset.isEmpty()) {
this.dataset = List.of(DATASET_VALUE_SOURCE); this.dataset = List.of(DATASET_VALUE_SOURCE);
} else { } else {
@ -71,7 +79,7 @@ public final class RerouteProcessor extends AbstractProcessor {
return ingestDocument; return ingestDocument;
} }
final String indexName = ingestDocument.getFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), String.class); final String indexName = ingestDocument.getFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), String.class);
final String type; final String currentType;
final String currentDataset; final String currentDataset;
final String currentNamespace; final String currentNamespace;
@ -84,10 +92,11 @@ public final class RerouteProcessor extends AbstractProcessor {
if (indexOfSecondDash < 0) { if (indexOfSecondDash < 0) {
throw new IllegalArgumentException(format(NAMING_SCHEME_ERROR_MESSAGE, indexName)); throw new IllegalArgumentException(format(NAMING_SCHEME_ERROR_MESSAGE, indexName));
} }
type = parseDataStreamType(indexName, indexOfFirstDash); currentType = parseDataStreamType(indexName, indexOfFirstDash);
currentDataset = parseDataStreamDataset(indexName, indexOfFirstDash, indexOfSecondDash); currentDataset = parseDataStreamDataset(indexName, indexOfFirstDash, indexOfSecondDash);
currentNamespace = parseDataStreamNamespace(indexName, indexOfSecondDash); currentNamespace = parseDataStreamNamespace(indexName, indexOfSecondDash);
String type = determineDataStreamField(ingestDocument, this.type, currentType);
String dataset = determineDataStreamField(ingestDocument, this.dataset, currentDataset); String dataset = determineDataStreamField(ingestDocument, this.dataset, currentDataset);
String namespace = determineDataStreamField(ingestDocument, this.namespace, currentNamespace); String namespace = determineDataStreamField(ingestDocument, this.namespace, currentNamespace);
String newTarget = type + "-" + dataset + "-" + namespace; String newTarget = type + "-" + dataset + "-" + namespace;
@ -168,6 +177,15 @@ public final class RerouteProcessor extends AbstractProcessor {
String description, String description,
Map<String, Object> config Map<String, Object> config
) throws Exception { ) throws Exception {
List<DataStreamValueSource> type;
try {
type = ConfigurationUtils.readOptionalListOrString(TYPE, tag, config, "type")
.stream()
.map(DataStreamValueSource::type)
.toList();
} catch (IllegalArgumentException e) {
throw newConfigurationException(TYPE, tag, "type", e.getMessage());
}
List<DataStreamValueSource> dataset; List<DataStreamValueSource> dataset;
try { try {
dataset = ConfigurationUtils.readOptionalListOrString(TYPE, tag, config, "dataset") dataset = ConfigurationUtils.readOptionalListOrString(TYPE, tag, config, "dataset")
@ -188,11 +206,11 @@ public final class RerouteProcessor extends AbstractProcessor {
} }
String destination = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "destination"); String destination = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "destination");
if (destination != null && (dataset.isEmpty() == false || namespace.isEmpty() == false)) { if (destination != null && (type.isEmpty() == false || dataset.isEmpty() == false || namespace.isEmpty() == false)) {
throw newConfigurationException(TYPE, tag, "destination", "can only be set if dataset and namespace are not set"); throw newConfigurationException(TYPE, tag, "destination", "can only be set if type, dataset, and namespace are not set");
} }
return new RerouteProcessor(tag, description, dataset, namespace, destination); return new RerouteProcessor(tag, description, type, dataset, namespace, destination);
} }
} }
@ -203,8 +221,10 @@ public final class RerouteProcessor extends AbstractProcessor {
private static final int MAX_LENGTH = 100; private static final int MAX_LENGTH = 100;
private static final String REPLACEMENT = "_"; private static final String REPLACEMENT = "_";
private static final Pattern DISALLOWED_IN_TYPE = Pattern.compile("[\\\\/*?\"<>| ,#:-]");
private static final Pattern DISALLOWED_IN_DATASET = Pattern.compile("[\\\\/*?\"<>| ,#:-]"); private static final Pattern DISALLOWED_IN_DATASET = Pattern.compile("[\\\\/*?\"<>| ,#:-]");
private static final Pattern DISALLOWED_IN_NAMESPACE = Pattern.compile("[\\\\/*?\"<>| ,#:]"); private static final Pattern DISALLOWED_IN_NAMESPACE = Pattern.compile("[\\\\/*?\"<>| ,#:]");
static final DataStreamValueSource TYPE_VALUE_SOURCE = type("{{" + DATA_STREAM_TYPE + "}}");
static final DataStreamValueSource DATASET_VALUE_SOURCE = dataset("{{" + DATA_STREAM_DATASET + "}}"); static final DataStreamValueSource DATASET_VALUE_SOURCE = dataset("{{" + DATA_STREAM_DATASET + "}}");
static final DataStreamValueSource NAMESPACE_VALUE_SOURCE = namespace("{{" + DATA_STREAM_NAMESPACE + "}}"); static final DataStreamValueSource NAMESPACE_VALUE_SOURCE = namespace("{{" + DATA_STREAM_NAMESPACE + "}}");
@ -212,6 +232,10 @@ public final class RerouteProcessor extends AbstractProcessor {
private final String fieldReference; private final String fieldReference;
private final Function<String, String> sanitizer; private final Function<String, String> sanitizer;
public static DataStreamValueSource type(String type) {
return new DataStreamValueSource(type, ds -> sanitizeDataStreamField(ds, DISALLOWED_IN_TYPE));
}
public static DataStreamValueSource dataset(String dataset) { public static DataStreamValueSource dataset(String dataset) {
return new DataStreamValueSource(dataset, ds -> sanitizeDataStreamField(ds, DISALLOWED_IN_DATASET)); return new DataStreamValueSource(dataset, ds -> sanitizeDataStreamField(ds, DISALLOWED_IN_DATASET));
} }

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java
View file

@ -47,7 +47,7 @@ public class RerouteProcessorFactoryTests extends ESTestCase {
ElasticsearchParseException.class, ElasticsearchParseException.class,
() -> create(Map.of("destination", "foo", "dataset", "bar")) () -> create(Map.of("destination", "foo", "dataset", "bar"))
); );
assertThat(e.getMessage(), equalTo("[destination] can only be set if dataset and namespace are not set")); assertThat(e.getMessage(), equalTo("[destination] can only be set if type, dataset, and namespace are not set"));
} }
public void testFieldReference() throws Exception { public void testFieldReference() throws Exception {

71
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java
View file

@ -27,16 +27,25 @@ public class RerouteProcessorTests extends ESTestCase {
public void testDefaults() throws Exception { public void testDefaults() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of(), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "generic", "default"); assertDataSetFields(ingestDocument, "logs", "generic", "default");
} }
public void testRouteOnType() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.setFieldValue("event.type", "foo");
RerouteProcessor processor = createRerouteProcessor(List.of("{{event.type}}"), List.of(), List.of());
processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "foo", "generic", "default");
}
public void testEventDataset() throws Exception { public void testEventDataset() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.setFieldValue("event.dataset", "foo"); ingestDocument.setFieldValue("event.dataset", "foo");
RerouteProcessor processor = createRerouteProcessor(List.of("{{event.dataset }}"), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{event.dataset }}"), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo", "default"); assertDataSetFields(ingestDocument, "logs", "foo", "default");
assertThat(ingestDocument.getFieldValue("event.dataset", String.class), equalTo("foo")); assertThat(ingestDocument.getFieldValue("event.dataset", String.class), equalTo("foo"));
@ -46,7 +55,7 @@ public class RerouteProcessorTests extends ESTestCase {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.getCtxMap().put("event.dataset", "foo"); ingestDocument.getCtxMap().put("event.dataset", "foo");
RerouteProcessor processor = createRerouteProcessor(List.of("{{ event.dataset}}"), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{ event.dataset}}"), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo", "default"); assertDataSetFields(ingestDocument, "logs", "foo", "default");
assertThat(ingestDocument.getCtxMap().get("event.dataset"), equalTo("foo")); assertThat(ingestDocument.getCtxMap().get("event.dataset"), equalTo("foo"));
@ -57,7 +66,7 @@ public class RerouteProcessorTests extends ESTestCase {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.setFieldValue("ds", "foo"); ingestDocument.setFieldValue("ds", "foo");
RerouteProcessor processor = createRerouteProcessor(List.of("{{ ds }}"), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{ ds }}"), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo", "default"); assertDataSetFields(ingestDocument, "logs", "foo", "default");
assertFalse(ingestDocument.hasField("event.dataset")); assertFalse(ingestDocument.hasField("event.dataset"));
@ -66,8 +75,8 @@ public class RerouteProcessorTests extends ESTestCase {
public void testSkipFirstProcessor() throws Exception { public void testSkipFirstProcessor() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor skippedProcessor = createRerouteProcessor(List.of("skip"), List.of()); RerouteProcessor skippedProcessor = createRerouteProcessor(List.of(), List.of("skip"), List.of());
RerouteProcessor executedProcessor = createRerouteProcessor(List.of("executed"), List.of()); RerouteProcessor executedProcessor = createRerouteProcessor(List.of(), List.of("executed"), List.of());
CompoundProcessor processor = new CompoundProcessor(new SkipProcessor(skippedProcessor), executedProcessor); CompoundProcessor processor = new CompoundProcessor(new SkipProcessor(skippedProcessor), executedProcessor);
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "executed", "default"); assertDataSetFields(ingestDocument, "logs", "executed", "default");
@ -76,8 +85,8 @@ public class RerouteProcessorTests extends ESTestCase {
public void testSkipLastProcessor() throws Exception { public void testSkipLastProcessor() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor executedProcessor = createRerouteProcessor(List.of("executed"), List.of()); RerouteProcessor executedProcessor = createRerouteProcessor(List.of(), List.of("executed"), List.of());
RerouteProcessor skippedProcessor = createRerouteProcessor(List.of("skip"), List.of()); RerouteProcessor skippedProcessor = createRerouteProcessor(List.of(), List.of("skip"), List.of());
CompoundProcessor processor = new CompoundProcessor(executedProcessor, skippedProcessor); CompoundProcessor processor = new CompoundProcessor(executedProcessor, skippedProcessor);
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "executed", "default"); assertDataSetFields(ingestDocument, "logs", "executed", "default");
@ -85,23 +94,24 @@ public class RerouteProcessorTests extends ESTestCase {
public void testDataStreamFieldsFromDocument() throws Exception { public void testDataStreamFieldsFromDocument() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.setFieldValue("data_stream.type", "eggplant");
ingestDocument.setFieldValue("data_stream.dataset", "foo"); ingestDocument.setFieldValue("data_stream.dataset", "foo");
ingestDocument.setFieldValue("data_stream.namespace", "bar"); ingestDocument.setFieldValue("data_stream.namespace", "bar");
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of(), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo", "bar"); assertDataSetFields(ingestDocument, "eggplant", "foo", "bar");
} }
public void testDataStreamFieldsFromDocumentDottedNotation() throws Exception { public void testDataStreamFieldsFromDocumentDottedNotation() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.getCtxMap().put("data_stream.type", "logs"); ingestDocument.getCtxMap().put("data_stream.type", "eggplant");
ingestDocument.getCtxMap().put("data_stream.dataset", "foo"); ingestDocument.getCtxMap().put("data_stream.dataset", "foo");
ingestDocument.getCtxMap().put("data_stream.namespace", "bar"); ingestDocument.getCtxMap().put("data_stream.namespace", "bar");
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of(), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo", "bar"); assertDataSetFields(ingestDocument, "eggplant", "foo", "bar");
} }
public void testInvalidDataStreamFieldsFromDocument() throws Exception { public void testInvalidDataStreamFieldsFromDocument() throws Exception {
@ -109,7 +119,7 @@ public class RerouteProcessorTests extends ESTestCase {
ingestDocument.setFieldValue("data_stream.dataset", "foo-bar"); ingestDocument.setFieldValue("data_stream.dataset", "foo-bar");
ingestDocument.setFieldValue("data_stream.namespace", "baz#qux"); ingestDocument.setFieldValue("data_stream.namespace", "baz#qux");
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of(), List.of());
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "foo_bar", "baz_qux"); assertDataSetFields(ingestDocument, "logs", "foo_bar", "baz_qux");
} }
@ -128,7 +138,7 @@ public class RerouteProcessorTests extends ESTestCase {
ingestDocument.setFieldValue("service.name", "opbeans-java"); ingestDocument.setFieldValue("service.name", "opbeans-java");
ingestDocument.setFieldValue("service.environment", "dev"); ingestDocument.setFieldValue("service.environment", "dev");
RerouteProcessor processor = createRerouteProcessor(List.of("{{service.name}}"), List.of("{{service.environment}}")); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{service.name}}"), List.of("{{service.environment}}"));
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "opbeans_java", "dev"); assertDataSetFields(ingestDocument, "logs", "opbeans_java", "dev");
} }
@ -136,7 +146,7 @@ public class RerouteProcessorTests extends ESTestCase {
public void testRerouteToCurrentTarget() throws Exception { public void testRerouteToCurrentTarget() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor reroute = createRerouteProcessor(List.of("generic"), List.of("default")); RerouteProcessor reroute = createRerouteProcessor(List.of(), List.of("generic"), List.of("default"));
CompoundProcessor processor = new CompoundProcessor( CompoundProcessor processor = new CompoundProcessor(
reroute, reroute,
new TestProcessor(doc -> doc.setFieldValue("pipeline_is_continued", true)) new TestProcessor(doc -> doc.setFieldValue("pipeline_is_continued", true))
@ -149,7 +159,7 @@ public class RerouteProcessorTests extends ESTestCase {
public void testFieldReferenceWithMissingReroutesToCurrentTarget() throws Exception { public void testFieldReferenceWithMissingReroutesToCurrentTarget() throws Exception {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor reroute = createRerouteProcessor(List.of("{{service.name}}"), List.of("{{service.environment}}")); RerouteProcessor reroute = createRerouteProcessor(List.of(), List.of("{{service.name}}"), List.of("{{service.environment}}"));
CompoundProcessor processor = new CompoundProcessor( CompoundProcessor processor = new CompoundProcessor(
reroute, reroute,
new TestProcessor(doc -> doc.setFieldValue("pipeline_is_continued", true)) new TestProcessor(doc -> doc.setFieldValue("pipeline_is_continued", true))
@ -166,6 +176,7 @@ public class RerouteProcessorTests extends ESTestCase {
ingestDocument.setFieldValue("data_stream.namespace", "namespace_from_doc"); ingestDocument.setFieldValue("data_stream.namespace", "namespace_from_doc");
RerouteProcessor processor = createRerouteProcessor( RerouteProcessor processor = createRerouteProcessor(
List.of(),
List.of("{{{data_stream.dataset}}}", "fallback"), List.of("{{{data_stream.dataset}}}", "fallback"),
List.of("{{data_stream.namespace}}", "fallback") List.of("{{data_stream.namespace}}", "fallback")
); );
@ -177,6 +188,7 @@ public class RerouteProcessorTests extends ESTestCase {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
RerouteProcessor processor = createRerouteProcessor( RerouteProcessor processor = createRerouteProcessor(
List.of(),
List.of("{{data_stream.dataset}}", "fallback"), List.of("{{data_stream.dataset}}", "fallback"),
List.of("{{data_stream.namespace}}", "fallback") List.of("{{data_stream.namespace}}", "fallback")
); );
@ -190,6 +202,7 @@ public class RerouteProcessorTests extends ESTestCase {
ingestDocument.setFieldValue("data_stream.namespace", "default"); ingestDocument.setFieldValue("data_stream.namespace", "default");
RerouteProcessor processor = createRerouteProcessor( RerouteProcessor processor = createRerouteProcessor(
List.of(),
List.of("{{data_stream.dataset}}", "fallback"), List.of("{{data_stream.dataset}}", "fallback"),
List.of("{{{data_stream.namespace}}}", "fallback") List.of("{{{data_stream.namespace}}}", "fallback")
); );
@ -202,7 +215,7 @@ public class RerouteProcessorTests extends ESTestCase {
ingestDocument.setFieldValue("data_stream.dataset", "foo"); ingestDocument.setFieldValue("data_stream.dataset", "foo");
ingestDocument.setFieldValue("data_stream.namespace", "bar"); ingestDocument.setFieldValue("data_stream.namespace", "bar");
RerouteProcessor processor = createRerouteProcessor(List.of("{{foo}}"), List.of("{{bar}}")); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{foo}}"), List.of("{{bar}}"));
processor.execute(ingestDocument); processor.execute(ingestDocument);
assertDataSetFields(ingestDocument, "logs", "generic", "default"); assertDataSetFields(ingestDocument, "logs", "generic", "default");
} }
@ -210,7 +223,7 @@ public class RerouteProcessorTests extends ESTestCase {
public void testInvalidDataStreamName() throws Exception { public void testInvalidDataStreamName() throws Exception {
{ {
IngestDocument ingestDocument = createIngestDocument("foo"); IngestDocument ingestDocument = createIngestDocument("foo");
RerouteProcessor processor = createRerouteProcessor(List.of(), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of(), List.of());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("invalid data stream name: [foo]; must follow naming scheme <type>-<dataset>-<namespace>")); assertThat(e.getMessage(), equalTo("invalid data stream name: [foo]; must follow naming scheme <type>-<dataset>-<namespace>"));
} }
@ -227,11 +240,16 @@ public class RerouteProcessorTests extends ESTestCase {
public void testRouteOnNonStringFieldFails() { public void testRouteOnNonStringFieldFails() {
IngestDocument ingestDocument = createIngestDocument("logs-generic-default"); IngestDocument ingestDocument = createIngestDocument("logs-generic-default");
ingestDocument.setFieldValue("numeric_field", 42); ingestDocument.setFieldValue("numeric_field", 42);
RerouteProcessor processor = createRerouteProcessor(List.of("{{numeric_field}}"), List.of()); RerouteProcessor processor = createRerouteProcessor(List.of(), List.of("{{numeric_field}}"), List.of());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [numeric_field] of type [java.lang.Integer] cannot be cast to [java.lang.String]")); assertThat(e.getMessage(), equalTo("field [numeric_field] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
} }
public void testTypeSanitization() {
assertTypeSanitization("\\/*?\"<>| ,#:-", "_____________");
assertTypeSanitization("foo*bar", "foo_bar");
}
public void testDatasetSanitization() { public void testDatasetSanitization() {
assertDatasetSanitization("\\/*?\"<>| ,#:-", "_____________"); assertDatasetSanitization("\\/*?\"<>| ,#:-", "_____________");
assertDatasetSanitization("foo*bar", "foo_bar"); assertDatasetSanitization("foo*bar", "foo_bar");
@ -242,6 +260,14 @@ public class RerouteProcessorTests extends ESTestCase {
assertNamespaceSanitization("foo*bar", "foo_bar"); assertNamespaceSanitization("foo*bar", "foo_bar");
} }
private static void assertTypeSanitization(String type, String sanitizedType) {
assertThat(
RerouteProcessor.DataStreamValueSource.type("{{foo}}")
.resolve(RandomDocumentPicks.randomIngestDocument(random(), Map.of("foo", type))),
equalTo(sanitizedType)
);
}
private static void assertDatasetSanitization(String dataset, String sanitizedDataset) { private static void assertDatasetSanitization(String dataset, String sanitizedDataset) {
assertThat( assertThat(
RerouteProcessor.DataStreamValueSource.dataset("{{foo}}") RerouteProcessor.DataStreamValueSource.dataset("{{foo}}")
@ -258,10 +284,11 @@ public class RerouteProcessorTests extends ESTestCase {
); );
} }
private RerouteProcessor createRerouteProcessor(List<String> dataset, List<String> namespace) { private RerouteProcessor createRerouteProcessor(List<String> type, List<String> dataset, List<String> namespace) {
return new RerouteProcessor( return new RerouteProcessor(
null, null,
null, null,
type.stream().map(RerouteProcessor.DataStreamValueSource::type).toList(),
dataset.stream().map(RerouteProcessor.DataStreamValueSource::dataset).toList(), dataset.stream().map(RerouteProcessor.DataStreamValueSource::dataset).toList(),
namespace.stream().map(RerouteProcessor.DataStreamValueSource::namespace).toList(), namespace.stream().map(RerouteProcessor.DataStreamValueSource::namespace).toList(),
null null
@ -269,7 +296,7 @@ public class RerouteProcessorTests extends ESTestCase {
} }
private RerouteProcessor createRerouteProcessor(String destination) { private RerouteProcessor createRerouteProcessor(String destination) {
return new RerouteProcessor(null, null, List.of(), List.of(), destination); return new RerouteProcessor(null, null, List.of(), List.of(), List.of(), destination);
} }
private void assertDataSetFields(IngestDocument ingestDocument, String type, String dataset, String namespace) { private void assertDataSetFields(IngestDocument ingestDocument, String type, String dataset, String namespace) {

5
modules/ingest-geoip/src/main/plugin-metadata/entitlement-policy.yaml Normal file
View file

@ -0,0 +1,5 @@
com.maxmind.db:
- files:
- relative_path: "ingest-geoip/"
relative_to: "config"
mode: "read_write"

83
modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
View file

@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.BlockDocValuesReader;
import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockLoader;
import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.BlockSourceReader;
import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParserContext;
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues;
import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperBuilderContext;
@ -195,7 +196,9 @@ public class ScaledFloatFieldMapper extends FieldMapper {
scalingFactor.getValue(), scalingFactor.getValue(),
nullValue.getValue(), nullValue.getValue(),
metric.getValue(), metric.getValue(),
indexMode indexMode,
coerce.getValue().value(),
context.isSourceSynthetic()
); );
return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this); return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
} }
@ -209,6 +212,8 @@ public class ScaledFloatFieldMapper extends FieldMapper {
private final Double nullValue; private final Double nullValue;
private final TimeSeriesParams.MetricType metricType; private final TimeSeriesParams.MetricType metricType;
private final IndexMode indexMode; private final IndexMode indexMode;
private final boolean coerce;
private final boolean isSyntheticSource;
public ScaledFloatFieldType( public ScaledFloatFieldType(
String name, String name,
@ -219,13 +224,17 @@ public class ScaledFloatFieldMapper extends FieldMapper {
double scalingFactor, double scalingFactor,
Double nullValue, Double nullValue,
TimeSeriesParams.MetricType metricType, TimeSeriesParams.MetricType metricType,
IndexMode indexMode IndexMode indexMode,
boolean coerce,
boolean isSyntheticSource
) { ) {
super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
this.scalingFactor = scalingFactor; this.scalingFactor = scalingFactor;
this.nullValue = nullValue; this.nullValue = nullValue;
this.metricType = metricType; this.metricType = metricType;
this.indexMode = indexMode; this.indexMode = indexMode;
this.coerce = coerce;
this.isSyntheticSource = isSyntheticSource;
} }
public ScaledFloatFieldType(String name, double scalingFactor) { public ScaledFloatFieldType(String name, double scalingFactor) {
@ -233,7 +242,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
} }
public ScaledFloatFieldType(String name, double scalingFactor, boolean indexed) { public ScaledFloatFieldType(String name, double scalingFactor, boolean indexed) {
this(name, indexed, false, true, Collections.emptyMap(), scalingFactor, null, null, null); this(name, indexed, false, true, Collections.emptyMap(), scalingFactor, null, null, null, false, false);
} }
public double getScalingFactor() { public double getScalingFactor() {
@ -315,6 +324,15 @@ public class ScaledFloatFieldMapper extends FieldMapper {
double scalingFactorInverse = 1d / scalingFactor; double scalingFactorInverse = 1d / scalingFactor;
return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse); return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse);
} }
if (isSyntheticSource) {
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
@Override
public Builder builder(BlockFactory factory, int expectedCount) {
return factory.doubles(expectedCount);
}
};
}
ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()));
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
@ -322,6 +340,57 @@ public class ScaledFloatFieldMapper extends FieldMapper {
return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup); return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
} }
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null;
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValue) {
@Override
public void convertValue(Object value, List<Double> accumulator) {
if (coerce && value.equals("")) {
if (nullValueAdjusted != null) {
accumulator.add(nullValueAdjusted);
}
}
try {
// Convert to doc_values format
var converted = adjustSourceValue(NumberFieldMapper.NumberType.objectToDouble(value), scalingFactor);
accumulator.add(converted);
} catch (Exception e) {
// Malformed value, skip it
}
}
@Override
protected void parseNonNullValue(XContentParser parser, List<Double> accumulator) throws IOException {
// Aligned with implementation of `parseCreateField(XContentParser)`
if (coerce && parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0) {
if (nullValueAdjusted != null) {
accumulator.add(nullValueAdjusted);
}
}
try {
double value = parser.doubleValue(coerce);
// Convert to doc_values format
var converted = adjustSourceValue(value, scalingFactor);
accumulator.add(converted);
} catch (Exception e) {
// Malformed value, skip it
}
}
@Override
public void writeToBlock(List<Double> values, BlockLoader.Builder blockBuilder) {
var longBuilder = (BlockLoader.DoubleBuilder) blockBuilder;
for (var value : values) {
longBuilder.appendDouble(value);
}
}
};
}
@Override @Override
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
FielddataOperation operation = fieldDataContext.fielddataOperation(); FielddataOperation operation = fieldDataContext.fielddataOperation();
@ -386,12 +455,16 @@ public class ScaledFloatFieldMapper extends FieldMapper {
doubleValue = NumberFieldMapper.NumberType.objectToDouble(value); doubleValue = NumberFieldMapper.NumberType.objectToDouble(value);
} }
double factor = getScalingFactor(); return adjustSourceValue(doubleValue, getScalingFactor());
return Math.round(doubleValue * factor) / factor;
} }
}; };
} }
// Adjusts precision of a double value so that it looks like it came from doc_values.
private static Double adjustSourceValue(double value, double scalingFactor) {
return Math.round(value * scalingFactor) / scalingFactor;
}
@Override @Override
public Object valueForDisplay(Object value) { public Object valueForDisplay(Object value) {
if (value == null) { if (value == null) {

48
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.index.mapper.extras;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType;
import org.elasticsearch.plugins.Plugin;
import java.util.Collection;
import java.util.List;
import java.util.Map;
public class ScaledFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
public ScaledFloatFieldBlockLoaderTests() {
super(FieldType.SCALED_FLOAT);
}
@Override
protected Double convert(Number value, Map<String, Object> fieldMapping) {
var scalingFactor = ((Number) fieldMapping.get("scaling_factor")).doubleValue();
var docValues = (boolean) fieldMapping.getOrDefault("doc_values", false);
// There is a slight inconsistency between values that are read from doc_values and from source.
// Due to how precision reduction is applied to source values so that they are consistent with doc_values.
// See #122547.
if (docValues) {
var reverseScalingFactor = 1d / scalingFactor;
return Math.round(value.doubleValue() * scalingFactor) * reverseScalingFactor;
}
// Adjust values coming from source to the way they are stored in doc_values.
// See mapper implementation.
return Math.round(value.doubleValue() * scalingFactor) / scalingFactor;
}
@Override
protected Collection<? extends Plugin> getPlugins() {
return List.of(new MapperExtrasPlugin());
}
}

4
modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
View file

@ -95,7 +95,9 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase {
0.1 + randomDouble() * 100, 0.1 + randomDouble() * 100,
null, null,
null, null,
null null,
false,
false
); );
Directory dir = newDirectory(); Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));

15
modules/repository-azure/build.gradle
View file

@ -63,20 +63,20 @@ dependencies {
api "com.github.stephenc.jcip:jcip-annotations:1.0-1" api "com.github.stephenc.jcip:jcip-annotations:1.0-1"
api "com.nimbusds:content-type:2.3" api "com.nimbusds:content-type:2.3"
api "com.nimbusds:lang-tag:1.7" api "com.nimbusds:lang-tag:1.7"
api("com.nimbusds:nimbus-jose-jwt:9.37.3"){ api("com.nimbusds:nimbus-jose-jwt:10.0.1"){
exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely
} }
api("com.nimbusds:oauth2-oidc-sdk:11.9.1"){ api("com.nimbusds:oauth2-oidc-sdk:11.22.2"){
exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely
} }
api "jakarta.activation:jakarta.activation-api:1.2.1" api "jakarta.activation:jakarta.activation-api:1.2.1"
api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3"
api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
api "net.minidev:accessors-smart:2.5.0" api "net.minidev:accessors-smart:2.5.2"
api "net.minidev:json-smart:2.5.0" api "net.minidev:json-smart:2.5.2"
api "org.codehaus.woodstox:stax2-api:4.2.2" api "org.codehaus.woodstox:stax2-api:4.2.2"
api "org.ow2.asm:asm:9.3" api "org.ow2.asm:asm:9.7.1"
runtimeOnly "com.google.code.gson:gson:2.11.0" runtimeOnly "com.google.code.gson:gson:2.11.0"
runtimeOnly "org.cryptomator:siv-mode:1.5.2" runtimeOnly "org.cryptomator:siv-mode:1.5.2"
@ -190,11 +190,6 @@ tasks.named("thirdPartyAudit").configure {
'org.bouncycastle.cert.X509CertificateHolder', 'org.bouncycastle.cert.X509CertificateHolder',
'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder', 'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
'org.bouncycastle.crypto.InvalidCipherTextException',
'org.bouncycastle.crypto.engines.AESEngine',
'org.bouncycastle.crypto.modes.GCMBlockCipher',
'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
'org.bouncycastle.jce.provider.BouncyCastleProvider',
'org.bouncycastle.openssl.PEMKeyPair', 'org.bouncycastle.openssl.PEMKeyPair',
'org.bouncycastle.openssl.PEMParser', 'org.bouncycastle.openssl.PEMParser',
'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter', 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',

5
modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
View file

@ -899,11 +899,10 @@ class S3BlobContainer extends AbstractBlobContainer {
logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", key), e); logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", key), e);
if (e instanceof AmazonS3Exception amazonS3Exception if (e instanceof AmazonS3Exception amazonS3Exception
&& (amazonS3Exception.getStatusCode() == 404 && (amazonS3Exception.getStatusCode() == 404
|| amazonS3Exception.getStatusCode() == 0 && "NoSuchUpload".equals(amazonS3Exception.getErrorCode()))) { || amazonS3Exception.getStatusCode() == 200 && "NoSuchUpload".equals(amazonS3Exception.getErrorCode()))) {
// An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it. // An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it.
// Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an // Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an
// <Error><Code>NoSuchUpload</Code>... in the response, which the SDK translates to status code 0. Either way, this means // <Error><Code>NoSuchUpload</Code>... in the response. Either way, this means that our write encountered contention:
// that our write encountered contention:
delegate.onResponse(OptionalBytesReference.MISSING); delegate.onResponse(OptionalBytesReference.MISSING);
} else { } else {
delegate.onFailure(e); delegate.onFailure(e);

4
modules/repository-s3/src/main/plugin-metadata/entitlement-policy.yaml
View file

@ -1,3 +1,7 @@
ALL-UNNAMED: ALL-UNNAMED:
- manage_threads - manage_threads
- outbound_network - outbound_network
- files:
- relative_path: "repository-s3/aws-web-identity-token-file"
relative_to: "config"
mode: "read"

87
muted-tests.yml
View file

@ -227,18 +227,6 @@ tests:
- class: org.elasticsearch.xpack.test.rest.XPackRestIT - class: org.elasticsearch.xpack.test.rest.XPackRestIT
method: test {p0=ml/*} method: test {p0=ml/*}
issue: https://github.com/elastic/elasticsearch/issues/120816 issue: https://github.com/elastic/elasticsearch/issues/120816
- class: org.elasticsearch.upgrades.VectorSearchIT
method: testBBQVectorSearch {upgradedNodes=0}
issue: https://github.com/elastic/elasticsearch/issues/121253
- class: org.elasticsearch.upgrades.VectorSearchIT
method: testBBQVectorSearch {upgradedNodes=1}
issue: https://github.com/elastic/elasticsearch/issues/121271
- class: org.elasticsearch.upgrades.VectorSearchIT
method: testBBQVectorSearch {upgradedNodes=2}
issue: https://github.com/elastic/elasticsearch/issues/121272
- class: org.elasticsearch.upgrades.VectorSearchIT
method: testBBQVectorSearch {upgradedNodes=3}
issue: https://github.com/elastic/elasticsearch/issues/121273
- class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactoryTests - class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactoryTests
issue: https://github.com/elastic/elasticsearch/issues/121285 issue: https://github.com/elastic/elasticsearch/issues/121285
- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
@ -288,9 +276,6 @@ tests:
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
method: testStopQueryLocal method: testStopQueryLocal
issue: https://github.com/elastic/elasticsearch/issues/121672 issue: https://github.com/elastic/elasticsearch/issues/121672
- class: org.elasticsearch.xpack.security.authz.IndexAliasesTests
method: testRemoveIndex
issue: https://github.com/elastic/elasticsearch/issues/122221
- class: org.elasticsearch.blocks.SimpleBlocksIT - class: org.elasticsearch.blocks.SimpleBlocksIT
method: testConcurrentAddBlock method: testConcurrentAddBlock
issue: https://github.com/elastic/elasticsearch/issues/122324 issue: https://github.com/elastic/elasticsearch/issues/122324
@ -312,8 +297,6 @@ tests:
issue: https://github.com/elastic/elasticsearch/issues/122377 issue: https://github.com/elastic/elasticsearch/issues/122377
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT
issue: https://github.com/elastic/elasticsearch/issues/122378 issue: https://github.com/elastic/elasticsearch/issues/122378
- class: org.elasticsearch.telemetry.apm.ApmAgentSettingsIT
issue: https://github.com/elastic/elasticsearch/issues/122546
- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests - class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests
method: testSnapshotRecovery {p0=false p1=false} method: testSnapshotRecovery {p0=false p1=false}
issue: https://github.com/elastic/elasticsearch/issues/122549 issue: https://github.com/elastic/elasticsearch/issues/122549
@ -332,71 +315,17 @@ tests:
- class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT - class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT
method: testScaleWhileShrinking method: testScaleWhileShrinking
issue: https://github.com/elastic/elasticsearch/issues/122119 issue: https://github.com/elastic/elasticsearch/issues/122119
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122688
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122689
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122690
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122691
- class: org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests
method: testCreateAndRestorePartialSearchableSnapshot
issue: https://github.com/elastic/elasticsearch/issues/122693
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122694
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
issue: https://github.com/elastic/elasticsearch/issues/122695
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122696
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
issue: https://github.com/elastic/elasticsearch/issues/122697
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]}
issue: https://github.com/elastic/elasticsearch/issues/122698
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122700
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122701
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122702
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]}
issue: https://github.com/elastic/elasticsearch/issues/122703
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
issue: https://github.com/elastic/elasticsearch/issues/122704
- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]}
issue: https://github.com/elastic/elasticsearch/issues/122705
- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
method: testSearchWithRandomDisconnects method: testSearchWithRandomDisconnects
issue: https://github.com/elastic/elasticsearch/issues/122707 issue: https://github.com/elastic/elasticsearch/issues/122707
- class: org.elasticsearch.indices.recovery.IndexRecoveryIT - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
method: testSourceThrottling issue: https://github.com/elastic/elasticsearch/issues/122810
issue: https://github.com/elastic/elasticsearch/issues/122712 - class: org.elasticsearch.snapshots.DedicatedClusterSnapshotRestoreIT
- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedNonModularIT method: testRestoreShrinkIndex
issue: https://github.com/elastic/elasticsearch/issues/122569 issue: https://github.com/elastic/elasticsearch/issues/121717
- class: org.elasticsearch.entitlement.qa.EntitlementsAllowedNonModularIT - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
issue: https://github.com/elastic/elasticsearch/issues/122568 method: test {yaml=reference/cat/allocation/cat-allocation-example}
- class: org.elasticsearch.entitlement.qa.EntitlementsAllowedIT issue: https://github.com/elastic/elasticsearch/issues/121976
issue: https://github.com/elastic/elasticsearch/issues/122680
- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT
issue: https://github.com/elastic/elasticsearch/issues/122566
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.S3RepositoryAnalysisRestIT
method: testRepositoryAnalysis
issue: https://github.com/elastic/elasticsearch/issues/122799
# Examples: # Examples:
# #

2
qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.bootstrap; package org.elasticsearch.bootstrap;
import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.jdk.RuntimeVersionFeature;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -42,6 +43,7 @@ public class ESPolicyUnitTests extends ESTestCase {
@BeforeClass @BeforeClass
public static void setupPolicy() { public static void setupPolicy() {
assumeTrue("test requires security manager to be supported", RuntimeVersionFeature.isSecurityManagerAvailable());
assumeTrue("test cannot run with security manager", System.getSecurityManager() == null); assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
DEFAULT_POLICY = PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), TEST_CODEBASES); DEFAULT_POLICY = PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), TEST_CODEBASES);
} }

2
qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.bootstrap; package org.elasticsearch.bootstrap;
import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.jdk.RuntimeVersionFeature;
import org.elasticsearch.plugins.PluginDescriptor; import org.elasticsearch.plugins.PluginDescriptor;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.Before; import org.junit.Before;
@ -40,6 +41,7 @@ public class PolicyUtilTests extends ESTestCase {
@Before @Before
public void assumeSecurityManagerDisabled() { public void assumeSecurityManagerDisabled() {
assumeTrue("test requires security manager to be supported", RuntimeVersionFeature.isSecurityManagerAvailable());
assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null); assumeTrue("test cannot run with security manager enabled", System.getSecurityManager() == null);
} }

7
qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java
View file

@ -11,8 +11,10 @@ package org.elasticsearch.plugins.cli;
import org.elasticsearch.bootstrap.PluginPolicyInfo; import org.elasticsearch.bootstrap.PluginPolicyInfo;
import org.elasticsearch.bootstrap.PolicyUtil; import org.elasticsearch.bootstrap.PolicyUtil;
import org.elasticsearch.jdk.RuntimeVersionFeature;
import org.elasticsearch.plugins.PluginDescriptor; import org.elasticsearch.plugins.PluginDescriptor;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
@ -26,6 +28,11 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
/** Tests plugin manager security check */ /** Tests plugin manager security check */
public class PluginSecurityTests extends ESTestCase { public class PluginSecurityTests extends ESTestCase {
@Before
public void assumeSecurityManagerSupported() {
assumeTrue("test requires security manager to be supported", RuntimeVersionFeature.isSecurityManagerAvailable());
}
PluginPolicyInfo makeDummyPlugin(String policy, String... files) throws IOException { PluginPolicyInfo makeDummyPlugin(String policy, String... files) throws IOException {
Path plugin = createTempDir(); Path plugin = createTempDir();
Files.copy(this.getDataPath(policy), plugin.resolve(PluginDescriptor.ES_PLUGIN_POLICY)); Files.copy(this.getDataPath(policy), plugin.resolve(PluginDescriptor.ES_PLUGIN_POLICY));

38
qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java
View file

@ -13,6 +13,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name;
import org.elasticsearch.client.Request; import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response; import org.elasticsearch.client.Response;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.common.xcontent.support.XContentMapValues;
@ -456,7 +457,11 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
} }
"""; """;
// create index and index 10 random floating point vectors // create index and index 10 random floating point vectors
createIndex(BBQ_INDEX_NAME, Settings.EMPTY, mapping); createIndex(
BBQ_INDEX_NAME,
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(),
mapping
);
index64DimVectors(BBQ_INDEX_NAME); index64DimVectors(BBQ_INDEX_NAME);
// force merge the index // force merge the index
client().performRequest(new Request("POST", "/" + BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1")); client().performRequest(new Request("POST", "/" + BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1"));
@ -485,8 +490,8 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
Map<String, Object> response = search(searchRequest); Map<String, Object> response = search(searchRequest);
assertThat(extractValue(response, "hits.total.value"), equalTo(7)); assertThat(extractValue(response, "hits.total.value"), equalTo(7));
List<Map<String, Object>> hits = extractValue(response, "hits.hits"); List<Map<String, Object>> hits = extractValue(response, "hits.hits");
assertThat(hits.get(0).get("_id"), equalTo("0")); assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0"));
assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
// search with knn // search with knn
searchRequest = new Request("POST", "/" + BBQ_INDEX_NAME + "/_search"); searchRequest = new Request("POST", "/" + BBQ_INDEX_NAME + "/_search");
@ -504,8 +509,12 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
response = search(searchRequest); response = search(searchRequest);
assertThat(extractValue(response, "hits.total.value"), equalTo(2)); assertThat(extractValue(response, "hits.total.value"), equalTo(2));
hits = extractValue(response, "hits.hits"); hits = extractValue(response, "hits.hits");
assertThat(hits.get(0).get("_id"), equalTo("0")); assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0"));
assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); assertThat(
"expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response,
(double) hits.get(0).get("_score"),
closeTo(0.9934857, 0.005)
);
} }
public void testFlatBBQVectorSearch() throws Exception { public void testFlatBBQVectorSearch() throws Exception {
@ -530,7 +539,11 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
} }
"""; """;
// create index and index 10 random floating point vectors // create index and index 10 random floating point vectors
createIndex(FLAT_BBQ_INDEX_NAME, Settings.EMPTY, mapping); createIndex(
FLAT_BBQ_INDEX_NAME,
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(),
mapping
);
index64DimVectors(FLAT_BBQ_INDEX_NAME); index64DimVectors(FLAT_BBQ_INDEX_NAME);
// force merge the index // force merge the index
client().performRequest(new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1")); client().performRequest(new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1"));
@ -559,8 +572,8 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
Map<String, Object> response = search(searchRequest); Map<String, Object> response = search(searchRequest);
assertThat(extractValue(response, "hits.total.value"), equalTo(7)); assertThat(extractValue(response, "hits.total.value"), equalTo(7));
List<Map<String, Object>> hits = extractValue(response, "hits.hits"); List<Map<String, Object>> hits = extractValue(response, "hits.hits");
assertThat(hits.get(0).get("_id"), equalTo("0")); assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0"));
assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
// search with knn // search with knn
searchRequest = new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_search"); searchRequest = new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_search");
@ -578,8 +591,12 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
response = search(searchRequest); response = search(searchRequest);
assertThat(extractValue(response, "hits.total.value"), equalTo(2)); assertThat(extractValue(response, "hits.total.value"), equalTo(2));
hits = extractValue(response, "hits.hits"); hits = extractValue(response, "hits.hits");
assertThat(hits.get(0).get("_id"), equalTo("0")); assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0"));
assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); assertThat(
"expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response,
(double) hits.get(0).get("_score"),
closeTo(0.9934857, 0.005)
);
} }
private void index64DimVectors(String indexName) throws Exception { private void index64DimVectors(String indexName) throws Exception {
@ -605,6 +622,7 @@ public class VectorSearchIT extends AbstractRollingUpgradeTestCase {
assertOK(client().performRequest(indexRequest)); assertOK(client().performRequest(indexRequest));
} }
// always refresh to ensure the data is visible // always refresh to ensure the data is visible
flush(indexName, true);
refresh(indexName); refresh(indexName);
} }

23
server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
View file

@ -342,9 +342,17 @@ public class IndexRecoveryIT extends AbstractIndexRecoveryIntegTestCase {
assertThat(recoveryStats.currentAsSource(), equalTo(0)); assertThat(recoveryStats.currentAsSource(), equalTo(0));
assertThat(recoveryStats.currentAsTarget(), equalTo(0)); assertThat(recoveryStats.currentAsTarget(), equalTo(0));
if (isRecoveryThrottlingNode) { if (isRecoveryThrottlingNode) {
assertThat("Throttling should be >0 for '" + nodeName + "'", recoveryStats.throttleTime().millis(), greaterThan(0L)); assertThat(
"Throttling should be >0 for '" + nodeName + "'. Node stats: " + nodesStatsResponse,
recoveryStats.throttleTime().millis(),
greaterThan(0L)
);
} else { } else {
assertThat("Throttling should be =0 for '" + nodeName + "'", recoveryStats.throttleTime().millis(), equalTo(0L)); assertThat(
"Throttling should be =0 for '" + nodeName + "'. Node stats: " + nodesStatsResponse,
recoveryStats.throttleTime().millis(),
equalTo(0L)
);
} }
} }
@ -1967,7 +1975,14 @@ public class IndexRecoveryIT extends AbstractIndexRecoveryIntegTestCase {
internalCluster().startMasterOnlyNode(); internalCluster().startMasterOnlyNode();
final var dataNode = internalCluster().startDataOnlyNode(); final var dataNode = internalCluster().startDataOnlyNode();
final var indexName = randomIdentifier(); final var indexName = randomIdentifier();
createIndex(indexName, indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false).build()); final var indexSettingsBuilder = indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false);
if (randomBoolean()) {
indexSettingsBuilder.put(
IndexMetadata.SETTING_VERSION_CREATED,
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.UPGRADE_TO_LUCENE_10_0_0, IndexVersion.current())
);
}
createIndex(indexName, indexSettingsBuilder.build());
final var initialSegmentCount = 20; final var initialSegmentCount = 20;
for (int i = 0; i < initialSegmentCount; i++) { for (int i = 0; i < initialSegmentCount; i++) {
@ -2051,7 +2066,7 @@ public class IndexRecoveryIT extends AbstractIndexRecoveryIntegTestCase {
IndexVersionUtils.randomVersionBetween( IndexVersionUtils.randomVersionBetween(
random(), random(),
IndexVersionUtils.getLowestWriteCompatibleVersion(), IndexVersionUtils.getLowestWriteCompatibleVersion(),
IndexVersionUtils.getPreviousVersion(IndexVersions.MERGE_ON_RECOVERY_VERSION) IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
) )
) )
.build() .build()

5
server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
View file

@ -127,10 +127,7 @@ public class RetrieverRewriteIT extends ESIntegTestCase {
SearchPhaseExecutionException.class, SearchPhaseExecutionException.class,
client().prepareSearch(testIndex).setSource(source)::get client().prepareSearch(testIndex).setSource(source)::get
); );
assertThat( assertThat(ex.getDetailedMessage(), containsString("Search rejected due to missing shards"));
ex.getDetailedMessage(),
containsString("[open_point_in_time] action requires all shards to be available. Missing shards")
);
} finally { } finally {
internalCluster().restartNode(randomDataNode); internalCluster().restartNode(randomDataNode);
} }

2
server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java
View file

@ -201,7 +201,7 @@ final class CanMatchPreFilterSearchPhase {
private void checkNoMissingShards(List<SearchShardIterator> shards) { private void checkNoMissingShards(List<SearchShardIterator> shards) {
assert assertSearchCoordinationThread(); assert assertSearchCoordinationThread();
SearchPhase.doCheckNoMissingShards("can_match", request, shards, SearchPhase::makeMissingShardsError); SearchPhase.doCheckNoMissingShards("can_match", request, shards);
} }
private Map<SendingTarget, List<SearchShardIterator>> groupByNode(List<SearchShardIterator> shards) { private Map<SendingTarget, List<SearchShardIterator>> groupByNode(List<SearchShardIterator> shards) {

20
server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
View file

@ -14,7 +14,6 @@ import org.elasticsearch.transport.Transport;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.function.Function;
/** /**
* Base class for all individual search phases like collecting distributed frequencies, fetching documents, querying shards. * Base class for all individual search phases like collecting distributed frequencies, fetching documents, querying shards.
@ -35,26 +34,13 @@ abstract class SearchPhase {
return name; return name;
} }
protected String missingShardsErrorMessage(StringBuilder missingShards) { private static String makeMissingShardsError(StringBuilder missingShards) {
return makeMissingShardsError(missingShards);
}
protected static String makeMissingShardsError(StringBuilder missingShards) {
return "Search rejected due to missing shards [" return "Search rejected due to missing shards ["
+ missingShards + missingShards
+ "]. Consider using `allow_partial_search_results` setting to bypass this error."; + "]. Consider using `allow_partial_search_results` setting to bypass this error.";
} }
protected void doCheckNoMissingShards(String phaseName, SearchRequest request, List<SearchShardIterator> shardsIts) { protected static void doCheckNoMissingShards(String phaseName, SearchRequest request, List<SearchShardIterator> shardsIts) {
doCheckNoMissingShards(phaseName, request, shardsIts, this::missingShardsErrorMessage);
}
protected static void doCheckNoMissingShards(
String phaseName,
SearchRequest request,
List<SearchShardIterator> shardsIts,
Function<StringBuilder, String> makeErrorMessage
) {
assert request.allowPartialSearchResults() != null : "SearchRequest missing setting for allowPartialSearchResults"; assert request.allowPartialSearchResults() != null : "SearchRequest missing setting for allowPartialSearchResults";
if (request.allowPartialSearchResults() == false) { if (request.allowPartialSearchResults() == false) {
final StringBuilder missingShards = new StringBuilder(); final StringBuilder missingShards = new StringBuilder();
@ -70,7 +56,7 @@ abstract class SearchPhase {
} }
if (missingShards.isEmpty() == false) { if (missingShards.isEmpty() == false) {
// Status red - shard is missing all copies and would produce partial results for an index search // Status red - shard is missing all copies and would produce partial results for an index search
final String msg = makeErrorMessage.apply(missingShards); final String msg = makeMissingShardsError(missingShards);
throw new SearchPhaseExecutionException(phaseName, msg, null, ShardSearchFailure.EMPTY_ARRAY); throw new SearchPhaseExecutionException(phaseName, msg, null, ShardSearchFailure.EMPTY_ARRAY);
} }
} }

6
server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
View file

@ -241,12 +241,6 @@ public class TransportOpenPointInTimeAction extends HandledTransportAction<OpenP
searchRequest.getMaxConcurrentShardRequests(), searchRequest.getMaxConcurrentShardRequests(),
clusters clusters
) { ) {
protected String missingShardsErrorMessage(StringBuilder missingShards) {
return "[open_point_in_time] action requires all shards to be available. Missing shards: ["
+ missingShards
+ "]. Consider using `allow_partial_search_results` setting to bypass this error.";
}
@Override @Override
protected void executePhaseOnShard( protected void executePhaseOnShard(
SearchShardIterator shardIt, SearchShardIterator shardIt,

6
server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
View file

@ -245,10 +245,12 @@ class Elasticsearch {
EntitlementBootstrap.bootstrap( EntitlementBootstrap.bootstrap(
pluginPolicies, pluginPolicies,
pluginsResolver::resolveClassToPluginName, pluginsResolver::resolveClassToPluginName,
nodeEnv.settings()::get,
nodeEnv.settings()::getGlobValues,
nodeEnv.dataDirs(), nodeEnv.dataDirs(),
nodeEnv.configDir(), nodeEnv.configDir(),
nodeEnv.tmpDir(), nodeEnv.logsDir(),
nodeEnv.logsDir() nodeEnv.tmpDir()
); );
} else { } else {
assert RuntimeVersionFeature.isSecurityManagerAvailable(); assert RuntimeVersionFeature.isSecurityManagerAvailable();

32
server/src/main/java/org/elasticsearch/common/collect/Iterators.java
View file

@ -34,22 +34,27 @@ public class Iterators {
* Returns a single element iterator over the supplied value. * Returns a single element iterator over the supplied value.
*/ */
public static <T> Iterator<T> single(T element) { public static <T> Iterator<T> single(T element) {
return new Iterator<>() { return new SingleIterator<>(element);
}
private T value = Objects.requireNonNull(element); private static final class SingleIterator<T> implements Iterator<T> {
private T value;
@Override SingleIterator(T element) {
public boolean hasNext() { value = Objects.requireNonNull(element);
return value != null; }
}
@Override @Override
public T next() { public boolean hasNext() {
final T res = value; return value != null;
value = null; }
return res;
} @Override
}; public T next() {
final T res = value;
value = null;
return res;
}
} }
@SafeVarargs @SafeVarargs
@ -533,5 +538,4 @@ public class Iterators {
} }
return result; return result;
} }
} }

18
server/src/main/java/org/elasticsearch/common/settings/Settings.java
View file

@ -290,6 +290,24 @@ public final class Settings implements ToXContentFragment, Writeable, Diffable<S
return retVal == null ? defaultValue : retVal; return retVal == null ? defaultValue : retVal;
} }
/**
* Returns the values for settings that match the given glob pattern.
* A single glob is supported.
*
* @param settingGlob setting name containing a glob
* @return zero or more values for any settings in this settings object that match the glob pattern
*/
public Stream<String> getGlobValues(String settingGlob) {
int globIndex = settingGlob.indexOf(".*.");
if (globIndex == -1) {
throw new IllegalArgumentException("Pattern [" + settingGlob + "] does not contain a glob [*]");
}
String prefix = settingGlob.substring(0, globIndex + 1);
String suffix = settingGlob.substring(globIndex + 2);
Settings subSettings = getByPrefix(prefix);
return subSettings.names().stream().map(k -> k + suffix).map(subSettings::get).filter(Objects::nonNull);
}
/** /**
* Returns the setting value (as float) associated with the setting key. If it does not exists, * Returns the setting value (as float) associated with the setting key. If it does not exists,
* returns the default value provided. * returns the default value provided.

2
server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java
View file

@ -293,6 +293,6 @@ public abstract class FallbackSyntheticSourceBlockLoader implements BlockLoader
parseNonNullValue(parser, accumulator); parseNonNullValue(parser, accumulator);
} }
abstract void parseNonNullValue(XContentParser parser, List<T> accumulator) throws IOException; protected abstract void parseNonNullValue(XContentParser parser, List<T> accumulator) throws IOException;
} }
} }

4
server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java
View file

@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner; import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner;
import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasable;
import org.elasticsearch.core.Strings; import org.elasticsearch.core.Strings;
import org.elasticsearch.core.UpdateForV10;
import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardId;
@ -46,6 +47,7 @@ class PostRecoveryMerger {
private static final boolean TRIGGER_MERGE_AFTER_RECOVERY; private static final boolean TRIGGER_MERGE_AFTER_RECOVERY;
static { static {
@UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_INDEXING) // remove this escape hatch
final var propertyValue = System.getProperty("es.trigger_merge_after_recovery"); final var propertyValue = System.getProperty("es.trigger_merge_after_recovery");
if (propertyValue == null) { if (propertyValue == null) {
TRIGGER_MERGE_AFTER_RECOVERY = true; TRIGGER_MERGE_AFTER_RECOVERY = true;
@ -95,7 +97,7 @@ class PostRecoveryMerger {
return recoveryListener; return recoveryListener;
} }
if (indexMetadata.getCreationVersion().before(IndexVersions.MERGE_ON_RECOVERY_VERSION)) { if (indexMetadata.getCreationVersion().before(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) {
return recoveryListener; return recoveryListener;
} }

6
server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
View file

@ -703,4 +703,10 @@ public class SettingsTests extends ESTestCase {
{"ant.bee":{"cat.dog":{"ewe":"value3"},"cat":"value2"},"ant":"value1"}""", Strings.toString(builder)); {"ant.bee":{"cat.dog":{"ewe":"value3"},"cat":"value2"},"ant":"value1"}""", Strings.toString(builder));
} }
public void testGlobValues() throws IOException {
Settings test = Settings.builder().put("foo.x.bar", "1").put("foo.y.bar", "2").build();
var values = test.getGlobValues("foo.*.bar").toList();
assertThat(values, containsInAnyOrder("1", "2"));
}
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> { public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
public ByteFieldBlockLoaderTests() { public ByteFieldBlockLoaderTests() {
super(FieldType.BYTE); super(FieldType.BYTE);
} }
@Override @Override
protected Integer convert(Number value) { protected Integer convert(Number value, Map<String, Object> fieldMapping) {
// All values that fit into int are represented as ints // All values that fit into int are represented as ints
return value.intValue(); return value.intValue();
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> { public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
public DoubleFieldBlockLoaderTests() { public DoubleFieldBlockLoaderTests() {
super(FieldType.DOUBLE); super(FieldType.DOUBLE);
} }
@Override @Override
protected Double convert(Number value) { protected Double convert(Number value, Map<String, Object> fieldMapping) {
return value.doubleValue(); return value.doubleValue();
} }
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> { public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
public FloatFieldBlockLoaderTests() { public FloatFieldBlockLoaderTests() {
super(FieldType.FLOAT); super(FieldType.FLOAT);
} }
@Override @Override
protected Double convert(Number value) { protected Double convert(Number value, Map<String, Object> fieldMapping) {
// All float values are represented as double // All float values are represented as double
return value.doubleValue(); return value.doubleValue();
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java
View file

@ -10,15 +10,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> { public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
public HalfFloatFieldBlockLoaderTests() { public HalfFloatFieldBlockLoaderTests() {
super(FieldType.HALF_FLOAT); super(FieldType.HALF_FLOAT);
} }
@Override @Override
protected Double convert(Number value) { protected Double convert(Number value, Map<String, Object> fieldMapping) {
// All float values are represented as double // All float values are represented as double
return (double) HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value.floatValue())); return (double) HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value.floatValue()));
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> { public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
public IntegerFieldBlockLoaderTests() { public IntegerFieldBlockLoaderTests() {
super(FieldType.INTEGER); super(FieldType.INTEGER);
} }
@Override @Override
protected Integer convert(Number value) { protected Integer convert(Number value, Map<String, Object> fieldMapping) {
return value.intValue(); return value.intValue();
} }
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Long> { public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Long> {
public LongFieldBlockLoaderTests() { public LongFieldBlockLoaderTests() {
super(FieldType.LONG); super(FieldType.LONG);
} }
@Override @Override
protected Long convert(Number value) { protected Long convert(Number value, Map<String, Object> fieldMapping) {
return value.longValue(); return value.longValue();
} }
} }

5
server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java
View file

@ -9,15 +9,18 @@
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper.blockloader;
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.Map;
public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> { public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
public ShortFieldBlockLoaderTests() { public ShortFieldBlockLoaderTests() {
super(FieldType.SHORT); super(FieldType.SHORT);
} }
@Override @Override
protected Integer convert(Number value) { protected Integer convert(Number value, Map<String, Object> fieldMapping) {
// All values that fit into int are represented as ints // All values that fit into int are represented as ints
return value.intValue(); return value.intValue();
} }

21
server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java → test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java
View file

@ -7,9 +7,8 @@
* License v3.0 only", or the "Server Side Public License, v 1". * License v3.0 only", or the "Server Side Public License, v 1".
*/ */
package org.elasticsearch.index.mapper.blockloader; package org.elasticsearch.index.mapper;
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.FieldType;
import java.util.List; import java.util.List;
@ -24,25 +23,29 @@ public abstract class NumberFieldBlockLoaderTestCase<T extends Number> extends B
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource) { protected Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource) {
var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value")) : null; var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value"), fieldMapping) : null;
if (value instanceof List<?> == false) { if (value instanceof List<?> == false) {
return convert(value, nullValue); return convert(value, nullValue, fieldMapping);
} }
if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { if ((boolean) fieldMapping.getOrDefault("doc_values", false)) {
// Sorted and no duplicates // Sorted and no duplicates
var resultList = ((List<Object>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).sorted().toList(); var resultList = ((List<Object>) value).stream()
.map(v -> convert(v, nullValue, fieldMapping))
.filter(Objects::nonNull)
.sorted()
.toList();
return maybeFoldList(resultList); return maybeFoldList(resultList);
} }
// parsing from source // parsing from source
var resultList = ((List<Object>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); var resultList = ((List<Object>) value).stream().map(v -> convert(v, nullValue, fieldMapping)).filter(Objects::nonNull).toList();
return maybeFoldList(resultList); return maybeFoldList(resultList);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private T convert(Object value, T nullValue) { private T convert(Object value, T nullValue, Map<String, Object> fieldMapping) {
if (value == null) { if (value == null) {
return nullValue; return nullValue;
} }
@ -51,12 +54,12 @@ public abstract class NumberFieldBlockLoaderTestCase<T extends Number> extends B
return nullValue; return nullValue;
} }
if (value instanceof Number n) { if (value instanceof Number n) {
return convert(n); return convert(n, fieldMapping);
} }
// Malformed values are excluded // Malformed values are excluded
return null; return null;
} }
protected abstract T convert(Number value); protected abstract T convert(Number value, Map<String, Object> fieldMapping);
} }

2
test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
View file

@ -102,7 +102,7 @@ public class DefaultMappingParametersHandler implements DataSourceHandler {
injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5)); injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5));
if (ESTestCase.randomDouble() <= 0.2) { if (ESTestCase.randomDouble() <= 0.2) {
injected.put("null_value", ESTestCase.randomFloat()); injected.put("null_value", ESTestCase.randomDouble());
} }
if (ESTestCase.randomBoolean()) { if (ESTestCase.randomBoolean()) {

5
x-pack/plugin/blob-cache/src/main/plugin-metadata/entitlement-policy.yaml Normal file
View file

@ -0,0 +1,5 @@
org.elasticsearch.blobcache:
- files:
- relative_path: "shared_snapshot_cache"
relative_to: "data"
mode: "read_write"

11
x-pack/plugin/ccr/build.gradle
View file

@ -1,6 +1,5 @@
apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-es-plugin'
apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.internal-java-rest-test'
esplugin { esplugin {
name = 'x-pack-ccr' name = 'x-pack-ccr'
description = 'Elasticsearch Expanded Pack Plugin - CCR' description = 'Elasticsearch Expanded Pack Plugin - CCR'
@ -34,16 +33,6 @@ tasks.named('internalClusterTestTestingConventions').configure {
baseClass 'org.elasticsearch.test.ESIntegTestCase' baseClass 'org.elasticsearch.test.ESIntegTestCase'
} }
tasks.named("javaRestTest").configure {
usesDefaultDistribution()
}
restResources {
restApi {
include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream'
}
}
addQaCheckDependencies(project) addQaCheckDependencies(project)
dependencies { dependencies {

399
x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/rest/ShardChangesRestIT.java
View file

@ -1,399 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.ccr.rest;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.Build;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.ClassRule;
import java.io.IOException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Locale;
import java.util.Map;
public class ShardChangesRestIT extends ESRestTestCase {
private static final String CCR_SHARD_CHANGES_ENDPOINT = "/%s/ccr/shard_changes";
private static final String BULK_INDEX_ENDPOINT = "/%s/_bulk";
private static final String DATA_STREAM_ENDPOINT = "/_data_stream/%s";
private static final String INDEX_TEMPLATE_ENDPOINT = "/_index_template/%s";
private static final String[] SHARD_RESPONSE_FIELDS = new String[] {
"took_in_millis",
"operations",
"shard_id",
"index_abstraction",
"index",
"settings_version",
"max_seq_no_of_updates_or_deletes",
"number_of_operations",
"mapping_version",
"aliases_version",
"max_seq_no",
"global_checkpoint" };
private static final String BULK_INDEX_TEMPLATE = """
{ "index": { "op_type": "create" } }
{ "@timestamp": "%s", "name": "%s" }
""";;
private static final String[] NAMES = { "skywalker", "leia", "obi-wan", "yoda", "chewbacca", "r2-d2", "c-3po", "darth-vader" };
@ClassRule
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.distribution(DistributionType.DEFAULT)
.setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial")
.build();
@Override
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}
@Before
public void assumeSnapshotBuild() {
assumeTrue("/{index}/ccr/shard_changes endpoint only available in snapshot builds", Build.current().isSnapshot());
}
public void testShardChangesNoOperation() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(
indexName,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
.build()
);
assertTrue(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
assertOK(client().performRequest(shardChangesRequest));
}
public void testShardChangesDefaultParams() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
final Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
.build();
final String mappings = """
{
"properties": {
"name": {
"type": "keyword"
}
}
}
""";
createIndex(indexName, settings, mappings);
assertTrue(indexExists(indexName));
assertOK(bulkIndex(indexName, randomIntBetween(10, 20)));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
final Response response = client().performRequest(shardChangesRequest);
assertOK(response);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
indexName
);
}
public void testDataStreamShardChangesDefaultParams() throws IOException {
final String templateName = randomAlphanumericOfLength(8).toLowerCase(Locale.ROOT);
assertOK(createIndexTemplate(templateName, """
{
"index_patterns": [ "test-*-*" ],
"data_stream": {},
"priority": 100,
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"type": "keyword"
}
}
}
}
}"""));
final String dataStreamName = "test-"
+ randomAlphanumericOfLength(5).toLowerCase(Locale.ROOT)
+ "-"
+ randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
assertOK(createDataStream(dataStreamName));
assertOK(bulkIndex(dataStreamName, randomIntBetween(10, 20)));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(dataStreamName));
final Response response = client().performRequest(shardChangesRequest);
assertOK(response);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
dataStreamName
);
}
public void testIndexAliasShardChangesDefaultParams() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
final String aliasName = randomAlphanumericOfLength(8).toLowerCase(Locale.ROOT);
final Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
.build();
final String mappings = """
{
"properties": {
"name": {
"type": "keyword"
}
}
}
""";
createIndex(indexName, settings, mappings);
assertTrue(indexExists(indexName));
final Request putAliasRequest = new Request("PUT", "/" + indexName + "/_alias/" + aliasName);
assertOK(client().performRequest(putAliasRequest));
assertOK(bulkIndex(aliasName, randomIntBetween(10, 20)));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(aliasName));
final Response response = client().performRequest(shardChangesRequest);
assertOK(response);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
aliasName
);
}
public void testShardChangesWithAllParameters() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(
indexName,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
.build()
);
assertTrue(indexExists(indexName));
assertOK(bulkIndex(indexName, randomIntBetween(100, 200)));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
shardChangesRequest.addParameter("from_seq_no", "0");
shardChangesRequest.addParameter("max_operations_count", "1");
shardChangesRequest.addParameter("poll_timeout", "10s");
shardChangesRequest.addParameter("max_batch_size", "1MB");
final Response response = client().performRequest(shardChangesRequest);
assertOK(response);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false),
indexName
);
}
public void testShardChangesMultipleRequests() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(
indexName,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
.build()
);
assertTrue(indexExists(indexName));
assertOK(bulkIndex(indexName, randomIntBetween(100, 200)));
final Request firstRequest = new Request("GET", shardChangesEndpoint(indexName));
firstRequest.addParameter("from_seq_no", "0");
firstRequest.addParameter("max_operations_count", "10");
firstRequest.addParameter("poll_timeout", "10s");
firstRequest.addParameter("max_batch_size", "1MB");
final Response firstResponse = client().performRequest(firstRequest);
assertOK(firstResponse);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(firstResponse.getEntity()), false),
indexName
);
final Request secondRequest = new Request("GET", shardChangesEndpoint(indexName));
secondRequest.addParameter("from_seq_no", "10");
secondRequest.addParameter("max_operations_count", "10");
secondRequest.addParameter("poll_timeout", "10s");
secondRequest.addParameter("max_batch_size", "1MB");
final Response secondResponse = client().performRequest(secondRequest);
assertOK(secondResponse);
assertShardChangesResponse(
XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(secondResponse.getEntity()), false),
indexName
);
}
public void testShardChangesInvalidFromSeqNo() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName);
assertTrue(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
shardChangesRequest.addParameter("from_seq_no", "-1");
final ResponseException ex = assertThrows(ResponseException.class, () -> client().performRequest(shardChangesRequest));
assertResponseException(ex, RestStatus.BAD_REQUEST, "Validation Failed: 1: fromSeqNo [-1] cannot be lower than 0");
}
public void testShardChangesInvalidMaxOperationsCount() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName);
assertTrue(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
shardChangesRequest.addParameter("max_operations_count", "-1");
final ResponseException ex = assertThrows(ResponseException.class, () -> client().performRequest(shardChangesRequest));
assertResponseException(ex, RestStatus.BAD_REQUEST, "Validation Failed: 1: maxOperationCount [-1] cannot be lower than 0");
}
public void testShardChangesNegativePollTimeout() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName);
assertTrue(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
shardChangesRequest.addParameter("poll_timeout", "-1s");
assertOK(client().performRequest(shardChangesRequest));
}
public void testShardChangesInvalidMaxBatchSize() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
createIndex(indexName);
assertTrue(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
shardChangesRequest.addParameter("max_batch_size", "-1MB");
final ResponseException ex = assertThrows(ResponseException.class, () -> client().performRequest(shardChangesRequest));
assertResponseException(
ex,
RestStatus.BAD_REQUEST,
"failed to parse setting [max_batch_size] with value [-1MB] as a size in bytes"
);
}
public void testShardChangesMissingIndex() throws IOException {
final String indexName = randomAlphanumericOfLength(10).toLowerCase(Locale.ROOT);
assertFalse(indexExists(indexName));
final Request shardChangesRequest = new Request("GET", shardChangesEndpoint(indexName));
final ResponseException ex = assertThrows(ResponseException.class, () -> client().performRequest(shardChangesRequest));
assertResponseException(ex, RestStatus.BAD_REQUEST, "Failed to process shard changes for index [" + indexName + "]");
}
private static Response bulkIndex(final String indexName, int numberOfDocuments) throws IOException {
final StringBuilder sb = new StringBuilder();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < numberOfDocuments; i++) {
sb.append(
String.format(
Locale.ROOT,
BULK_INDEX_TEMPLATE,
Instant.ofEpochMilli(timestamp).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME),
randomFrom(NAMES)
)
);
timestamp += 1000; // 1 second
}
final Request request = new Request("POST", bulkEndpoint(indexName));
request.setJsonEntity(sb.toString());
request.addParameter("refresh", "true");
return client().performRequest(request);
}
private Response createDataStream(final String dataStreamName) throws IOException {
return client().performRequest(new Request("PUT", dataStreamEndpoint(dataStreamName)));
}
private static Response createIndexTemplate(final String templateName, final String mappings) throws IOException {
final Request request = new Request("PUT", indexTemplateEndpoint(templateName));
request.setJsonEntity(mappings);
return client().performRequest(request);
}
private static String shardChangesEndpoint(final String indexName) {
return String.format(Locale.ROOT, CCR_SHARD_CHANGES_ENDPOINT, indexName);
}
private static String bulkEndpoint(final String indexName) {
return String.format(Locale.ROOT, BULK_INDEX_ENDPOINT, indexName);
}
private static String dataStreamEndpoint(final String dataStreamName) {
return String.format(Locale.ROOT, DATA_STREAM_ENDPOINT, dataStreamName);
}
private static String indexTemplateEndpoint(final String templateName) {
return String.format(Locale.ROOT, INDEX_TEMPLATE_ENDPOINT, templateName);
}
private void assertResponseException(final ResponseException ex, final RestStatus restStatus, final String error) {
assertEquals(restStatus.getStatus(), ex.getResponse().getStatusLine().getStatusCode());
assertThat(ex.getMessage(), Matchers.containsString(error));
}
private void assertShardChangesResponse(final Map<String, Object> shardChangesResponseBody, final String indexAbstractionName) {
for (final String fieldName : SHARD_RESPONSE_FIELDS) {
final Object fieldValue = shardChangesResponseBody.get(fieldName);
assertNotNull("Field " + fieldName + " is missing or has a null value.", fieldValue);
if ("index_abstraction".equals(fieldName)) {
assertEquals(indexAbstractionName, fieldValue);
}
if ("operations".equals(fieldName)) {
if (fieldValue instanceof List<?> operationsList) {
assertFalse("Field 'operations' is empty.", operationsList.isEmpty());
for (final Object operation : operationsList) {
assertNotNull("Operation is null.", operation);
if (operation instanceof Map<?, ?> operationMap) {
assertNotNull("seq_no is missing in operation.", operationMap.get("seq_no"));
assertNotNull("op_type is missing in operation.", operationMap.get("op_type"));
assertNotNull("primary_term is missing in operation.", operationMap.get("primary_term"));
}
}
}
}
}
}
}

50
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
View file

@ -7,7 +7,6 @@
package org.elasticsearch.xpack.ccr; package org.elasticsearch.xpack.ccr;
import org.apache.lucene.util.SetOnce; import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Build;
import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersion;
import org.elasticsearch.TransportVersions; import org.elasticsearch.TransportVersions;
import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequest;
@ -92,7 +91,6 @@ import org.elasticsearch.xpack.ccr.rest.RestPutAutoFollowPatternAction;
import org.elasticsearch.xpack.ccr.rest.RestPutFollowAction; import org.elasticsearch.xpack.ccr.rest.RestPutFollowAction;
import org.elasticsearch.xpack.ccr.rest.RestResumeAutoFollowPatternAction; import org.elasticsearch.xpack.ccr.rest.RestResumeAutoFollowPatternAction;
import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction; import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction;
import org.elasticsearch.xpack.ccr.rest.RestShardChangesAction;
import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction; import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
import org.elasticsearch.xpack.core.XPackFeatureUsage; import org.elasticsearch.xpack.core.XPackFeatureUsage;
import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.XPackField;
@ -114,7 +112,6 @@ import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask;
import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; import org.elasticsearch.xpack.core.ccr.action.UnfollowAction;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
@ -143,34 +140,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
public static final String REQUESTED_OPS_MISSING_METADATA_KEY = "es.requested_operations_missing"; public static final String REQUESTED_OPS_MISSING_METADATA_KEY = "es.requested_operations_missing";
public static final TransportVersion TRANSPORT_VERSION_ACTION_WITH_SHARD_ID = TransportVersions.V_8_9_X; public static final TransportVersion TRANSPORT_VERSION_ACTION_WITH_SHARD_ID = TransportVersions.V_8_9_X;
private static final List<RestHandler> BASE_REST_HANDLERS = Arrays.asList(
// stats API
new RestFollowStatsAction(),
new RestCcrStatsAction(),
new RestFollowInfoAction(),
// follow APIs
new RestPutFollowAction(),
new RestResumeFollowAction(),
new RestPauseFollowAction(),
new RestUnfollowAction(),
// auto-follow APIs
new RestDeleteAutoFollowPatternAction(),
new RestPutAutoFollowPatternAction(),
new RestGetAutoFollowPatternAction(),
new RestPauseAutoFollowPatternAction(),
new RestResumeAutoFollowPatternAction(),
// forget follower API
new RestForgetFollowerAction()
);
private static final List<RestHandler> REST_HANDLERS = Collections.unmodifiableList(BASE_REST_HANDLERS);
private static final List<RestHandler> SNAPSHOT_BUILD_REST_HANDLERS;
static {
List<RestHandler> snapshotBuildHandlers = new ArrayList<>(BASE_REST_HANDLERS);
snapshotBuildHandlers.add(new RestShardChangesAction());
SNAPSHOT_BUILD_REST_HANDLERS = Collections.unmodifiableList(snapshotBuildHandlers);
}
private final boolean enabled; private final boolean enabled;
private final Settings settings; private final Settings settings;
private final CcrLicenseChecker ccrLicenseChecker; private final CcrLicenseChecker ccrLicenseChecker;
@ -302,7 +272,25 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
return emptyList(); return emptyList();
} }
return Build.current().isSnapshot() ? SNAPSHOT_BUILD_REST_HANDLERS : REST_HANDLERS; return Arrays.asList(
// stats API
new RestFollowStatsAction(),
new RestCcrStatsAction(),
new RestFollowInfoAction(),
// follow APIs
new RestPutFollowAction(),
new RestResumeFollowAction(),
new RestPauseFollowAction(),
new RestUnfollowAction(),
// auto-follow APIs
new RestDeleteAutoFollowPatternAction(),
new RestPutAutoFollowPatternAction(),
new RestGetAutoFollowPatternAction(),
new RestPauseAutoFollowPatternAction(),
new RestResumeAutoFollowPatternAction(),
// forget follower API
new RestForgetFollowerAction()
);
} }
public List<NamedWriteableRegistry.Entry> getNamedWriteables() { public List<NamedWriteableRegistry.Entry> getNamedWriteables() {

366
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestShardChangesAction.java
View file

@ -1,366 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.ccr.rest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.client.internal.node.NodeClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.RestUtils;
import org.elasticsearch.rest.action.RestActionListener;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xpack.ccr.Ccr;
import org.elasticsearch.xpack.ccr.action.ShardChangesAction;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import static org.elasticsearch.rest.RestRequest.Method.GET;
/**
* A REST handler that retrieves shard changes in a specific index, data stream or alias whose name is
* provided as a parameter. It handles GET requests to the "/{index}/ccr/shard_changes" endpoint retrieving
* shard-level changes, such as Translog operations, mapping version, settings version, aliases version,
* the global checkpoint, maximum sequence number and maximum sequence number of updates or deletes.
* <p>
* In the case of a data stream, the first backing index is considered the target for retrieving shard changes.
* In the case of an alias, the first index that the alias points to is considered the target for retrieving
* shard changes.
* <p>
* Note: This handler is only available for snapshot builds.
*/
public class RestShardChangesAction extends BaseRestHandler {
private static final long DEFAULT_FROM_SEQ_NO = 0L;
private static final ByteSizeValue DEFAULT_MAX_BATCH_SIZE = ByteSizeValue.of(32, ByteSizeUnit.MB);
private static final TimeValue DEFAULT_POLL_TIMEOUT = new TimeValue(1, TimeUnit.MINUTES);
private static final int DEFAULT_MAX_OPERATIONS_COUNT = 1024;
private static final int DEFAULT_TIMEOUT_SECONDS = 60;
private static final TimeValue GET_INDEX_UUID_TIMEOUT = new TimeValue(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
private static final TimeValue SHARD_STATS_TIMEOUT = new TimeValue(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
private static final String INDEX_PARAM_NAME = "index";
private static final String FROM_SEQ_NO_PARAM_NAME = "from_seq_no";
private static final String MAX_BATCH_SIZE_PARAM_NAME = "max_batch_size";
private static final String POLL_TIMEOUT_PARAM_NAME = "poll_timeout";
private static final String MAX_OPERATIONS_COUNT_PARAM_NAME = "max_operations_count";
@Override
public String getName() {
return "ccr_shard_changes_action";
}
@Override
public List<Route> routes() {
return List.of(new Route(GET, "/{index}/ccr/shard_changes"));
}
/**
* Prepares the request for retrieving shard changes.
*
* @param restRequest The REST request.
* @param client The NodeClient for executing the request.
* @return A RestChannelConsumer for handling the request.
* @throws IOException If an error occurs while preparing the request.
*/
@Override
protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
final var indexAbstractionName = restRequest.param(INDEX_PARAM_NAME);
final var fromSeqNo = restRequest.paramAsLong(FROM_SEQ_NO_PARAM_NAME, DEFAULT_FROM_SEQ_NO);
final var maxBatchSize = restRequest.paramAsSize(MAX_BATCH_SIZE_PARAM_NAME, DEFAULT_MAX_BATCH_SIZE);
final var pollTimeout = restRequest.paramAsTime(POLL_TIMEOUT_PARAM_NAME, DEFAULT_POLL_TIMEOUT);
final var maxOperationsCount = restRequest.paramAsInt(MAX_OPERATIONS_COUNT_PARAM_NAME, DEFAULT_MAX_OPERATIONS_COUNT);
// NOTE: we first retrieve the concrete index name in case we are dealing with an alias or data stream.
// Then we use the concrete index name to retrieve the index UUID and shard stats.
final CompletableFuture<String> indexNameCompletableFuture = asyncGetIndexName(
client,
indexAbstractionName,
client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME)
);
final CompletableFuture<String> indexUUIDCompletableFuture = indexNameCompletableFuture.thenCompose(
concreteIndexName -> asyncGetIndexUUID(
client,
concreteIndexName,
client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME),
RestUtils.getMasterNodeTimeout(restRequest)
)
);
final CompletableFuture<ShardStats> shardStatsCompletableFuture = indexNameCompletableFuture.thenCompose(
concreteIndexName -> asyncShardStats(client, concreteIndexName, client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME))
);
return channel -> CompletableFuture.allOf(indexUUIDCompletableFuture, shardStatsCompletableFuture).thenRun(() -> {
try {
final String concreteIndexName = indexNameCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
final String indexUUID = indexUUIDCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
final ShardStats shardStats = shardStatsCompletableFuture.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
final ShardId shardId = shardStats.getShardRouting().shardId();
final String expectedHistoryUUID = shardStats.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY);
final ShardChangesAction.Request shardChangesRequest = shardChangesRequest(
concreteIndexName,
indexUUID,
shardId,
expectedHistoryUUID,
fromSeqNo,
maxBatchSize,
pollTimeout,
maxOperationsCount
);
client.execute(ShardChangesAction.INSTANCE, shardChangesRequest, new RestActionListener<>(channel) {
@Override
protected void processResponse(final ShardChangesAction.Response response) {
channel.sendResponse(
new RestResponse(
RestStatus.OK,
shardChangesResponseToXContent(response, indexAbstractionName, concreteIndexName, shardId)
)
);
}
});
} catch (InterruptedException | ExecutionException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Error while retrieving shard changes", e);
} catch (TimeoutException te) {
throw new IllegalStateException("Timeout while waiting for shard stats or index UUID", te);
}
}).exceptionally(ex -> {
channel.sendResponse(
new RestResponse(
RestStatus.BAD_REQUEST,
"Failed to process shard changes for index [" + indexAbstractionName + "] " + ex.getMessage()
)
);
return null;
});
}
/**
* Creates a ShardChangesAction.Request object with the provided parameters.
*
* @param indexName The name of the index for which to retrieve shard changes.
* @param indexUUID The UUID of the index.
* @param shardId The ShardId for which to retrieve shard changes.
* @param expectedHistoryUUID The expected history UUID of the shard.
* @param fromSeqNo The sequence number from which to start retrieving shard changes.
* @param maxBatchSize The maximum size of a batch of operations to retrieve.
* @param pollTimeout The maximum time to wait for shard changes.
* @param maxOperationsCount The maximum number of operations to retrieve in a single request.
* @return A ShardChangesAction.Request object with the provided parameters.
*/
private static ShardChangesAction.Request shardChangesRequest(
final String indexName,
final String indexUUID,
final ShardId shardId,
final String expectedHistoryUUID,
long fromSeqNo,
final ByteSizeValue maxBatchSize,
final TimeValue pollTimeout,
int maxOperationsCount
) {
final ShardChangesAction.Request shardChangesRequest = new ShardChangesAction.Request(
new ShardId(new Index(indexName, indexUUID), shardId.id()),
expectedHistoryUUID
);
shardChangesRequest.setFromSeqNo(fromSeqNo);
shardChangesRequest.setMaxBatchSize(maxBatchSize);
shardChangesRequest.setPollTimeout(pollTimeout);
shardChangesRequest.setMaxOperationCount(maxOperationsCount);
return shardChangesRequest;
}
/**
* Converts the response to XContent JSOn format.
*
* @param response The ShardChangesAction response.
* @param indexAbstractionName The name of the index abstraction.
* @param concreteIndexName The name of the index.
* @param shardId The ShardId.
*/
private static XContentBuilder shardChangesResponseToXContent(
final ShardChangesAction.Response response,
final String indexAbstractionName,
final String concreteIndexName,
final ShardId shardId
) {
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
builder.startObject();
builder.field("index_abstraction", indexAbstractionName);
builder.field("index", concreteIndexName);
builder.field("shard_id", shardId);
builder.field("mapping_version", response.getMappingVersion());
builder.field("settings_version", response.getSettingsVersion());
builder.field("aliases_version", response.getAliasesVersion());
builder.field("global_checkpoint", response.getGlobalCheckpoint());
builder.field("max_seq_no", response.getMaxSeqNo());
builder.field("max_seq_no_of_updates_or_deletes", response.getMaxSeqNoOfUpdatesOrDeletes());
builder.field("took_in_millis", response.getTookInMillis());
if (response.getOperations() != null && response.getOperations().length > 0) {
operationsToXContent(response, builder);
}
builder.endObject();
return builder;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Converts the operations from a ShardChangesAction response to XContent JSON format.
*
* @param response The ShardChangesAction response containing the operations to be converted.
* @param builder The XContentBuilder to which the converted operations will be added.
* @throws IOException If an error occurs while writing to the XContentBuilder.
*/
private static void operationsToXContent(final ShardChangesAction.Response response, final XContentBuilder builder) throws IOException {
builder.field("number_of_operations", response.getOperations().length);
builder.field("operations");
builder.startArray();
for (final Translog.Operation operation : response.getOperations()) {
builder.startObject();
builder.field("op_type", operation.opType());
builder.field("seq_no", operation.seqNo());
builder.field("primary_term", operation.primaryTerm());
builder.endObject();
}
builder.endArray();
}
/**
* Execute an asynchronous task using a task supplier and an executor service.
*
* @param <T> The type of data to be retrieved.
* @param task The supplier task that provides the data.
* @param executorService The executorService service for executing the asynchronous task.
* @param errorMessage The error message to be thrown if the task execution fails.
* @return A CompletableFuture that completes with the retrieved data.
*/
private static <T> CompletableFuture<T> supplyAsyncTask(
final Supplier<T> task,
final ExecutorService executorService,
final String errorMessage
) {
return CompletableFuture.supplyAsync(() -> {
try {
return task.get();
} catch (Exception e) {
throw new ElasticsearchException(errorMessage, e);
}
}, executorService);
}
/**
* Asynchronously retrieves the index name for a given index, alias or data stream.
* If the name represents a data stream, the name of the first backing index is returned.
* If the name represents an alias, the name of the first index that the alias points to is returned.
*
* @param client The NodeClient for executing the asynchronous request.
* @param indexAbstractionName The name of the index, alias or data stream.
* @return A CompletableFuture that completes with the retrieved index name.
*/
private static CompletableFuture<String> asyncGetIndexName(
final NodeClient client,
final String indexAbstractionName,
final ExecutorService executorService
) {
return supplyAsyncTask(() -> {
final ClusterState clusterState = client.admin()
.cluster()
.prepareState(new TimeValue(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
.get(GET_INDEX_UUID_TIMEOUT)
.getState();
final IndexAbstraction indexAbstraction = clusterState.metadata().getProject().getIndicesLookup().get(indexAbstractionName);
if (indexAbstraction == null) {
throw new IllegalArgumentException(
String.format(Locale.ROOT, "Invalid index or data stream name [%s]", indexAbstractionName)
);
}
if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM
|| indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
return indexAbstraction.getIndices().getFirst().getName();
}
return indexAbstractionName;
}, executorService, "Error while retrieving index name for index or data stream [" + indexAbstractionName + "]");
}
/**
* Asynchronously retrieves the shard stats for a given index using an executor service.
*
* @param client The NodeClient for executing the asynchronous request.
* @param concreteIndexName The name of the index for which to retrieve shard statistics.
* @param executorService The executorService service for executing the asynchronous task.
* @return A CompletableFuture that completes with the retrieved ShardStats.
* @throws ElasticsearchException If an error occurs while retrieving shard statistics.
*/
private static CompletableFuture<ShardStats> asyncShardStats(
final NodeClient client,
final String concreteIndexName,
final ExecutorService executorService
) {
return supplyAsyncTask(
() -> Arrays.stream(client.admin().indices().prepareStats(concreteIndexName).clear().get(SHARD_STATS_TIMEOUT).getShards())
.max(Comparator.comparingLong(shardStats -> shardStats.getCommitStats().getGeneration()))
.orElseThrow(() -> new ElasticsearchException("Unable to retrieve shard stats for index: " + concreteIndexName)),
executorService,
"Error while retrieving shard stats for index [" + concreteIndexName + "]"
);
}
/**
* Asynchronously retrieves the index UUID for a given index using an executor service.
*
* @param client The NodeClient for executing the asynchronous request.
* @param concreteIndexName The name of the index for which to retrieve the index UUID.
* @param executorService The executorService service for executing the asynchronous task.
* @param masterTimeout The timeout for waiting until the cluster is unblocked.
* @return A CompletableFuture that completes with the retrieved index UUID.
* @throws ElasticsearchException If an error occurs while retrieving the index UUID.
*/
private static CompletableFuture<String> asyncGetIndexUUID(
final NodeClient client,
final String concreteIndexName,
final ExecutorService executorService,
TimeValue masterTimeout
) {
return supplyAsyncTask(
() -> client.admin()
.indices()
.prepareGetIndex(masterTimeout)
.setIndices(concreteIndexName)
.get(GET_INDEX_UUID_TIMEOUT)
.getSetting(concreteIndexName, IndexMetadata.SETTING_INDEX_UUID),
executorService,
"Error while retrieving index UUID for index [" + concreteIndexName + "]"
);
}
}

77
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackBuild.java
View file

@ -1,77 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.core;
import org.elasticsearch.core.PathUtils;
import org.elasticsearch.core.SuppressForbidden;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.jar.JarInputStream;
import java.util.jar.Manifest;
/**
* Information about the built version of x-pack that is running.
*/
public class XPackBuild {
public static final XPackBuild CURRENT;
static {
final String shortHash;
final String date;
Path path = getElasticsearchCodebase();
if (path.toString().endsWith(".jar")) {
try (JarInputStream jar = new JarInputStream(Files.newInputStream(path))) {
Manifest manifest = jar.getManifest();
shortHash = manifest.getMainAttributes().getValue("Change");
date = manifest.getMainAttributes().getValue("Build-Date");
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
// not running from a jar (unit tests, IDE)
shortHash = "Unknown";
date = "Unknown";
}
CURRENT = new XPackBuild(shortHash, date);
}
/**
* Returns path to xpack codebase path
*/
@SuppressForbidden(reason = "looks up path of xpack.jar directly")
static Path getElasticsearchCodebase() {
URL url = XPackBuild.class.getProtectionDomain().getCodeSource().getLocation();
try {
return PathUtils.get(url.toURI());
} catch (URISyntaxException bogus) {
throw new RuntimeException(bogus);
}
}
private String shortHash;
private String date;
XPackBuild(String shortHash, String date) {
this.shortHash = shortHash;
this.date = date;
}
public String shortHash() {
return shortHash;
}
public String date() {
return date;
}
}

4
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
View file

@ -6,6 +6,7 @@
*/ */
package org.elasticsearch.xpack.core.action; package org.elasticsearch.xpack.core.action;
import org.elasticsearch.Build;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionFilters;
@ -23,7 +24,6 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.Featur
import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo;
import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.XPackBuild;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -58,7 +58,7 @@ public class TransportXPackInfoAction extends HandledTransportAction<XPackInfoRe
XPackInfoResponse.BuildInfo buildInfo = null; XPackInfoResponse.BuildInfo buildInfo = null;
if (request.getCategories().contains(XPackInfoRequest.Category.BUILD)) { if (request.getCategories().contains(XPackInfoRequest.Category.BUILD)) {
buildInfo = new XPackInfoResponse.BuildInfo(XPackBuild.CURRENT.shortHash(), XPackBuild.CURRENT.date()); buildInfo = new XPackInfoResponse.BuildInfo(Build.current().hash(), Build.current().date());
} }
LicenseInfo licenseInfo = null; LicenseInfo licenseInfo = null;

2
x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
View file

@ -35,7 +35,6 @@ import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction;
import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState; import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState;
import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; import org.elasticsearch.xpack.core.downsample.DownsampleShardTask;
@ -66,7 +65,6 @@ public class Downsample extends Plugin implements ActionPlugin, PersistentTaskPl
@Override @Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() { public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return List.of( return List.of(
new ActionHandler<>(DownsampleIndexerAction.INSTANCE, TransportDownsampleIndexerAction.class),
new ActionHandler<>(DownsampleAction.INSTANCE, TransportDownsampleAction.class), new ActionHandler<>(DownsampleAction.INSTANCE, TransportDownsampleAction.class),
new ActionHandler<>( new ActionHandler<>(
DownsampleShardPersistentTaskExecutor.DelegatingAction.INSTANCE, DownsampleShardPersistentTaskExecutor.DelegatingAction.INSTANCE,

2
x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
View file

@ -25,8 +25,6 @@ import java.util.Map;
* - Add a constant for its name, following the naming conventions for metrics. * - Add a constant for its name, following the naming conventions for metrics.
* - Register it in method {@link #doStart}. * - Register it in method {@link #doStart}.
* - Add a function for recording its value. * - Add a function for recording its value.
* - If needed, inject {@link DownsampleMetrics} to the action containing the logic
* that records the metric value. For reference, see {@link TransportDownsampleIndexerAction}.
*/ */
public class DownsampleMetrics extends AbstractLifecycleComponent { public class DownsampleMetrics extends AbstractLifecycleComponent {

2
x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
View file

@ -104,7 +104,7 @@ import static org.elasticsearch.xpack.core.ilm.DownsampleAction.DOWNSAMPLED_INDE
/** /**
* The master downsample action that coordinates * The master downsample action that coordinates
* - creating the downsample index * - creating the downsample index
* - instantiating {@link DownsampleShardIndexer}s to index downsample documents * - instantiating {@link org.elasticsearch.persistent.PersistentTasksExecutor} to start a persistent downsample task
* - cleaning up state * - cleaning up state
*/ */
public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAction<DownsampleAction.Request> { public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAction<DownsampleAction.Request> {

206
x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
View file

@ -1,206 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.downsample;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.NoShardAvailableActionException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.client.internal.OriginSettingClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ProjectState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.project.ProjectResolver;
import org.elasticsearch.cluster.routing.ShardIterator;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.injection.guice.Inject;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction;
import org.elasticsearch.xpack.core.downsample.DownsampleShardIndexerStatus;
import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState;
import org.elasticsearch.xpack.core.downsample.DownsampleShardTask;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* A {@link TransportBroadcastAction} that downsamples all the shards of a source index into a new downsample index.
*
* TODO: Enforce that we don't retry on another replica if we throw an error after sending some buckets.
*/
public class TransportDownsampleIndexerAction extends TransportBroadcastAction<
DownsampleIndexerAction.Request,
DownsampleIndexerAction.Response,
DownsampleIndexerAction.ShardDownsampleRequest,
DownsampleIndexerAction.ShardDownsampleResponse> {
private final Client client;
private final IndicesService indicesService;
private final ProjectResolver projectResolver;
private final DownsampleMetrics downsampleMetrics;
@Inject
public TransportDownsampleIndexerAction(
Client client,
ClusterService clusterService,
TransportService transportService,
IndicesService indicesService,
ActionFilters actionFilters,
ProjectResolver projectResolver,
IndexNameExpressionResolver indexNameExpressionResolver,
DownsampleMetrics downsampleMetrics
) {
super(
DownsampleIndexerAction.NAME,
clusterService,
transportService,
actionFilters,
indexNameExpressionResolver,
DownsampleIndexerAction.Request::new,
DownsampleIndexerAction.ShardDownsampleRequest::new,
transportService.getThreadPool().executor(Downsample.DOWNSAMPLE_TASK_THREAD_POOL_NAME)
);
this.client = new OriginSettingClient(client, ClientHelper.ROLLUP_ORIGIN);
this.indicesService = indicesService;
this.projectResolver = projectResolver;
this.downsampleMetrics = downsampleMetrics;
}
@Override
protected List<ShardIterator> shards(ClusterState clusterState, DownsampleIndexerAction.Request request, String[] concreteIndices) {
if (concreteIndices.length > 1) {
throw new IllegalArgumentException("multiple indices: " + Arrays.toString(concreteIndices));
}
ProjectState project = projectResolver.getProjectState(clusterState);
final List<ShardIterator> groups = clusterService.operationRouting().searchShards(project, concreteIndices, null, null);
for (ShardIterator group : groups) {
// fails fast if any non-active groups
if (group.size() == 0) {
throw new NoShardAvailableActionException(group.shardId());
}
}
return groups;
}
@Override
protected ClusterBlockException checkGlobalBlock(ClusterState state, DownsampleIndexerAction.Request request) {
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
}
@Override
protected ClusterBlockException checkRequestBlock(
ClusterState state,
DownsampleIndexerAction.Request request,
String[] concreteIndices
) {
return state.blocks().indicesBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_WRITE, concreteIndices);
}
@Override
protected void doExecute(
Task task,
DownsampleIndexerAction.Request request,
ActionListener<DownsampleIndexerAction.Response> listener
) {
new Async(task, request, listener).start();
}
@Override
protected DownsampleIndexerAction.ShardDownsampleRequest newShardRequest(
int numShards,
ShardRouting shard,
DownsampleIndexerAction.Request request
) {
return new DownsampleIndexerAction.ShardDownsampleRequest(shard.shardId(), request);
}
@Override
protected DownsampleIndexerAction.ShardDownsampleResponse shardOperation(
DownsampleIndexerAction.ShardDownsampleRequest request,
Task task
) throws IOException {
IndexService indexService = indicesService.indexService(request.shardId().getIndex());
DownsampleShardIndexer indexer = new DownsampleShardIndexer(
(DownsampleShardTask) task,
client,
indexService,
downsampleMetrics,
request.shardId(),
request.getDownsampleIndex(),
request.getRollupConfig(),
request.getMetricFields(),
request.getLabelFields(),
request.getDimensionFields(),
new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null)
);
return indexer.execute();
}
@Override
protected DownsampleIndexerAction.ShardDownsampleResponse readShardResponse(StreamInput in) throws IOException {
return new DownsampleIndexerAction.ShardDownsampleResponse(in);
}
@Override
protected DownsampleIndexerAction.Response newResponse(
DownsampleIndexerAction.Request request,
AtomicReferenceArray<?> shardsResponses,
ClusterState clusterState
) {
long numIndexed = 0;
int successfulShards = 0;
for (int i = 0; i < shardsResponses.length(); i++) {
Object shardResponse = shardsResponses.get(i);
if (shardResponse == null) {
throw new ElasticsearchException("missing shard");
} else if (shardResponse instanceof DownsampleIndexerAction.ShardDownsampleResponse r) {
successfulShards++;
numIndexed += r.getNumIndexed();
} else if (shardResponse instanceof Exception e) {
throw new ElasticsearchException(e);
} else {
assert false : "unknown response [" + shardResponse + "]";
throw new IllegalStateException("unknown response [" + shardResponse + "]");
}
}
return new DownsampleIndexerAction.Response(true, shardsResponses.length(), successfulShards, 0, numIndexed);
}
private class Async extends AsyncBroadcastAction {
private final DownsampleIndexerAction.Request request;
private final ActionListener<DownsampleIndexerAction.Response> listener;
protected Async(Task task, DownsampleIndexerAction.Request request, ActionListener<DownsampleIndexerAction.Response> listener) {
super(task, request, listener);
this.request = request;
this.listener = listener;
}
@Override
protected void finishHim() {
try {
DownsampleIndexerAction.Response resp = newResponse(request, shardsResponses, clusterService.state());
listener.onResponse(resp);
} catch (Exception e) {
listener.onFailure(e);
}
}
}
}

4
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
View file

@ -105,6 +105,10 @@ public final class Source implements Writeable {
return text + location; return text + location;
} }
/**
* @deprecated Sources created by this can't be correctly deserialized. For use in tests only.
*/
@Deprecated
public static Source synthetic(String text) { public static Source synthetic(String text) {
return new Source(Location.EMPTY, text); return new Source(Location.EMPTY, text);
} }

15
x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/Clusters.java
View file

@ -14,15 +14,22 @@ import org.elasticsearch.test.cluster.util.Version;
public class Clusters { public class Clusters {
public static ElasticsearchCluster mixedVersionCluster() { public static ElasticsearchCluster mixedVersionCluster() {
Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); Version oldVersion = Version.fromString(System.getProperty("tests.old_cluster_version"));
return ElasticsearchCluster.local() var cluster = ElasticsearchCluster.local()
.distribution(DistributionType.DEFAULT) .distribution(DistributionType.DEFAULT)
.withNode(node -> node.version(oldVersion)) .withNode(node -> node.version(oldVersion))
.withNode(node -> node.version(Version.CURRENT)) .withNode(node -> node.version(Version.CURRENT))
.withNode(node -> node.version(oldVersion)) .withNode(node -> node.version(oldVersion))
.withNode(node -> node.version(Version.CURRENT)) .withNode(node -> node.version(Version.CURRENT))
.setting("xpack.security.enabled", "false") .setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial") .setting("xpack.license.self_generated.type", "trial");
.setting("cluster.routing.rebalance.enable", "none") // disable relocation until we have retry in ESQL if (supportRetryOnShardFailures(oldVersion) == false) {
.build(); cluster.setting("cluster.routing.rebalance.enable", "none");
}
return cluster.build();
}
private static boolean supportRetryOnShardFailures(Version version) {
return version.onOrAfter(Version.fromString("9.1.0"))
|| (version.onOrAfter(Version.fromString("8.19.0")) && version.before(Version.fromString("9.0.0")));
} }
} }

31
x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java
View file

@ -17,17 +17,20 @@ public class Clusters {
static final String LOCAL_CLUSTER_NAME = "local_cluster"; static final String LOCAL_CLUSTER_NAME = "local_cluster";
public static ElasticsearchCluster remoteCluster() { public static ElasticsearchCluster remoteCluster() {
return ElasticsearchCluster.local() Version version = distributionVersion("tests.version.remote_cluster");
var cluster = ElasticsearchCluster.local()
.name(REMOTE_CLUSTER_NAME) .name(REMOTE_CLUSTER_NAME)
.distribution(DistributionType.DEFAULT) .distribution(DistributionType.DEFAULT)
.version(distributionVersion("tests.version.remote_cluster")) .version(version)
.nodes(2) .nodes(2)
.setting("node.roles", "[data,ingest,master]") .setting("node.roles", "[data,ingest,master]")
.setting("xpack.security.enabled", "false") .setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial") .setting("xpack.license.self_generated.type", "trial")
.shared(true) .shared(true);
.setting("cluster.routing.rebalance.enable", "none") if (supportRetryOnShardFailures(version) == false) {
.build(); cluster.setting("cluster.routing.rebalance.enable", "none");
}
return cluster.build();
} }
public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteCluster) { public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteCluster) {
@ -35,10 +38,11 @@ public class Clusters {
} }
public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteCluster, Boolean skipUnavailable) { public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteCluster, Boolean skipUnavailable) {
return ElasticsearchCluster.local() Version version = distributionVersion("tests.version.local_cluster");
var cluster = ElasticsearchCluster.local()
.name(LOCAL_CLUSTER_NAME) .name(LOCAL_CLUSTER_NAME)
.distribution(DistributionType.DEFAULT) .distribution(DistributionType.DEFAULT)
.version(distributionVersion("tests.version.local_cluster")) .version(version)
.nodes(2) .nodes(2)
.setting("xpack.security.enabled", "false") .setting("xpack.security.enabled", "false")
.setting("xpack.license.self_generated.type", "trial") .setting("xpack.license.self_generated.type", "trial")
@ -46,9 +50,11 @@ public class Clusters {
.setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"") .setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"")
.setting("cluster.remote.connections_per_cluster", "1") .setting("cluster.remote.connections_per_cluster", "1")
.setting("cluster.remote." + REMOTE_CLUSTER_NAME + ".skip_unavailable", skipUnavailable.toString()) .setting("cluster.remote." + REMOTE_CLUSTER_NAME + ".skip_unavailable", skipUnavailable.toString())
.shared(true) .shared(true);
.setting("cluster.routing.rebalance.enable", "none") if (supportRetryOnShardFailures(version) == false) {
.build(); cluster.setting("cluster.routing.rebalance.enable", "none");
}
return cluster.build();
} }
public static org.elasticsearch.Version localClusterVersion() { public static org.elasticsearch.Version localClusterVersion() {
@ -65,4 +71,9 @@ public class Clusters {
final String val = System.getProperty(key); final String val = System.getProperty(key);
return val != null ? Version.fromString(val) : Version.CURRENT; return val != null ? Version.fromString(val) : Version.CURRENT;
} }
private static boolean supportRetryOnShardFailures(Version version) {
return version.onOrAfter(Version.fromString("9.1.0"))
|| (version.onOrAfter(Version.fromString("8.19.0")) && version.before(Version.fromString("9.0.0")));
}
} }

45
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
View file

@ -64,6 +64,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull;
import org.elasticsearch.xpack.esql.parser.ExpressionBuilder; import org.elasticsearch.xpack.esql.parser.ExpressionBuilder;
import org.elasticsearch.xpack.esql.planner.Layout; import org.elasticsearch.xpack.esql.planner.Layout;
@ -102,6 +103,7 @@ import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext;
import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize;
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.mapParam; import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.mapParam;
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.param; import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.param;
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.paramWithoutAnnotation; import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.paramWithoutAnnotation;
@ -331,7 +333,7 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : "";
String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition); String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition);
String name = types.get(badArgPosition).typeName(); String name = types.get(badArgPosition).typeName();
return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]"; return ordinal + "argument of [source] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]";
} }
@FunctionalInterface @FunctionalInterface
@ -522,7 +524,7 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
* <strong>except</strong> those that have been marked with {@link TestCaseSupplier.TypedData#forceLiteral()}. * <strong>except</strong> those that have been marked with {@link TestCaseSupplier.TypedData#forceLiteral()}.
*/ */
protected final Expression buildFieldExpression(TestCaseSupplier.TestCase testCase) { protected final Expression buildFieldExpression(TestCaseSupplier.TestCase testCase) {
return build(testCase.getSource(), testCase.getDataAsFields()); return randomSerializeDeserialize(build(testCase.getSource(), testCase.getDataAsFields()));
} }
/** /**
@ -531,12 +533,47 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
* those that have been marked with {@link TestCaseSupplier.TypedData#forceLiteral()}. * those that have been marked with {@link TestCaseSupplier.TypedData#forceLiteral()}.
*/ */
protected final Expression buildDeepCopyOfFieldExpression(TestCaseSupplier.TestCase testCase) { protected final Expression buildDeepCopyOfFieldExpression(TestCaseSupplier.TestCase testCase) {
// We don't use `randomSerializeDeserialize()` here as the deep copied fields aren't deserializable right now
return build(testCase.getSource(), testCase.getDataAsDeepCopiedFields()); return build(testCase.getSource(), testCase.getDataAsDeepCopiedFields());
} }
private Expression randomSerializeDeserialize(Expression expression) {
if (randomBoolean()) {
return expression;
}
return serializeDeserializeExpression(expression);
}
/**
* Returns the expression after being serialized and deserialized.
* <p>
* Tests randomly go through this method to ensure that the function retains the same logic after serialization and deserialization.
* </p>
* <p>
* Can be overridden to provide custom serialization and deserialization logic, or disable it if needed.
* </p>
*/
protected Expression serializeDeserializeExpression(Expression expression) {
Expression newExpression = serializeDeserialize(
expression,
PlanStreamOutput::writeNamedWriteable,
in -> in.readNamedWriteable(Expression.class),
testCase.getConfiguration() // The configuration query should be == to the source text of the function for this to work
);
// Fields use synthetic sources, which can't be serialized. So we replace with the originals instead.
var dummyChildren = newExpression.children()
.stream()
.<Expression>map(c -> new Literal(Source.EMPTY, "anything that won't match any test case", c.dataType()))
.toList();
// We first replace them with other unrelated expressions to force a replace, as some replaceChildren() will check for equality
return newExpression.replaceChildrenSameSize(dummyChildren).replaceChildrenSameSize(expression.children());
}
protected final Expression buildLiteralExpression(TestCaseSupplier.TestCase testCase) { protected final Expression buildLiteralExpression(TestCaseSupplier.TestCase testCase) {
assumeTrue("Data can't be converted to literals", testCase.canGetDataAsLiterals()); assumeTrue("Data can't be converted to literals", testCase.canGetDataAsLiterals());
return build(testCase.getSource(), testCase.getDataAsLiterals()); return randomSerializeDeserialize(build(testCase.getSource(), testCase.getDataAsLiterals()));
} }
public static EvaluatorMapper.ToEvaluator toEvaluator() { public static EvaluatorMapper.ToEvaluator toEvaluator() {
@ -711,7 +748,7 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
} }
public void testSerializationOfSimple() { public void testSerializationOfSimple() {
assertSerialization(buildFieldExpression(testCase)); assertSerialization(buildFieldExpression(testCase), testCase.getConfiguration());
} }
/** /**

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
View file

@ -401,8 +401,8 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
evaluator + "[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", evaluator + "[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
dataType, dataType,
is(nullValue()) is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning("Line -1:-1: java.lang.ArithmeticException: " + typeNameOverflow) .withWarning("Line 1:1: java.lang.ArithmeticException: " + typeNameOverflow)
); );
} }
} }

17
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
View file

@ -16,12 +16,15 @@ import org.elasticsearch.geo.ShapeTestUtils;
import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger; import org.elasticsearch.logging.Logger;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.expression.MapExpression;
import org.elasticsearch.xpack.esql.core.tree.Location;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.core.util.NumericUtils;
import org.elasticsearch.xpack.esql.session.Configuration;
import org.elasticsearch.xpack.versionfield.Version; import org.elasticsearch.xpack.versionfield.Version;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
@ -54,6 +57,9 @@ public record TestCaseSupplier(String name, List<DataType> types, Supplier<TestC
implements implements
Supplier<TestCaseSupplier.TestCase> { Supplier<TestCaseSupplier.TestCase> {
public static final Source TEST_SOURCE = new Source(new Location(1, 0), "source");
public static final Configuration TEST_CONFIGURATION = EsqlTestUtils.configuration(TEST_SOURCE.text());
private static final Logger logger = LogManager.getLogger(TestCaseSupplier.class); private static final Logger logger = LogManager.getLogger(TestCaseSupplier.class);
/** /**
@ -1388,6 +1394,10 @@ public record TestCaseSupplier(String name, List<DataType> types, Supplier<TestC
* The {@link Source} this test case should be run with * The {@link Source} this test case should be run with
*/ */
private final Source source; private final Source source;
/**
* The {@link Configuration} this test case should use
*/
private final Configuration configuration;
/** /**
* The parameter values and types to pass into the function for this test run * The parameter values and types to pass into the function for this test run
*/ */
@ -1490,7 +1500,8 @@ public record TestCaseSupplier(String name, List<DataType> types, Supplier<TestC
Object extra, Object extra,
boolean canBuildEvaluator boolean canBuildEvaluator
) { ) {
this.source = Source.EMPTY; this.source = TEST_SOURCE;
this.configuration = TEST_CONFIGURATION;
this.data = data; this.data = data;
this.evaluatorToString = evaluatorToString; this.evaluatorToString = evaluatorToString;
this.expectedType = expectedType == null ? null : expectedType.noText(); this.expectedType = expectedType == null ? null : expectedType.noText();
@ -1510,6 +1521,10 @@ public record TestCaseSupplier(String name, List<DataType> types, Supplier<TestC
return source; return source;
} }
public Configuration getConfiguration() {
return configuration;
}
public List<TypedData> getData() { public List<TypedData> getData() {
return data; return data;
} }

14
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java
View file

@ -241,7 +241,7 @@ public class TopTests extends AbstractAggregationTestCase {
new TestCaseSupplier.TypedData(0, DataType.INTEGER, "limit").forceLiteral(), new TestCaseSupplier.TypedData(0, DataType.INTEGER, "limit").forceLiteral(),
new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral()
), ),
"Limit must be greater than 0 in [], found [0]" "Limit must be greater than 0 in [source], found [0]"
) )
), ),
new TestCaseSupplier( new TestCaseSupplier(
@ -252,7 +252,7 @@ public class TopTests extends AbstractAggregationTestCase {
new TestCaseSupplier.TypedData(2, DataType.INTEGER, "limit").forceLiteral(), new TestCaseSupplier.TypedData(2, DataType.INTEGER, "limit").forceLiteral(),
new TestCaseSupplier.TypedData(new BytesRef("wrong-order"), DataType.KEYWORD, "order").forceLiteral() new TestCaseSupplier.TypedData(new BytesRef("wrong-order"), DataType.KEYWORD, "order").forceLiteral()
), ),
"Invalid order value in [], expected [ASC, DESC] but got [wrong-order]" "Invalid order value in [source], expected [ASC, DESC] but got [wrong-order]"
) )
), ),
new TestCaseSupplier( new TestCaseSupplier(
@ -263,7 +263,7 @@ public class TopTests extends AbstractAggregationTestCase {
new TestCaseSupplier.TypedData(null, DataType.INTEGER, "limit").forceLiteral(), new TestCaseSupplier.TypedData(null, DataType.INTEGER, "limit").forceLiteral(),
new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral()
), ),
"second argument of [] cannot be null, received [limit]" "second argument of [source] cannot be null, received [limit]"
) )
), ),
new TestCaseSupplier( new TestCaseSupplier(
@ -274,7 +274,7 @@ public class TopTests extends AbstractAggregationTestCase {
new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(), new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(),
new TestCaseSupplier.TypedData(null, DataType.KEYWORD, "order").forceLiteral() new TestCaseSupplier.TypedData(null, DataType.KEYWORD, "order").forceLiteral()
), ),
"third argument of [] cannot be null, received [order]" "third argument of [source] cannot be null, received [order]"
) )
) )
) )
@ -317,4 +317,10 @@ public class TopTests extends AbstractAggregationTestCase {
); );
}); });
} }
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
// TODO: This aggregation doesn't serialize the Source, and must be fixed.
return expression;
}
} }

17
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java
View file

@ -19,11 +19,13 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.FunctionName;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.function.Supplier; import java.util.function.Supplier;
import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize;
import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
@ -83,4 +85,19 @@ public class MatchTests extends AbstractMatchFullTextFunctionTests {
} }
return match; return match;
} }
/**
* Copy of the overridden method that doesn't check for children size, as the {@code options} child isn't serialized in Match.
*/
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
Expression newExpression = serializeDeserialize(
expression,
PlanStreamOutput::writeNamedWriteable,
in -> in.readNamedWriteable(Expression.class),
testCase.getConfiguration() // The configuration query should be == to the source text of the function for this to work
);
// Fields use synthetic sources, which can't be serialized. So we use the originals instead.
return newExpression.replaceChildren(expression.children());
}
} }

32
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
View file

@ -8,7 +8,6 @@
package org.elasticsearch.xpack.esql.expression.function.scalar; package org.elasticsearch.xpack.esql.expression.function.scalar;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils;
@ -27,10 +26,22 @@ public abstract class AbstractConfigurationFunctionTestCase extends AbstractScal
@Override @Override
protected Expression build(Source source, List<Expression> args) { protected Expression build(Source source, List<Expression> args) {
return buildWithConfiguration(source, args, EsqlTestUtils.TEST_CFG); return buildWithConfiguration(source, args, testCase.getConfiguration());
} }
static Configuration randomConfiguration() { public void testSerializationWithConfiguration() {
Configuration config = randomConfiguration();
Expression expr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), config);
assertSerialization(expr, config);
Configuration differentConfig = randomValueOtherThan(config, AbstractConfigurationFunctionTestCase::randomConfiguration);
Expression differentExpr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), differentConfig);
assertNotEquals(expr, differentExpr);
}
private static Configuration randomConfiguration() {
// TODO: Randomize the query and maybe the pragmas. // TODO: Randomize the query and maybe the pragmas.
return new Configuration( return new Configuration(
randomZone(), randomZone(),
@ -47,19 +58,4 @@ public abstract class AbstractConfigurationFunctionTestCase extends AbstractScal
randomBoolean() randomBoolean()
); );
} }
public void testSerializationWithConfiguration() {
Configuration config = randomConfiguration();
Expression expr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), config);
assertSerialization(expr, config);
Configuration differentConfig;
do {
differentConfig = randomConfiguration();
} while (config.equals(differentConfig));
Expression differentExpr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), differentConfig);
assertFalse(expr.equals(differentExpr));
}
} }

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java
View file

@ -75,4 +75,10 @@ public class FromAggregateMetricDoubleTests extends AbstractScalarFunctionTestCa
) )
); );
} }
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
// AggregateMetricDoubleLiteral can't be serialized when it's a literal
return expression;
}
} }

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
View file

@ -49,8 +49,8 @@ public class ToCartesianPointTests extends AbstractScalarFunctionTestCase {
bytesRef -> { bytesRef -> {
var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString())); var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: " + exception "Line 1:1: " + exception
); );
} }
); );

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
View file

@ -50,8 +50,8 @@ public class ToCartesianShapeTests extends AbstractScalarFunctionTestCase {
bytesRef -> { bytesRef -> {
var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString())); var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: " + exception "Line 1:1: " + exception
); );
} }
); );

20
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java
View file

@ -70,8 +70,8 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase {
Long.MIN_VALUE, Long.MIN_VALUE,
-1L, -1L,
List.of( List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." "Line 1:1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."
) )
); );
TestCaseSupplier.forUnaryUnsignedLong( TestCaseSupplier.forUnaryUnsignedLong(
@ -91,8 +91,8 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase {
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO),
UNSIGNED_LONG_MAX, UNSIGNED_LONG_MAX,
bi -> List.of( bi -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range"
) )
); );
TestCaseSupplier.forUnaryDouble( TestCaseSupplier.forUnaryDouble(
@ -103,8 +103,8 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
-Double.MIN_VALUE, -Double.MIN_VALUE,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." "Line 1:1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."
) )
); );
TestCaseSupplier.forUnaryDouble( TestCaseSupplier.forUnaryDouble(
@ -115,8 +115,8 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase {
9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18)
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range"
) )
); );
TestCaseSupplier.forUnaryStrings( TestCaseSupplier.forUnaryStrings(
@ -125,8 +125,8 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase {
DataType.DATE_NANOS, DataType.DATE_NANOS,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: " "Line 1:1: java.lang.IllegalArgumentException: "
+ (bytesRef.utf8ToString().isEmpty() + (bytesRef.utf8ToString().isEmpty()
? "cannot parse empty datetime" ? "cannot parse empty datetime"
: ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]")) : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]"))

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java
View file

@ -83,4 +83,10 @@ public class ToDatePeriodTests extends AbstractScalarFunctionTestCase {
public void testSerializationOfSimple() { public void testSerializationOfSimple() {
assertTrue("Serialization test does not apply", true); assertTrue("Serialization test does not apply", true);
} }
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
// Can't be serialized
return expression;
}
} }

24
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
View file

@ -81,8 +81,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO),
UNSIGNED_LONG_MAX, UNSIGNED_LONG_MAX,
bi -> List.of( bi -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range"
) )
); );
TestCaseSupplier.forUnaryDouble( TestCaseSupplier.forUnaryDouble(
@ -93,8 +93,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
-9.223372036854777E18, // a "convenient" value smaller than `(double) Long.MIN_VALUE` (== ...776E18) -9.223372036854777E18, // a "convenient" value smaller than `(double) Long.MIN_VALUE` (== ...776E18)
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range"
) )
); );
TestCaseSupplier.forUnaryDouble( TestCaseSupplier.forUnaryDouble(
@ -105,8 +105,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18)
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range"
) )
); );
TestCaseSupplier.forUnaryStrings( TestCaseSupplier.forUnaryStrings(
@ -115,8 +115,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
DataType.DATETIME, DataType.DATETIME,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: " "Line 1:1: java.lang.IllegalArgumentException: "
+ (bytesRef.utf8ToString().isEmpty() + (bytesRef.utf8ToString().isEmpty()
? "cannot parse empty datetime" ? "cannot parse empty datetime"
: ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time]")) : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time]"))
@ -151,8 +151,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
DataType.DATETIME, DataType.DATETIME,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: failed to parse date field [" "Line 1:1: java.lang.IllegalArgumentException: failed to parse date field ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "] with format [strict_date_optional_time]" + "] with format [strict_date_optional_time]"
) )
@ -171,8 +171,8 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase {
DataType.DATETIME, DataType.DATETIME,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: failed to parse date field [" "Line 1:1: java.lang.IllegalArgumentException: failed to parse date field ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "] with format [strict_date_optional_time]" + "] with format [strict_date_optional_time]"
) )

8
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
View file

@ -67,8 +67,8 @@ public class ToDegreesTests extends AbstractScalarFunctionTestCase {
double deg = Math.toDegrees(d); double deg = Math.toDegrees(d);
ArrayList<String> warnings = new ArrayList<>(2); ArrayList<String> warnings = new ArrayList<>(2);
if (Double.isNaN(deg) || Double.isInfinite(deg)) { if (Double.isNaN(deg) || Double.isInfinite(deg)) {
warnings.add("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded."); warnings.add("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.");
warnings.add("Line -1:-1: java.lang.ArithmeticException: not a finite double number: " + deg); warnings.add("Line 1:1: java.lang.ArithmeticException: not a finite double number: " + deg);
} }
return warnings; return warnings;
}); });
@ -84,8 +84,8 @@ public class ToDegreesTests extends AbstractScalarFunctionTestCase {
DataType.DOUBLE, DataType.DOUBLE,
d -> null, d -> null,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.ArithmeticException: not a finite double number: " + ((double) d > 0 ? "Infinity" : "-Infinity") "Line 1:1: java.lang.ArithmeticException: not a finite double number: " + ((double) d > 0 ? "Infinity" : "-Infinity")
) )
); );

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
View file

@ -65,8 +65,8 @@ public class ToDoubleTests extends AbstractScalarFunctionTestCase {
() -> EsqlDataTypeConverter.stringToDouble(bytesRef.utf8ToString()) () -> EsqlDataTypeConverter.stringToDouble(bytesRef.utf8ToString())
); );
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: " + exception "Line 1:1: " + exception
); );
}); });
TestCaseSupplier.forUnaryUnsignedLong( TestCaseSupplier.forUnaryUnsignedLong(

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
View file

@ -44,8 +44,8 @@ public class ToGeoPointTests extends AbstractScalarFunctionTestCase {
TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_POINT, bytesRef -> null, bytesRef -> { TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_POINT, bytesRef -> null, bytesRef -> {
var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString())); var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: " + exception "Line 1:1: " + exception
); );
}); });
// strings that are geo point representations // strings that are geo point representations

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
View file

@ -45,8 +45,8 @@ public class ToGeoShapeTests extends AbstractScalarFunctionTestCase {
TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_SHAPE, bytesRef -> null, bytesRef -> { TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_SHAPE, bytesRef -> null, bytesRef -> {
var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString())); var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: " + exception "Line 1:1: " + exception
); );
}); });
// strings that are geo_shape representations // strings that are geo_shape representations

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
View file

@ -47,8 +47,8 @@ public class ToIPTests extends AbstractScalarFunctionTestCase {
DataType.IP, DataType.IP,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.IllegalArgumentException: '" + bytesRef.utf8ToString() + "' is not an IP string literal." "Line 1:1: java.lang.IllegalArgumentException: '" + bytesRef.utf8ToString() + "' is not an IP string literal."
) )
); );

36
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
View file

@ -60,8 +60,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
DataType.INTEGER, DataType.INTEGER,
l -> null, l -> null,
l -> List.of( l -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: ["
+ ((Instant) l).toEpochMilli() + ((Instant) l).toEpochMilli()
+ "] out of [integer] range" + "] out of [integer] range"
) )
@ -73,8 +73,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
DataType.INTEGER, DataType.INTEGER,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ bytesRef.utf8ToString() + bytesRef.utf8ToString()
+ "]" + "]"
) )
@ -98,8 +98,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
Integer.MIN_VALUE - 1d, Integer.MIN_VALUE - 1d,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [integer] range"
) )
); );
// from doubles outside Integer's range, positive // from doubles outside Integer's range, positive
@ -111,8 +111,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
Integer.MAX_VALUE + 1d, Integer.MAX_VALUE + 1d,
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [integer] range"
) )
); );
@ -135,8 +135,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
BigInteger.valueOf(Integer.MAX_VALUE).add(BigInteger.ONE), BigInteger.valueOf(Integer.MAX_VALUE).add(BigInteger.ONE),
UNSIGNED_LONG_MAX, UNSIGNED_LONG_MAX,
ul -> List.of( ul -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + ul + "] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + ul + "] out of [integer] range"
) )
); );
@ -160,8 +160,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
Long.MIN_VALUE, Long.MIN_VALUE,
Integer.MIN_VALUE - 1L, Integer.MIN_VALUE - 1L,
l -> List.of( l -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [integer] range"
) )
); );
@ -174,8 +174,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + 1L,
Long.MAX_VALUE, Long.MAX_VALUE,
l -> List.of( l -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [integer] range"
) )
); );
@ -232,8 +232,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
DataType.INTEGER, DataType.INTEGER,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "]" + "]"
) )
@ -255,8 +255,8 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase {
DataType.INTEGER, DataType.INTEGER,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "]" + "]"
) )

24
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
View file

@ -59,8 +59,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
DataType.LONG, DataType.LONG,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ bytesRef.utf8ToString() + bytesRef.utf8ToString()
+ "]" + "]"
) )
@ -84,8 +84,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
Long.MIN_VALUE - 1d, Long.MIN_VALUE - 1d,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range"
) )
); );
// from doubles outside long's range, positive // from doubles outside long's range, positive
@ -97,8 +97,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
Long.MAX_VALUE + 1d, Long.MAX_VALUE + 1d,
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range"
) )
); );
@ -120,8 +120,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE),
UNSIGNED_LONG_MAX, UNSIGNED_LONG_MAX,
ul -> List.of( ul -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + ul + "] out of [long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + ul + "] out of [long] range"
) )
); );
@ -190,8 +190,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
DataType.LONG, DataType.LONG,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "]" + "]"
) )
@ -213,8 +213,8 @@ public class ToLongTests extends AbstractScalarFunctionTestCase {
DataType.LONG, DataType.LONG,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "]" + "]"
) )

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java
View file

@ -82,4 +82,10 @@ public class ToTimeDurationTests extends AbstractScalarFunctionTestCase {
public void testSerializationOfSimple() { public void testSerializationOfSimple() {
assertTrue("Serialization test does not apply", true); assertTrue("Serialization test does not apply", true);
} }
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
// Can't be serialized
return expression;
}
} }

28
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
View file

@ -74,8 +74,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
// this shortcut here. // this shortcut here.
Exception e = expectThrows(NumberFormatException.class, () -> new BigDecimal(bytesRef.utf8ToString())); Exception e = expectThrows(NumberFormatException.class, () -> new BigDecimal(bytesRef.utf8ToString()));
return List.of( return List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.NumberFormatException: " + e.getMessage() "Line 1:1: java.lang.NumberFormatException: " + e.getMessage()
); );
}); });
// from doubles within unsigned_long's range // from doubles within unsigned_long's range
@ -97,8 +97,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
-1d, -1d,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [unsigned_long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [unsigned_long] range"
) )
); );
// from doubles outside Long's range, positive // from doubles outside Long's range, positive
@ -110,8 +110,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
UNSIGNED_LONG_MAX_AS_DOUBLE + 10e5, UNSIGNED_LONG_MAX_AS_DOUBLE + 10e5,
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
d -> List.of( d -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [unsigned_long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [unsigned_long] range"
) )
); );
@ -134,8 +134,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
Long.MIN_VALUE, Long.MIN_VALUE,
-1L, -1L,
l -> List.of( l -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [unsigned_long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [unsigned_long] range"
) )
); );
@ -158,8 +158,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
Integer.MIN_VALUE, Integer.MIN_VALUE,
-1, -1,
l -> List.of( l -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [unsigned_long] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [unsigned_long] range"
) )
); );
@ -216,8 +216,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
DataType.UNSIGNED_LONG, DataType.UNSIGNED_LONG,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "] out of [unsigned_long] range" + "] out of [unsigned_long] range"
) )
@ -239,8 +239,8 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase {
DataType.UNSIGNED_LONG, DataType.UNSIGNED_LONG,
bytesRef -> null, bytesRef -> null,
bytesRef -> List.of( bytesRef -> List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: ["
+ ((BytesRef) bytesRef).utf8ToString() + ((BytesRef) bytesRef).utf8ToString()
+ "] out of [unsigned_long] range" + "] out of [unsigned_long] range"
) )

12
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
View file

@ -91,7 +91,7 @@ public class DateDiffTests extends AbstractScalarFunctionTestCase {
zdtStart2, zdtStart2,
zdtEnd2, zdtEnd2,
"nanoseconds", "nanoseconds",
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [300000000000] out of [integer] range" "Line 1:1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [300000000000] out of [integer] range"
) )
); );
@ -241,7 +241,7 @@ public class DateDiffTests extends AbstractScalarFunctionTestCase {
+ "endTimestamp=Attribute[channel=2]]", + "endTimestamp=Attribute[channel=2]]",
DataType.INTEGER, DataType.INTEGER,
equalTo(null) equalTo(null)
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning(warning) .withWarning(warning)
), ),
// Units as text case // Units as text case
@ -258,7 +258,7 @@ public class DateDiffTests extends AbstractScalarFunctionTestCase {
+ "endTimestamp=Attribute[channel=2]]", + "endTimestamp=Attribute[channel=2]]",
DataType.INTEGER, DataType.INTEGER,
equalTo(null) equalTo(null)
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning(warning) .withWarning(warning)
) )
); );
@ -268,4 +268,10 @@ public class DateDiffTests extends AbstractScalarFunctionTestCase {
protected Expression build(Source source, List<Expression> args) { protected Expression build(Source source, List<Expression> args) {
return new DateDiff(source, args.get(0), args.get(1), args.get(2)); return new DateDiff(source, args.get(0), args.get(1), args.get(2));
} }
@Override
protected Expression serializeDeserializeExpression(Expression expression) {
// TODO: This function doesn't serialize the Source, and must be fixed.
return expression;
}
} }

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
View file

@ -93,12 +93,12 @@ public class DateExtractTests extends AbstractConfigurationFunctionTestCase {
"DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]",
DataType.LONG, DataType.LONG,
is(nullValue()) is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning( .withWarning(
"Line -1:-1: java.lang.IllegalArgumentException: " "Line 1:1: java.lang.IllegalArgumentException: "
+ "No enum constant java.time.temporal.ChronoField.NOT A UNIT" + "No enum constant java.time.temporal.ChronoField.NOT A UNIT"
) )
.withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") .withFoldingException(InvalidArgumentException.class, "invalid date field for [source]: not a unit")
) )
) )
); );

10
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
View file

@ -117,13 +117,13 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME, DataType.DATETIME,
is(nullValue()) is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning( .withWarning(
"Line -1:-1: java.lang.IllegalArgumentException: Invalid format: " + "[not a format]: Unknown pattern letter: o" "Line 1:1: java.lang.IllegalArgumentException: Invalid format: [not a format]: Unknown pattern letter: o"
) )
.withFoldingException( .withFoldingException(
InvalidArgumentException.class, InvalidArgumentException.class,
"invalid date pattern for []: Invalid format: [not a format]: Unknown pattern letter: o" "invalid date pattern for [source]: Invalid format: [not a format]: Unknown pattern letter: o"
) )
), ),
new TestCaseSupplier( new TestCaseSupplier(
@ -137,9 +137,9 @@ public class DateParseTests extends AbstractScalarFunctionTestCase {
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME, DataType.DATETIME,
is(nullValue()) is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning( .withWarning(
"Line -1:-1: java.lang.IllegalArgumentException: " "Line 1:1: java.lang.IllegalArgumentException: "
+ "failed to parse date field [not a date] with format [yyyy-MM-dd]" + "failed to parse date field [not a date] with format [yyyy-MM-dd]"
) )
) )

5
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
View file

@ -10,7 +10,6 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date;
import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
@ -42,7 +41,7 @@ public class NowTests extends AbstractConfigurationFunctionTestCase {
List.of(), List.of(),
matchesPattern("LiteralsEvaluator\\[lit=.*]"), matchesPattern("LiteralsEvaluator\\[lit=.*]"),
DataType.DATETIME, DataType.DATETIME,
equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli()) equalTo(TestCaseSupplier.TEST_CONFIGURATION.now().toInstant().toEpochMilli())
) )
) )
) )
@ -56,7 +55,7 @@ public class NowTests extends AbstractConfigurationFunctionTestCase {
@Override @Override
protected Matcher<Object> allNullsMatcher() { protected Matcher<Object> allNullsMatcher() {
return equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli()); return equalTo(testCase.getConfiguration().now().toInstant().toEpochMilli());
} }
} }

8
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
View file

@ -38,8 +38,8 @@ public class AcosTests extends AbstractScalarFunctionTestCase {
Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY,
Math.nextDown(-1d), Math.nextDown(-1d),
List.of( List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.ArithmeticException: Acos input out of range" "Line 1:1: java.lang.ArithmeticException: Acos input out of range"
) )
) )
); );
@ -51,8 +51,8 @@ public class AcosTests extends AbstractScalarFunctionTestCase {
Math.nextUp(1d), Math.nextUp(1d),
Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY,
List.of( List.of(
"Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.",
"Line -1:-1: java.lang.ArithmeticException: Acos input out of range" "Line 1:1: java.lang.ArithmeticException: Acos input out of range"
) )
) )
); );

Some files were not shown because too many files have changed in this diff Show more