github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 23:27:25 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge main into multi-project

Browse source
This commit is contained in:
Niels Bauman 2025-02-03 20:50:15 +10:00
commit 1b342b26ef
196 changed files with 4203 additions and 1535 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/pipelines/intake.template.yml
View file

@ -96,6 +96,7 @@ steps:
# - trigger: elasticsearch-dra-workflow
# label: Trigger DRA snapshot workflow
# async: true
# branches: "main 9.* 8.* 7.17"
# build:
# branch: "$BUILDKITE_BRANCH"
# commit: "$BUILDKITE_COMMIT"

2
.buildkite/pipelines/intake.yml
View file

@ -97,7 +97,7 @@ steps:
# - trigger: elasticsearch-dra-workflow
# label: Trigger DRA snapshot workflow
# async: true
# branches: "main 8.* 7.17"
# branches: "main 9.* 8.* 7.17"
# build:
# branch: "$BUILDKITE_BRANCH"
# commit: "$BUILDKITE_COMMIT"

5
docs/changelog/117176.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 117176
summary: Integrate IBM watsonx to Inference API for re-ranking task
area: Experiences
type: enhancement
issues: []

5
docs/changelog/120852.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 120852
summary: Correct line and column numbers of missing named parameters
area: ES|QL
type: bug
issues: []

5
docs/changelog/121260.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 121260
summary: Introduce a pre-mapping logical plan processing step
area: ES|QL
type: enhancement
issues: []

6
docs/changelog/121324.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 121324
summary: Support duplicate suggestions in completion field
area: Suggesters
type: bug
issues:
- 82432

6
docs/changelog/121428.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 121428
summary: Fix infer on and elasticsearch service endpoint created with a deployment
id
area: Machine Learning
type: bug
issues: []

2
docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
View file

@ -286,7 +286,7 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
/**
* Compares the results of running two analyzers against many random
* strings. The goal is to figure out if two anlayzers are "the same" by
* strings. The goal is to figure out if two analyzers are "the same" by
* comparing their results. This is far from perfect but should be fairly
* accurate, especially for gross things like missing {@code decimal_digit}
* token filters, and should be fairly fast because it compares a fairly

18
libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.core;
/**
* A {@link java.util.function.Supplier}-like interface which allows throwing checked exceptions.
*/
@FunctionalInterface
public interface CheckedSupplier<T, E extends Exception> {
T get() throws E;
}

5
libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java
View file

@ -52,10 +52,9 @@ import javax.net.ssl.SSLSocketFactory;
* <p>
* A bit like Mockito but way more painful.
*/
class DummyImplementations {
static class DummyLocaleServiceProvider extends LocaleServiceProvider {
public class DummyImplementations {
public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
@Override
public Locale[] getAvailableLocales() {
throw unexpected();

29
libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.entitlement.qa.test;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface EntitlementTest {
enum ExpectedAccess {
PLUGINS,
ES_MODULES_ONLY,
ALWAYS_DENIED
}
ExpectedAccess expectedAccess();
int fromJavaVersion() default -1;
}

11
libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java
View file

@ -22,6 +22,8 @@ import java.nio.file.Paths;
import java.nio.file.attribute.UserPrincipal;
import java.util.Scanner;
import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS;
@SuppressForbidden(reason = "Explicitly checking APIs that are forbidden")
class FileCheckActions {
@ -43,38 +45,47 @@ class FileCheckActions {
return testRootDir.resolve("read_write_file");
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createScannerFile() throws FileNotFoundException {
new Scanner(readFile().toFile());
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createScannerFileWithCharset() throws IOException {
new Scanner(readFile().toFile(), StandardCharsets.UTF_8);
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createScannerFileWithCharsetName() throws FileNotFoundException {
new Scanner(readFile().toFile(), "UTF-8");
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileOutputStreamString() throws IOException {
new FileOutputStream(readWriteFile().toString()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileOutputStreamStringWithAppend() throws IOException {
new FileOutputStream(readWriteFile().toString(), false).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileOutputStreamFile() throws IOException {
new FileOutputStream(readWriteFile().toFile()).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void createFileOutputStreamFileWithAppend() throws IOException {
new FileOutputStream(readWriteFile().toFile(), false).close();
}
@EntitlementTest(expectedAccess = PLUGINS)
static void filesProbeContentType() throws IOException {
Files.probeContentType(readFile());
}
@EntitlementTest(expectedAccess = PLUGINS)
static void filesSetOwner() throws IOException {
UserPrincipal owner = EntitledActions.getFileOwner(readWriteFile());
Files.setOwner(readWriteFile(), owner); // set to existing owner, just trying to execute the method

335
libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java
View file

@ -33,6 +33,9 @@ import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.HttpURLConnection;
@ -51,8 +54,10 @@ import java.net.URLConnection;
import java.net.URLStreamHandler;
import java.net.spi.URLStreamHandlerProvider;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -61,6 +66,7 @@ import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import static java.util.Map.entry;
import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS;
import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.alwaysDenied;
import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.deniedToPlugins;
import static org.elasticsearch.entitlement.qa.test.RestEntitlementsCheckAction.CheckAction.forPlugins;
@ -88,143 +94,188 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
}
}
private static final Map<String, CheckAction> checkActions = Stream.<Map.Entry<String, CheckAction>>of(
entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)),
entry("set_https_connection_properties", forPlugins(RestEntitlementsCheckAction::setHttpsConnectionProperties)),
entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)),
entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)),
entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)),
entry("system_setIn", alwaysDenied(RestEntitlementsCheckAction::system$$setIn)),
entry("system_setOut", alwaysDenied(RestEntitlementsCheckAction::system$$setOut)),
entry("system_setErr", alwaysDenied(RestEntitlementsCheckAction::system$$setErr)),
entry("runtime_addShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$addShutdownHook)),
entry("runtime_removeShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$$removeShutdownHook)),
entry(
"thread_setDefaultUncaughtExceptionHandler",
alwaysDenied(RestEntitlementsCheckAction::thread$$setDefaultUncaughtExceptionHandler)
private static final Map<String, CheckAction> checkActions = Stream.concat(
Stream.<Entry<String, CheckAction>>of(
entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)),
entry("set_https_connection_properties", forPlugins(RestEntitlementsCheckAction::setHttpsConnectionProperties)),
entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)),
entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)),
entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)),
entry("system_setIn", alwaysDenied(RestEntitlementsCheckAction::system$$setIn)),
entry("system_setOut", alwaysDenied(RestEntitlementsCheckAction::system$$setOut)),
entry("system_setErr", alwaysDenied(RestEntitlementsCheckAction::system$$setErr)),
entry("runtime_addShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$addShutdownHook)),
entry("runtime_removeShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$$removeShutdownHook)),
entry(
"thread_setDefaultUncaughtExceptionHandler",
alwaysDenied(RestEntitlementsCheckAction::thread$$setDefaultUncaughtExceptionHandler)
),
entry("localeServiceProvider", alwaysDenied(RestEntitlementsCheckAction::localeServiceProvider$)),
entry("breakIteratorProvider", alwaysDenied(RestEntitlementsCheckAction::breakIteratorProvider$)),
entry("collatorProvider", alwaysDenied(RestEntitlementsCheckAction::collatorProvider$)),
entry("dateFormatProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatProvider$)),
entry("dateFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatSymbolsProvider$)),
entry("decimalFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::decimalFormatSymbolsProvider$)),
entry("numberFormatProvider", alwaysDenied(RestEntitlementsCheckAction::numberFormatProvider$)),
entry("calendarDataProvider", alwaysDenied(RestEntitlementsCheckAction::calendarDataProvider$)),
entry("calendarNameProvider", alwaysDenied(RestEntitlementsCheckAction::calendarNameProvider$)),
entry("currencyNameProvider", alwaysDenied(RestEntitlementsCheckAction::currencyNameProvider$)),
entry("localeNameProvider", alwaysDenied(RestEntitlementsCheckAction::localeNameProvider$)),
entry("timeZoneNameProvider", alwaysDenied(RestEntitlementsCheckAction::timeZoneNameProvider$)),
entry("logManager", alwaysDenied(RestEntitlementsCheckAction::logManager$)),
entry("locale_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultLocale)),
entry("locale_setDefaultForCategory", alwaysDenied(WritePropertiesCheckActions::setDefaultLocaleForCategory)),
entry("timeZone_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultTimeZone)),
entry("system_setProperty", forPlugins(WritePropertiesCheckActions::setSystemProperty)),
entry("system_clearProperty", forPlugins(WritePropertiesCheckActions::clearSystemProperty)),
entry("system_setSystemProperties", alwaysDenied(WritePropertiesCheckActions::setSystemProperties)),
// This group is a bit nasty: if entitlements don't prevent these, then networking is
// irreparably borked for the remainder of the test run.
entry(
"datagramSocket_setDatagramSocketImplFactory",
alwaysDenied(RestEntitlementsCheckAction::datagramSocket$$setDatagramSocketImplFactory)
),
entry("httpURLConnection_setFollowRedirects", alwaysDenied(RestEntitlementsCheckAction::httpURLConnection$$setFollowRedirects)),
entry("serverSocket_setSocketFactory", alwaysDenied(RestEntitlementsCheckAction::serverSocket$$setSocketFactory)),
entry("socket_setSocketImplFactory", alwaysDenied(RestEntitlementsCheckAction::socket$$setSocketImplFactory)),
entry("url_setURLStreamHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::url$$setURLStreamHandlerFactory)),
entry("urlConnection_setFileNameMap", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setFileNameMap)),
entry(
"urlConnection_setContentHandlerFactory",
alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)
),
entry("proxySelector_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultProxySelector)),
entry("responseCache_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultResponseCache)),
entry(
"createInetAddressResolverProvider",
new CheckAction(VersionSpecificNetworkChecks::createInetAddressResolverProvider, true, 18)
),
entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)),
entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)),
entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)),
entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)),
entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)),
entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)),
entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)),
entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)),
entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)),
entry("create_socket_with_proxy", forPlugins(NetworkAccessCheckActions::createSocketWithProxy)),
entry("socket_bind", forPlugins(NetworkAccessCheckActions::socketBind)),
entry("socket_connect", forPlugins(NetworkAccessCheckActions::socketConnect)),
entry("server_socket_bind", forPlugins(NetworkAccessCheckActions::serverSocketBind)),
entry("server_socket_accept", forPlugins(NetworkAccessCheckActions::serverSocketAccept)),
entry("url_open_connection_proxy", forPlugins(NetworkAccessCheckActions::urlOpenConnectionWithProxy)),
entry("http_client_send", forPlugins(VersionSpecificNetworkChecks::httpClientSend)),
entry("http_client_send_async", forPlugins(VersionSpecificNetworkChecks::httpClientSendAsync)),
entry("create_ldap_cert_store", forPlugins(NetworkAccessCheckActions::createLDAPCertStore)),
entry("server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::serverSocketChannelBind)),
entry("server_socket_channel_bind_backlog", forPlugins(NetworkAccessCheckActions::serverSocketChannelBindWithBacklog)),
entry("server_socket_channel_accept", forPlugins(NetworkAccessCheckActions::serverSocketChannelAccept)),
entry("asynchronous_server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBind)),
entry(
"asynchronous_server_socket_channel_bind_backlog",
forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBindWithBacklog)
),
entry(
"asynchronous_server_socket_channel_accept",
forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAccept)
),
entry(
"asynchronous_server_socket_channel_accept_with_handler",
forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAcceptWithHandler)
),
entry("socket_channel_bind", forPlugins(NetworkAccessCheckActions::socketChannelBind)),
entry("socket_channel_connect", forPlugins(NetworkAccessCheckActions::socketChannelConnect)),
entry("asynchronous_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelBind)),
entry("asynchronous_socket_channel_connect", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnect)),
entry(
"asynchronous_socket_channel_connect_with_completion",
forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnectWithCompletion)
),
entry("datagram_channel_bind", forPlugins(NetworkAccessCheckActions::datagramChannelBind)),
entry("datagram_channel_connect", forPlugins(NetworkAccessCheckActions::datagramChannelConnect)),
entry("datagram_channel_send", forPlugins(NetworkAccessCheckActions::datagramChannelSend)),
entry("datagram_channel_receive", forPlugins(NetworkAccessCheckActions::datagramChannelReceive)),
entry("runtime_load", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoad)),
entry("runtime_load_library", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoadLibrary)),
entry("system_load", forPlugins(LoadNativeLibrariesCheckActions::systemLoad)),
entry("system_load_library", forPlugins(LoadNativeLibrariesCheckActions::systemLoadLibrary)),
entry("enable_native_access", new CheckAction(VersionSpecificNativeChecks::enableNativeAccess, false, 22)),
entry("address_target_layout", new CheckAction(VersionSpecificNativeChecks::addressLayoutWithTargetLayout, false, 22)),
entry("donwncall_handle", new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandle, false, 22)),
entry(
"donwncall_handle_with_address",
new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandleWithAddress, false, 22)
),
entry("upcall_stub", new CheckAction(VersionSpecificNativeChecks::linkerUpcallStub, false, 22)),
entry("reinterpret", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpret, false, 22)),
entry("reinterpret_cleanup", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithCleanup, false, 22)),
entry(
"reinterpret_size_cleanup",
new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithSizeAndCleanup, false, 22)
),
entry("symbol_lookup_name", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithName, false, 22)),
entry("symbol_lookup_path", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithPath, false, 22))
),
entry("localeServiceProvider", alwaysDenied(RestEntitlementsCheckAction::localeServiceProvider$)),
entry("breakIteratorProvider", alwaysDenied(RestEntitlementsCheckAction::breakIteratorProvider$)),
entry("collatorProvider", alwaysDenied(RestEntitlementsCheckAction::collatorProvider$)),
entry("dateFormatProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatProvider$)),
entry("dateFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatSymbolsProvider$)),
entry("decimalFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::decimalFormatSymbolsProvider$)),
entry("numberFormatProvider", alwaysDenied(RestEntitlementsCheckAction::numberFormatProvider$)),
entry("calendarDataProvider", alwaysDenied(RestEntitlementsCheckAction::calendarDataProvider$)),
entry("calendarNameProvider", alwaysDenied(RestEntitlementsCheckAction::calendarNameProvider$)),
entry("currencyNameProvider", alwaysDenied(RestEntitlementsCheckAction::currencyNameProvider$)),
entry("localeNameProvider", alwaysDenied(RestEntitlementsCheckAction::localeNameProvider$)),
entry("timeZoneNameProvider", alwaysDenied(RestEntitlementsCheckAction::timeZoneNameProvider$)),
entry("logManager", alwaysDenied(RestEntitlementsCheckAction::logManager$)),
entry("locale_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultLocale)),
entry("locale_setDefaultForCategory", alwaysDenied(WritePropertiesCheckActions::setDefaultLocaleForCategory)),
entry("timeZone_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultTimeZone)),
entry("system_setProperty", forPlugins(WritePropertiesCheckActions::setSystemProperty)),
entry("system_clearProperty", forPlugins(WritePropertiesCheckActions::clearSystemProperty)),
entry("system_setSystemProperties", alwaysDenied(WritePropertiesCheckActions::setSystemProperties)),
// This group is a bit nasty: if entitlements don't prevent these, then networking is
// irreparably borked for the remainder of the test run.
entry(
"datagramSocket_setDatagramSocketImplFactory",
alwaysDenied(RestEntitlementsCheckAction::datagramSocket$$setDatagramSocketImplFactory)
),
entry("httpURLConnection_setFollowRedirects", alwaysDenied(RestEntitlementsCheckAction::httpURLConnection$$setFollowRedirects)),
entry("serverSocket_setSocketFactory", alwaysDenied(RestEntitlementsCheckAction::serverSocket$$setSocketFactory)),
entry("socket_setSocketImplFactory", alwaysDenied(RestEntitlementsCheckAction::socket$$setSocketImplFactory)),
entry("url_setURLStreamHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::url$$setURLStreamHandlerFactory)),
entry("urlConnection_setFileNameMap", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setFileNameMap)),
entry("urlConnection_setContentHandlerFactory", alwaysDenied(RestEntitlementsCheckAction::urlConnection$$setContentHandlerFactory)),
entry("proxySelector_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultProxySelector)),
entry("responseCache_setDefault", alwaysDenied(RestEntitlementsCheckAction::setDefaultResponseCache)),
entry(
"createInetAddressResolverProvider",
new CheckAction(VersionSpecificNetworkChecks::createInetAddressResolverProvider, true, 18)
),
entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)),
entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)),
entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)),
entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)),
entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)),
entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)),
entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)),
entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)),
entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)),
entry("create_socket_with_proxy", forPlugins(NetworkAccessCheckActions::createSocketWithProxy)),
entry("socket_bind", forPlugins(NetworkAccessCheckActions::socketBind)),
entry("socket_connect", forPlugins(NetworkAccessCheckActions::socketConnect)),
entry("server_socket_bind", forPlugins(NetworkAccessCheckActions::serverSocketBind)),
entry("server_socket_accept", forPlugins(NetworkAccessCheckActions::serverSocketAccept)),
entry("url_open_connection_proxy", forPlugins(NetworkAccessCheckActions::urlOpenConnectionWithProxy)),
entry("http_client_send", forPlugins(VersionSpecificNetworkChecks::httpClientSend)),
entry("http_client_send_async", forPlugins(VersionSpecificNetworkChecks::httpClientSendAsync)),
entry("create_ldap_cert_store", forPlugins(NetworkAccessCheckActions::createLDAPCertStore)),
entry("server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::serverSocketChannelBind)),
entry("server_socket_channel_bind_backlog", forPlugins(NetworkAccessCheckActions::serverSocketChannelBindWithBacklog)),
entry("server_socket_channel_accept", forPlugins(NetworkAccessCheckActions::serverSocketChannelAccept)),
entry("asynchronous_server_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBind)),
entry(
"asynchronous_server_socket_channel_bind_backlog",
forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelBindWithBacklog)
),
entry("asynchronous_server_socket_channel_accept", forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAccept)),
entry(
"asynchronous_server_socket_channel_accept_with_handler",
forPlugins(NetworkAccessCheckActions::asynchronousServerSocketChannelAcceptWithHandler)
),
entry("socket_channel_bind", forPlugins(NetworkAccessCheckActions::socketChannelBind)),
entry("socket_channel_connect", forPlugins(NetworkAccessCheckActions::socketChannelConnect)),
entry("asynchronous_socket_channel_bind", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelBind)),
entry("asynchronous_socket_channel_connect", forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnect)),
entry(
"asynchronous_socket_channel_connect_with_completion",
forPlugins(NetworkAccessCheckActions::asynchronousSocketChannelConnectWithCompletion)
),
entry("datagram_channel_bind", forPlugins(NetworkAccessCheckActions::datagramChannelBind)),
entry("datagram_channel_connect", forPlugins(NetworkAccessCheckActions::datagramChannelConnect)),
entry("datagram_channel_send", forPlugins(NetworkAccessCheckActions::datagramChannelSend)),
entry("datagram_channel_receive", forPlugins(NetworkAccessCheckActions::datagramChannelReceive)),
entry("runtime_load", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoad)),
entry("runtime_load_library", forPlugins(LoadNativeLibrariesCheckActions::runtimeLoadLibrary)),
entry("system_load", forPlugins(LoadNativeLibrariesCheckActions::systemLoad)),
entry("system_load_library", forPlugins(LoadNativeLibrariesCheckActions::systemLoadLibrary)),
entry("enable_native_access", new CheckAction(VersionSpecificNativeChecks::enableNativeAccess, false, 22)),
entry("address_target_layout", new CheckAction(VersionSpecificNativeChecks::addressLayoutWithTargetLayout, false, 22)),
entry("donwncall_handle", new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandle, false, 22)),
entry("donwncall_handle_with_address", new CheckAction(VersionSpecificNativeChecks::linkerDowncallHandleWithAddress, false, 22)),
entry("upcall_stub", new CheckAction(VersionSpecificNativeChecks::linkerUpcallStub, false, 22)),
entry("reinterpret", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpret, false, 22)),
entry("reinterpret_cleanup", new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithCleanup, false, 22)),
entry(
"reinterpret_size_cleanup",
new CheckAction(VersionSpecificNativeChecks::memorySegmentReinterpretWithSizeAndCleanup, false, 22)
),
entry("symbol_lookup_name", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithName, false, 22)),
entry("symbol_lookup_path", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithPath, false, 22)),
entry("create_scanner", forPlugins(FileCheckActions::createScannerFile)),
entry("create_scanner_with_charset", forPlugins(FileCheckActions::createScannerFileWithCharset)),
entry("create_scanner_with_charset_name", forPlugins(FileCheckActions::createScannerFileWithCharsetName)),
entry("create_file_output_stream_string", forPlugins(FileCheckActions::createFileOutputStreamString)),
entry("create_file_output_stream_string_with_append", forPlugins(FileCheckActions::createFileOutputStreamStringWithAppend)),
entry("create_file_output_stream_file", forPlugins(FileCheckActions::createFileOutputStreamFile)),
entry("create_file_output_stream_file_with_append", forPlugins(FileCheckActions::createFileOutputStreamFileWithAppend)),
entry("files_probe_content_type", forPlugins(FileCheckActions::filesProbeContentType)),
entry("files_set_owner", forPlugins(FileCheckActions::filesSetOwner))
getTestEntries(FileCheckActions.class)
)
.filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion())
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
.collect(Collectors.toUnmodifiableMap(Entry::getKey, Entry::getValue));
@SuppressForbidden(reason = "Need package private methods so we don't have to make them all public")
private static Method[] getDeclaredMethods(Class<?> clazz) {
return clazz.getDeclaredMethods();
}
private static Stream<Entry<String, CheckAction>> getTestEntries(Class<?> actionsClass) {
List<Entry<String, CheckAction>> entries = new ArrayList<>();
for (var method : getDeclaredMethods(actionsClass)) {
var testAnnotation = method.getAnnotation(EntitlementTest.class);
if (testAnnotation == null) {
continue;
}
if (Modifier.isStatic(method.getModifiers()) == false) {
throw new AssertionError("Entitlement test method [" + method + "] must be static");
}
if (method.getParameterTypes().length != 0) {
throw new AssertionError("Entitlement test method [" + method + "] must not have parameters");
}
CheckedRunnable<Exception> runnable = () -> {
try {
method.invoke(null);
} catch (IllegalAccessException e) {
throw new AssertionError(e);
} catch (InvocationTargetException e) {
if (e.getCause() instanceof Exception exc) {
throw exc;
} else {
throw new AssertionError(e);
}
}
};
boolean deniedToPlugins = testAnnotation.expectedAccess() == PLUGINS;
Integer fromJavaVersion = testAnnotation.fromJavaVersion() == -1 ? null : testAnnotation.fromJavaVersion();
entries.add(entry(method.getName(), new CheckAction(runnable, deniedToPlugins, fromJavaVersion)));
}
return entries.stream();
}
private static void createURLStreamHandlerProvider() {
var x = new URLStreamHandlerProvider() {
@ -290,6 +341,11 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
System.exit(123);
}
private static void staticMethodNeverEntitledViaReflection() throws Exception {
Method systemExit = System.class.getMethod("exit", int.class);
systemExit.invoke(null, 123);
}
private static void createClassLoader() throws IOException {
try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
logger.info("Created URLClassLoader [{}]", classLoader.getName());
@ -300,6 +356,11 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
new ProcessBuilder("").start();
}
private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
Method processBuilderStart = ProcessBuilder.class.getMethod("start");
processBuilderStart.invoke(new ProcessBuilder(""));
}
private static void processBuilder_startPipeline() throws IOException {
ProcessBuilder.startPipeline(List.of());
}
@ -338,6 +399,10 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
new DummyLocaleServiceProvider();
}
private static void constructorNeverEntitledViaReflection() throws Exception {
DummyLocaleServiceProvider.class.getConstructor().newInstance();
}
private static void breakIteratorProvider$() {
new DummyBreakIteratorProvider();
}
@ -470,7 +535,7 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
return checkActions.entrySet()
.stream()
.filter(kv -> kv.getValue().isAlwaysDeniedToPlugins() == false)
.map(Map.Entry::getKey)
.map(Entry::getKey)
.collect(Collectors.toSet());
}

71
libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
View file

@ -14,6 +14,8 @@ import com.sun.tools.attach.AgentLoadException;
import com.sun.tools.attach.AttachNotSupportedException;
import com.sun.tools.attach.VirtualMachine;
import org.elasticsearch.core.CheckedConsumer;
import org.elasticsearch.core.CheckedSupplier;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
@ -22,8 +24,10 @@ import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.util.Map;
import java.util.function.Function;
@ -31,10 +35,22 @@ import static java.util.Objects.requireNonNull;
public class EntitlementBootstrap {
public record BootstrapArgs(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
public record BootstrapArgs(
Map<String, Policy> pluginPolicies,
Function<Class<?>, String> pluginResolver,
Path[] dataDirs,
Path configDir,
Path tempDir
) {
public BootstrapArgs {
requireNonNull(pluginPolicies);
requireNonNull(pluginResolver);
requireNonNull(dataDirs);
if (dataDirs.length == 0) {
throw new IllegalArgumentException("must provide at least one data directory");
}
requireNonNull(configDir);
requireNonNull(tempDir);
}
}
@ -50,13 +66,22 @@ public class EntitlementBootstrap {
*
* @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
* @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
* @param dataDirs data directories for Elasticsearch
* @param configDir the config directory for Elasticsearch
* @param tempDir the temp directory for Elasticsearch
*/
public static void bootstrap(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
public static void bootstrap(
Map<String, Policy> pluginPolicies,
Function<Class<?>, String> pluginResolver,
Path[] dataDirs,
Path configDir,
Path tempDir
) {
logger.debug("Loading entitlement agent");
if (EntitlementBootstrap.bootstrapArgs != null) {
throw new IllegalStateException("plugin data is already set");
}
EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver);
EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir);
exportInitializationToAgent();
loadAgent(findAgentJar());
selfTest();
@ -123,30 +148,31 @@ public class EntitlementBootstrap {
* @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing
*/
private static void selfTest() {
ensureCannotStartProcess();
ensureCanCreateTempFile();
ensureCannotStartProcess(ProcessBuilder::start);
ensureCanCreateTempFile(EntitlementBootstrap::createTempFile);
// Try again with reflection
ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess);
ensureCanCreateTempFile(EntitlementBootstrap::reflectiveCreateTempFile);
}
private static void ensureCannotStartProcess() {
private static void ensureCannotStartProcess(CheckedConsumer<ProcessBuilder, ?> startProcess) {
try {
// The command doesn't matter; it doesn't even need to exist
new ProcessBuilder("").start();
startProcess.accept(new ProcessBuilder(""));
} catch (NotEntitledException e) {
logger.debug("Success: Entitlement protection correctly prevented process creation");
return;
} catch (IOException e) {
} catch (Exception e) {
throw new IllegalStateException("Failed entitlement protection self-test", e);
}
throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted");
}
/**
* Originally {@code Security.selfTest}.
*/
@SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
private static void ensureCanCreateTempFile() {
private static void ensureCanCreateTempFile(CheckedSupplier<Path, ?> createTempFile) {
try {
Path p = Files.createTempFile(null, null);
Path p = createTempFile.get();
p.toFile().deleteOnExit();
// Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally.
@ -163,5 +189,24 @@ public class EntitlementBootstrap {
logger.debug("Success: Entitlement protection correctly permitted temp file creation");
}
@SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
private static Path createTempFile() throws Exception {
return Files.createTempFile(null, null);
}
private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception {
try {
var start = ProcessBuilder.class.getMethod("start");
start.invoke(pb);
} catch (InvocationTargetException e) {
throw (Exception) e.getCause();
}
}
private static Path reflectiveCreateTempFile() throws Exception {
return (Path) Files.class.getMethod("createTempFile", String.class, String.class, FileAttribute[].class)
.invoke(null, null, null, new FileAttribute<?>[0]);
}
private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class);
}

12
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java
View file

@ -9,10 +9,8 @@
package org.elasticsearch.entitlement.runtime.policy;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@ -51,20 +49,10 @@ public final class FileAccessTree {
return checkPath(normalize(path), readPaths);
}
@SuppressForbidden(reason = "Explicitly checking File apis")
boolean canRead(File file) {
return checkPath(normalize(file.toPath()), readPaths);
}
boolean canWrite(Path path) {
return checkPath(normalize(path), writePaths);
}
@SuppressForbidden(reason = "Explicitly checking File apis")
boolean canWrite(File file) {
return checkPath(normalize(file.toPath()), writePaths);
}
private static String normalize(Path path) {
return path.toAbsolutePath().normalize().toString();
}

59
libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
View file

@ -169,23 +169,7 @@ public class PolicyManager {
}
public void checkStartProcess(Class<?> callerClass) {
neverEntitled(callerClass, "start process");
}
private void neverEntitled(Class<?> callerClass, String operationDescription) {
var requestingClass = requestingClass(callerClass);
if (isTriviallyAllowed(requestingClass)) {
return;
}
throw new NotEntitledException(
Strings.format(
"Not entitled: caller [%s], module [%s], operation [%s]",
callerClass,
requestingClass.getModule() == null ? "<none>" : requestingClass.getModule().getName(),
operationDescription
)
);
neverEntitled(callerClass, () -> "start process");
}
/**
@ -241,31 +225,9 @@ public class PolicyManager {
checkChangeJVMGlobalState(callerClass);
}
/**
* Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
*/
public void checkReadSensitiveNetworkInformation(Class<?> callerClass) {
neverEntitled(callerClass, "access sensitive network information");
}
@SuppressForbidden(reason = "Explicitly checking File apis")
public void checkFileRead(Class<?> callerClass, File file) {
var requestingClass = requestingClass(callerClass);
if (isTriviallyAllowed(requestingClass)) {
return;
}
ModuleEntitlements entitlements = getEntitlements(requestingClass);
if (entitlements.fileAccess().canRead(file) == false) {
throw new NotEntitledException(
Strings.format(
"Not entitled: caller [%s], module [%s], entitlement [file], operation [read], path [%s]",
callerClass,
requestingClass.getModule(),
file
)
);
}
checkFileRead(callerClass, file.toPath());
}
public void checkFileRead(Class<?> callerClass, Path path) {
@ -289,22 +251,7 @@ public class PolicyManager {
@SuppressForbidden(reason = "Explicitly checking File apis")
public void checkFileWrite(Class<?> callerClass, File file) {
var requestingClass = requestingClass(callerClass);
if (isTriviallyAllowed(requestingClass)) {
return;
}
ModuleEntitlements entitlements = getEntitlements(requestingClass);
if (entitlements.fileAccess().canWrite(file) == false) {
throw new NotEntitledException(
Strings.format(
"Not entitled: caller [%s], module [%s], entitlement [file], operation [write], path [%s]",
callerClass,
requestingClass.getModule(),
file
)
);
}
checkFileWrite(callerClass, file.toPath());
}
public void checkFileWrite(Class<?> callerClass, Path path) {

8
libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
View file

@ -238,7 +238,6 @@ public class PolicyManagerTests extends ESTestCase {
}
public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException {
var agentsClass = new TestAgent();
var entitlementsClass = makeClassInItsOwnModule(); // A class in the entitlements library itself
var requestingClass = makeClassInItsOwnModule(); // This guy is always the right answer
var instrumentedClass = makeClassInItsOwnModule(); // The class that called the check method
@ -365,13 +364,6 @@ public class PolicyManagerTests extends ESTestCase {
return layer.findLoader("org.example.plugin").loadClass("q.B");
}
private static Class<?> makeClassInItsOwnUnnamedModule() throws IOException, ClassNotFoundException {
final Path home = createTempDir();
Path jar = createMockPluginJar(home);
var layer = createLayerForJar(jar, "org.example.plugin");
return layer.findLoader("org.example.plugin").loadClass("q.B");
}
private static PolicyManager policyManager(String agentsPackageName, Module entitlementsModule) {
return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", agentsPackageName, entitlementsModule);
}

88
muted-tests.yml
View file

@ -143,9 +143,6 @@ tests:
- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
issue: https://github.com/elastic/elasticsearch/issues/118217
- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
method: testBottomFieldSort
issue: https://github.com/elastic/elasticsearch/issues/118214
- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
method: testSearcherId
issue: https://github.com/elastic/elasticsearch/issues/118374
@ -231,9 +228,6 @@ tests:
- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
method: test {p0=data_stream/140_data_stream_aliases/Create data stream aliases using wildcard expression}
issue: https://github.com/elastic/elasticsearch/issues/120890
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
method: test {p0=ml/inference_crud/*}
issue: https://github.com/elastic/elasticsearch/issues/120816
- class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
method: testAuthenticateShouldNotFallThroughInCaseOfFailure
issue: https://github.com/elastic/elasticsearch/issues/120902
@ -330,6 +324,10 @@ tests:
method: testCrossClusterAsyncQueryStop
issue: https://github.com/elastic/elasticsearch/issues/121249
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
method: test {p0=transform/*}
issue: https://github.com/elastic/elasticsearch/issues/120816
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
method: test {p0=ml/*}
issue: https://github.com/elastic/elasticsearch/issues/120816
- class: org.elasticsearch.upgrades.VectorSearchIT
method: testBBQVectorSearch {upgradedNodes=0}
@ -359,14 +357,6 @@ tests:
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
issue: https://github.com/elastic/elasticsearch/issues/121290
- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
method: testRerouting_HandlesTransportException_FromOtherNode
issue: https://github.com/elastic/elasticsearch/issues/121292
- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
method: testRerouting_ToOtherNode
issue: https://github.com/elastic/elasticsearch/issues/121293
- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
issue: https://github.com/elastic/elasticsearch/issues/121294
- class: org.elasticsearch.env.NodeEnvironmentTests
method: testGetBestDowngradeVersion
issue: https://github.com/elastic/elasticsearch/issues/121316
@ -391,6 +381,76 @@ tests:
- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
method: test {yaml=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
issue: https://github.com/elastic/elasticsearch/issues/121350
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
method: test {p0=search.vectors/42_knn_search_int4_flat/KNN Vector similarity search only}
issue: https://github.com/elastic/elasticsearch/issues/121395
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
issue: https://github.com/elastic/elasticsearch/issues/121407
- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
method: testDependentVariableIsAliasToNested
issue: https://github.com/elastic/elasticsearch/issues/121415
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
method: testLookupExplosionBigStringManyMatches
issue: https://github.com/elastic/elasticsearch/issues/121465
- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
method: testClientSecretRotation
issue: https://github.com/elastic/elasticsearch/issues/120985
- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
method: testGrantApiKeyForJWT
issue: https://github.com/elastic/elasticsearch/issues/121039
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=cluster.health/10_basic/cluster health basic test}
issue: https://github.com/elastic/elasticsearch/issues/121478
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
method: testLookupExplosionManyMatches
issue: https://github.com/elastic/elasticsearch/issues/121481
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
method: testGetUsersWithProfileUid
issue: https://github.com/elastic/elasticsearch/issues/121483
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=cat.aliases/10_basic/Empty cluster}
issue: https://github.com/elastic/elasticsearch/issues/121484
- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
method: testTransformLifecycle_RangeQueryThatMatchesNoShards
issue: https://github.com/elastic/elasticsearch/issues/121480
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
method: testStopQueryLocal
issue: https://github.com/elastic/elasticsearch/issues/121487
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
method: testSuccessfulPathways
issue: https://github.com/elastic/elasticsearch/issues/121488
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
method: testAsyncQueriesWithLimit0
issue: https://github.com/elastic/elasticsearch/issues/121489
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
method: testSuggestProfilesWithHint
issue: https://github.com/elastic/elasticsearch/issues/121116
- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT
method: test {docs.testFilterToday}
issue: https://github.com/elastic/elasticsearch/issues/121474
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
method: testSuggestProfileWithData
issue: https://github.com/elastic/elasticsearch/issues/121258
- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT
method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED}
issue: https://github.com/elastic/elasticsearch/issues/121115
- class: org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStepTests
method: testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs
issue: https://github.com/elastic/elasticsearch/issues/121495
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias}
issue: https://github.com/elastic/elasticsearch/issues/121128
- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
issue: https://github.com/elastic/elasticsearch/issues/121412
- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
issue: https://github.com/elastic/elasticsearch/issues/121294
- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
method: testDependentVariableIsAliasToKeyword
issue: https://github.com/elastic/elasticsearch/issues/121492
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=cat.aliases/10_basic/Complex alias}
issue: https://github.com/elastic/elasticsearch/issues/121513
# Examples:
#

79
rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_cluster.json
View file

@ -1,55 +1,56 @@
{
"indices.resolve_cluster":{
"documentation":{
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-cluster-api.html",
"description":"Resolves the specified index expressions to return information about each cluster, including the local cluster, if included."
"indices.resolve_cluster": {
"documentation": {
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-resolve-cluster-api.html",
"description": "Resolves the specified index expressions to return information about each cluster. If no index expression is provided, this endpoint will return information about all the remote clusters that are configured on the local cluster."
},
"stability":"stable",
"visibility":"public",
"headers":{
"accept": [ "application/json"]
"stability": "stable",
"visibility": "public",
"headers": {
"accept": ["application/json"]
},
"url":{
"paths":[
"url": {
"paths": [
{
"path":"/_resolve/cluster/{name}",
"methods":[
"GET"
],
"parts":{
"name":{
"type":"list",
"description":"A comma-separated list of cluster:index names or wildcard expressions"
"path": "/_resolve/cluster",
"methods": ["GET"]
},
{
"path": "/_resolve/cluster/{name}",
"methods": ["GET"],
"parts": {
"name": {
"type": "list",
"description": "A comma-separated list of cluster:index names or wildcard expressions"
}
}
}
]
},
"params":{
"ignore_unavailable":{
"type":"boolean",
"description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)"
"params": {
"ignore_unavailable": {
"type": "boolean",
"description": "Whether specified concrete indices should be ignored when unavailable (missing or closed). Only allowed when providing an index expression."
},
"ignore_throttled":{
"type":"boolean",
"description":"Whether specified concrete, expanded or aliased indices should be ignored when throttled"
"ignore_throttled": {
"type": "boolean",
"description": "Whether specified concrete, expanded or aliased indices should be ignored when throttled. Only allowed when providing an index expression."
},
"allow_no_indices":{
"type":"boolean",
"description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
"allow_no_indices": {
"type": "boolean",
"description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified). Only allowed when providing an index expression."
},
"expand_wildcards":{
"type":"enum",
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],
"default":"open",
"description":"Whether wildcard expressions should get expanded to open or closed indices (default: open)"
"expand_wildcards": {
"type": "enum",
"options": ["open", "closed", "hidden", "none", "all"],
"default": "open",
"description": "Whether wildcard expressions should get expanded to open or closed indices (default: open). Only allowed when providing an index expression."
},
"timeout": {
"type": "time",
"description": "The maximum time to wait for remote clusters to respond"
}
}
}
}

35
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/10_basic_resolve_cluster.yml
View file

@ -39,7 +39,7 @@ setup:
- do:
indices.resolve_cluster:
name: '*'
expand_wildcards: [closed]
expand_wildcards: closed
- match: {(local).connected: true}
- match: {(local).skip_unavailable: false}
@ -65,7 +65,7 @@ setup:
- do:
indices.resolve_cluster:
name: 'index2*'
expand_wildcards: [open,closed]
expand_wildcards: open,closed
- match: {(local).connected: true}
- match: {(local).skip_unavailable: false}
@ -75,7 +75,7 @@ setup:
- do:
indices.resolve_cluster:
name: 'index2*'
expand_wildcards: [closed]
expand_wildcards: closed
- match: {(local).connected: true}
- match: {(local).skip_unavailable: false}
@ -115,7 +115,7 @@ setup:
- do:
indices.resolve_cluster:
name: 'my_alias2,doesnotexist*'
expand_wildcards: [all]
expand_wildcards: all
- match: {(local).connected: true}
- match: {(local).skip_unavailable: false}
@ -141,10 +141,10 @@ setup:
- do:
indices.resolve_cluster:
name: '*'
expand_wildcards: [all]
ignore_unavailable: [true]
ignore_throttled: [true]
allow_no_indices: [true]
expand_wildcards: all
ignore_unavailable: true
ignore_throttled: true
allow_no_indices: true
allowed_warnings:
- "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
@ -157,10 +157,10 @@ setup:
- do:
indices.resolve_cluster:
name: '*'
expand_wildcards: [open]
ignore_unavailable: [false]
ignore_throttled: [false]
allow_no_indices: [false]
expand_wildcards: open
ignore_unavailable: false
ignore_throttled: false
allow_no_indices: false
allowed_warnings:
- "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
@ -170,3 +170,14 @@ setup:
- is_false: (local).error # should not be present
- exists: (local).version.number
---
"Resolve cluster with no index expression":
- requires:
cluster_features: ["gte_v8.18.0"]
reason: "resolve cluster with no index expression introduced in 8.18"
- do:
indices.resolve_cluster:
timeout: 400s
- is_false: (local).error # should not be present - body should be empty since no remotes configured

72
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/30_context.yml
View file

@ -395,3 +395,75 @@ setup:
field: suggest_multi_contexts
contexts:
location: []
---
"Duplicate suggestions in different contexts":
- requires:
cluster_features: [ "search.completion_field.duplicate.support" ]
reason: "Support for duplicate suggestions in different contexts"
- do:
index:
refresh: true
index: test
id: "1"
body:
suggest_context:
-
input: "foox"
weight: 2
contexts:
color: ["red", "yellow"]
-
input: "foox"
weight: 3
contexts:
color: ["blue", "green", "yellow"]
- do:
search:
body:
suggest:
result:
text: "foo"
completion:
field: suggest_context
contexts:
color: "red"
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
- match: { suggest.result.0.options.0.text: "foox" }
- match: { suggest.result.0.options.0._score: 2 }
- do:
search:
body:
suggest:
result:
text: "foo"
completion:
field: suggest_context
contexts:
color: "yellow"
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
- match: { suggest.result.0.options.0.text: "foox" }
# the highest weight wins
- match: { suggest.result.0.options.0._score: 3 }
- do:
search:
body:
suggest:
result:
text: "foo"
completion:
field: suggest_context
contexts:
color: "blue"
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
- match: { suggest.result.0.options.0.text: "foox" }
- match: { suggest.result.0.options.0._score: 3 }

77
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml
View file

@ -268,3 +268,80 @@
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
---
"Duplicate suggestions in different contexts in sub-fields":
- requires:
cluster_features: [ "search.completion_field.duplicate.support" ]
reason: "Support for duplicate suggestions in different contexts"
- do:
indices.create:
index: completion_with_context
body:
mappings:
"properties":
"suggest_1":
"type": "completion"
"contexts":
-
"name": "color"
"type": "category"
"fields":
"suggest_2":
"type": "completion"
"contexts":
-
"name": "color"
"type": "category"
- do:
index:
refresh: true
index: completion_with_context
id: "1"
body:
suggest_1:
-
input: "foox"
weight: 2
contexts:
color: ["red"]
-
input: "foox"
weight: 3
contexts:
color: ["blue", "green"]
- do:
search:
body:
suggest:
result:
text: "foo"
completion:
field: suggest_1.suggest_2
contexts:
color: "red"
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
- match: { suggest.result.0.options.0.text: "foox" }
- match: { suggest.result.0.options.0._score: 2 }
- do:
search:
body:
suggest:
result:
text: "foo"
completion:
field: suggest_1.suggest_2
contexts:
color: "blue"
- length: { suggest.result: 1 }
- length: { suggest.result.0.options: 1 }
- match: { suggest.result.0.options.0.text: "foox" }
- match: { suggest.result.0.options.0._score: 3 }

12
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml
View file

@ -14,6 +14,12 @@ setup:
- synonyms: "bye => goodbye"
id: "synonym-rule-2"
# This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
- do:
cluster.health:
index: .synonyms
wait_for_status: green
# Create synonyms synonyms_set2
- do:
synonyms.put_synonym:
@ -25,12 +31,6 @@ setup:
- synonyms: "bye => goodbye"
id: "synonym-rule-2"
# This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
- do:
cluster.health:
index: .synonyms
wait_for_status: green
# Create my_index1 with synonym_filter that uses synonyms_set1
- do:
indices.create:

55
server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingIT.java Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.action.admin.indices.mapping.put;
import org.apache.logging.log4j.Level;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.MockLog;
import org.elasticsearch.test.junit.annotations.TestLogging;
import static org.hamcrest.Matchers.equalTo;
public class PutMappingIT extends ESSingleNodeTestCase {
@TestLogging(
reason = "testing DEBUG logging",
value = "org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction:DEBUG"
)
public void testFailureLogging() {
final var indexName = randomIdentifier();
createIndex(indexName);
final var fieldName = randomIdentifier();
safeGet(client().execute(TransportPutMappingAction.TYPE, new PutMappingRequest(indexName).source(fieldName, "type=keyword")));
MockLog.assertThatLogger(
() -> assertThat(
asInstanceOf(
IllegalArgumentException.class,
safeAwaitFailure(
AcknowledgedResponse.class,
l -> client().execute(
TransportPutMappingAction.TYPE,
new PutMappingRequest(indexName).source(fieldName, "type=long"),
l
)
)
).getMessage(),
equalTo("mapper [" + fieldName + "] cannot be changed from type [keyword] to [long]")
),
TransportPutMappingAction.class,
new MockLog.SeenEventExpectation(
"failure message",
TransportPutMappingAction.class.getCanonicalName(),
Level.DEBUG,
"failed to put mappings on indices [[" + indexName
)
);
}
}

1
server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java
View file

@ -238,7 +238,6 @@ public class SearchCancellationIT extends AbstractSearchCancellationTestCase {
}
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99929")
public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception {
// Have at least two nodes so that we have parallel execution of two request guaranteed even if max concurrent requests per node
// are limited to 1

8
server/src/main/java/org/elasticsearch/TransportVersions.java
View file

@ -172,6 +172,8 @@ public class TransportVersions {
public static final TransportVersion LINEAR_RETRIEVER_SUPPORT = def(8_837_00_0);
public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_00_0);
public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_00_0);
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_00_0);
public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_00_0);
/*
* WARNING: DO NOT MERGE INTO MAIN!
@ -241,15 +243,13 @@ public class TransportVersions {
* Reference to the earliest compatible transport version to this version of the codebase.
* This should be the transport version used by the highest minor version of the previous major.
*/
@UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
// This needs to be bumped to the 8.last
public static final TransportVersion MINIMUM_COMPATIBLE = V_7_17_0;
public static final TransportVersion MINIMUM_COMPATIBLE = BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1;
/**
* Reference to the minimum transport version that can be used with CCS.
* This should be the transport version used by the previous minor release.
*/
public static final TransportVersion MINIMUM_CCS_VERSION = V_8_15_0;
public static final TransportVersion MINIMUM_CCS_VERSION = BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1;
/**
* Sorted list of all versions defined in this class

35
server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
View file

@ -39,7 +39,6 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -139,7 +138,7 @@ public class TransportPutMappingAction extends AcknowledgedTransportMasterNodeAc
performMappingUpdate(concreteIndices, request, listener, metadataMappingService, false);
} catch (IndexNotFoundException ex) {
logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(request.indices() + "]"), ex);
logger.debug(() -> "failed to put mappings on indices " + Arrays.toString(request.indices()), ex);
throw ex;
}
}
@ -179,25 +178,21 @@ public class TransportPutMappingAction extends AcknowledgedTransportMasterNodeAc
MetadataMappingService metadataMappingService,
boolean autoUpdate
) {
final ActionListener<AcknowledgedResponse> wrappedListener = listener.delegateResponse((l, e) -> {
logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(concreteIndices) + "]", e);
ActionListener.run(listener.delegateResponse((l, e) -> {
logger.debug(() -> "failed to put mappings on indices " + Arrays.toString(concreteIndices), e);
l.onFailure(e);
});
final PutMappingClusterStateUpdateRequest updateRequest;
try {
updateRequest = new PutMappingClusterStateUpdateRequest(
request.masterNodeTimeout(),
request.ackTimeout(),
request.source(),
autoUpdate,
concreteIndices
);
} catch (IOException e) {
wrappedListener.onFailure(e);
return;
}
metadataMappingService.putMapping(updateRequest, wrappedListener);
}),
wrappedListener -> metadataMappingService.putMapping(
new PutMappingClusterStateUpdateRequest(
request.masterNodeTimeout(),
request.ackTimeout(),
request.source(),
autoUpdate,
concreteIndices
),
wrappedListener
)
);
}
static String checkForFailureStoreViolations(ProjectMetadata projectMetadata, Index[] concreteIndices, PutMappingRequest request) {

2
server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java
View file

@ -76,7 +76,7 @@ public abstract class TransportBroadcastUnpromotableAction<Request extends Broad
@Override
protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
final var unpromotableShards = request.indexShardRoutingTable.unpromotableShards();
final var unpromotableShards = request.indexShardRoutingTable.assignedUnpromotableShards();
final var responses = new ArrayList<Response>(unpromotableShards.size());
try (var listeners = new RefCountingListener(listener.map(v -> combineUnpromotableShardResponses(responses)))) {

2
server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java
View file

@ -65,7 +65,7 @@ public class PostWriteRefresh {
}
});
case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> {
if (indexShard.getReplicationGroup().getRoutingTable().allUnpromotableShards().size() > 0) {
if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) {
sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout);
} else {
l.onResponse(true);

8
server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
View file

@ -242,7 +242,13 @@ class Elasticsearch {
pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, findPluginsWithNativeAccess(pluginPolicies));
var pluginsResolver = PluginsResolver.create(pluginsLoader);
EntitlementBootstrap.bootstrap(pluginPolicies, pluginsResolver::resolveClassToPluginName);
EntitlementBootstrap.bootstrap(
pluginPolicies,
pluginsResolver::resolveClassToPluginName,
nodeEnv.dataFiles(),
nodeEnv.configFile(),
nodeEnv.tmpFile()
);
} else if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
// no need to explicitly enable native access for legacy code
pluginsLoader = PluginsLoader.createPluginsLoader(modulesBundles, pluginsBundles, Map.of());

2
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java
View file

@ -56,7 +56,7 @@ public class MetadataMappingService {
public MetadataMappingService(ClusterService clusterService, IndicesService indicesService) {
this.clusterService = clusterService;
this.indicesService = indicesService;
taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor());
this.taskQueue = clusterService.createTaskQueue("put-mapping", Priority.HIGH, new PutMappingExecutor());
}
record PutMappingClusterStateUpdateTask(PutMappingClusterStateUpdateRequest request, ActionListener<AcknowledgedResponse> listener)

4
server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
View file

@ -193,7 +193,7 @@ public class IndexShardRoutingTable {
*
* @return a {@link List} of shards
*/
public List<ShardRouting> unpromotableShards() {
public List<ShardRouting> assignedUnpromotableShards() {
return this.assignedUnpromotableShards;
}
@ -202,7 +202,7 @@ public class IndexShardRoutingTable {
*
* @return a {@link List} of shards
*/
public List<ShardRouting> allUnpromotableShards() {
public List<ShardRouting> unpromotableShards() {
return this.unpromotableShards;
}

8
server/src/main/java/org/elasticsearch/common/logging/internal/LoggerFactoryImpl.java
View file

@ -22,6 +22,12 @@ public class LoggerFactoryImpl extends LoggerFactory {
@Override
public Logger getLogger(Class<?> clazz) {
return new LoggerImpl(LogManager.getLogger(clazz));
// Elasticsearch configures logging at the root level, it does not support
// programmatic configuration at the logger level. Log4j's method for
// getting a logger by Class doesn't just use the class name, but also
// scans the classloader hierarchy for programmatic configuration. Here we
// just delegate to use the String class name so that regardless of which
// classloader a class comes from, we will use the root logging config.
return getLogger(clazz.getName());
}
}

1
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View file

@ -614,6 +614,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING,
IndicesClusterStateService.SHARD_LOCK_RETRY_INTERVAL_SETTING,
IndicesClusterStateService.SHARD_LOCK_RETRY_TIMEOUT_SETTING,
IndicesClusterStateService.CONCURRENT_SHARD_CLOSE_LIMIT,
IngestSettings.GROK_WATCHDOG_INTERVAL,
IngestSettings.GROK_WATCHDOG_MAX_EXECUTION_TIME,
TDigestExecutionHint.SETTING,

94
server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
View file

@ -392,7 +392,7 @@ public class CompletionFieldMapper extends FieldMapper {
// parse
XContentParser parser = context.parser();
Token token = parser.currentToken();
Map<String, CompletionInputMetadata> inputMap = Maps.newMapWithExpectedSize(1);
Map<String, CompletionInputMetadataContainer> inputMap = Maps.newMapWithExpectedSize(1);
if (token == Token.VALUE_NULL) { // ignore null values
return;
@ -405,7 +405,7 @@ public class CompletionFieldMapper extends FieldMapper {
}
// index
for (Map.Entry<String, CompletionInputMetadata> completionInput : inputMap.entrySet()) {
for (Map.Entry<String, CompletionInputMetadataContainer> completionInput : inputMap.entrySet()) {
String input = completionInput.getKey();
if (input.trim().isEmpty()) {
context.addIgnoredField(mappedFieldType.name());
@ -420,21 +420,33 @@ public class CompletionFieldMapper extends FieldMapper {
}
input = input.substring(0, len);
}
CompletionInputMetadata metadata = completionInput.getValue();
CompletionInputMetadataContainer cmc = completionInput.getValue();
if (fieldType().hasContextMappings()) {
fieldType().getContextMappings().addField(context.doc(), fieldType().name(), input, metadata.weight, metadata.contexts);
for (CompletionInputMetadata metadata : cmc.getValues()) {
fieldType().getContextMappings().addField(context.doc(), fieldType().name(), input, metadata.weight, metadata.contexts);
}
} else {
context.doc().add(new SuggestField(fieldType().name(), input, metadata.weight));
context.doc().add(new SuggestField(fieldType().name(), input, cmc.getWeight()));
}
}
context.addToFieldNames(fieldType().name());
for (CompletionInputMetadata metadata : inputMap.values()) {
multiFields().parse(
this,
context,
() -> context.switchParser(new MultiFieldParser(metadata, fieldType().name(), context.parser().getTokenLocation()))
);
for (CompletionInputMetadataContainer cmc : inputMap.values()) {
if (fieldType().hasContextMappings()) {
for (CompletionInputMetadata metadata : cmc.getValues()) {
multiFields().parse(
this,
context,
() -> context.switchParser(new MultiFieldParser(metadata, fieldType().name(), context.parser().getTokenLocation()))
);
}
} else {
CompletionInputMetadata metadata = cmc.getValue();
multiFields().parse(
this,
context,
() -> context.switchParser(new MultiFieldParser(metadata, fieldType().name(), context.parser().getTokenLocation()))
);
}
}
}
@ -447,11 +459,13 @@ public class CompletionFieldMapper extends FieldMapper {
DocumentParserContext documentParserContext,
Token token,
XContentParser parser,
Map<String, CompletionInputMetadata> inputMap
Map<String, CompletionInputMetadataContainer> inputMap
) throws IOException {
String currentFieldName = null;
if (token == Token.VALUE_STRING) {
inputMap.put(parser.text(), new CompletionInputMetadata(parser.text(), Collections.<String, Set<String>>emptyMap(), 1));
CompletionInputMetadataContainer cmc = new CompletionInputMetadataContainer(fieldType().hasContextMappings());
cmc.add(new CompletionInputMetadata(parser.text(), Collections.emptyMap(), 1));
inputMap.put(parser.text(), cmc);
} else if (token == Token.START_OBJECT) {
Set<String> inputs = new HashSet<>();
int weight = 1;
@ -531,8 +545,14 @@ public class CompletionFieldMapper extends FieldMapper {
}
}
for (String input : inputs) {
if (inputMap.containsKey(input) == false || inputMap.get(input).weight < weight) {
inputMap.put(input, new CompletionInputMetadata(input, contextsMap, weight));
CompletionInputMetadata cm = new CompletionInputMetadata(input, contextsMap, weight);
CompletionInputMetadataContainer cmc = inputMap.get(input);
if (cmc != null) {
cmc.add(cm);
} else {
cmc = new CompletionInputMetadataContainer(fieldType().hasContextMappings());
cmc.add(cm);
inputMap.put(input, cmc);
}
}
} else {
@ -543,10 +563,46 @@ public class CompletionFieldMapper extends FieldMapper {
}
}
static class CompletionInputMetadataContainer {
private final boolean hasContexts;
private final List<CompletionInputMetadata> list;
private CompletionInputMetadata single;
CompletionInputMetadataContainer(boolean hasContexts) {
this.hasContexts = hasContexts;
this.list = hasContexts ? new ArrayList<>() : null;
}
void add(CompletionInputMetadata cm) {
if (hasContexts) {
list.add(cm);
} else {
if (single == null || single.weight < cm.weight) {
single = cm;
}
}
}
List<CompletionInputMetadata> getValues() {
assert hasContexts;
return list;
}
CompletionInputMetadata getValue() {
assert hasContexts == false;
return single;
}
int getWeight() {
assert hasContexts == false;
return single.weight;
}
}
static class CompletionInputMetadata {
public final String input;
public final Map<String, Set<String>> contexts;
public final int weight;
private final String input;
private final Map<String, Set<String>> contexts;
private final int weight;
CompletionInputMetadata(String input, Map<String, Set<String>> contexts, int weight) {
this.input = input;

3
server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
View file

@ -29,6 +29,7 @@ import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.engine.TranslogOperationAsserter;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.search.lookup.Source;
import java.io.Closeable;
import java.io.IOException;
@ -298,8 +299,10 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
+ "], with different data. "
+ "prvOp ["
+ prvOp
+ (prvOp instanceof Translog.Index index ? " source: " + Source.fromBytes(index.source()).source() : "")
+ "], newOp ["
+ newOp
+ (newOp instanceof Translog.Index index ? " source: " + Source.fromBytes(index.source()).source() : "")
+ "]",
previous.v2()
);

21
server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View file

@ -118,6 +118,18 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
Setting.Property.NodeScope
);
/**
* Maximum number of shards to try and close concurrently. Defaults to the smaller of {@code node.processors} and {@code 10}, but can be
* set to any positive integer.
*/
public static final Setting<Integer> CONCURRENT_SHARD_CLOSE_LIMIT = Setting.intSetting(
"indices.store.max_concurrent_closing_shards",
settings -> Integer.toString(Math.min(10, EsExecutors.NODE_PROCESSORS_SETTING.get(settings).roundUp())),
1,
Integer.MAX_VALUE,
Setting.Property.NodeScope
);
final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
private final ClusterService clusterService;
private final ThreadPool threadPool;
@ -1356,7 +1368,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
}
}
private static class ShardCloseExecutor implements Executor {
static class ShardCloseExecutor implements Executor {
private final ThrottledTaskRunner throttledTaskRunner;
@ -1369,8 +1381,11 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
// can't close the old ones down fast enough. Maybe we could block or throttle new shards starting while old shards are still
// shutting down, given that starting new shards is already async. Since this seems unlikely in practice, we opt for the simple
// approach here.
final var maxThreads = Math.max(EsExecutors.NODE_PROCESSORS_SETTING.get(settings).roundUp(), 10);
throttledTaskRunner = new ThrottledTaskRunner(IndicesClusterStateService.class.getCanonicalName(), maxThreads, delegate);
throttledTaskRunner = new ThrottledTaskRunner(
IndicesClusterStateService.class.getCanonicalName(),
CONCURRENT_SHARD_CLOSE_LIMIT.get(settings),
delegate
);
}
@Override

2
server/src/main/java/org/elasticsearch/node/NodeConstruction.java
View file

@ -1088,7 +1088,7 @@ class NodeConstruction {
actionModule.getReservedClusterStateService().installClusterStateHandler(new ReservedRepositoryAction(repositoriesService));
actionModule.getReservedClusterStateService().installProjectStateHandler(new ReservedPipelineAction());
FileSettingsHealthIndicatorService fileSettingsHealthIndicatorService = new FileSettingsHealthIndicatorService();
FileSettingsHealthIndicatorService fileSettingsHealthIndicatorService = new FileSettingsHealthIndicatorService(settings);
FileSettingsService fileSettingsService = pluginsService.loadSingletonServiceProvider(
FileSettingsServiceProvider.class,
() -> FileSettingsService::new

33
server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
View file

@ -22,6 +22,8 @@ import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.file.MasterNodeFileWatchingService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.health.HealthIndicatorDetails;
import org.elasticsearch.health.HealthIndicatorImpact;
@ -235,7 +237,7 @@ public class FileSettingsService extends MasterNodeFileWatchingService implement
}
@Override
protected void processInitialFilesMissing() throws ExecutionException, InterruptedException, IOException {
protected void processInitialFilesMissing() throws ExecutionException, InterruptedException {
PlainActionFuture<ActionResponse.Empty> completion = new PlainActionFuture<>();
logger().info("setting file [{}] not found, initializing [{}] as empty", watchedFile, NAMESPACE);
stateService.initEmpty(NAMESPACE, completion);
@ -259,11 +261,29 @@ public class FileSettingsService extends MasterNodeFileWatchingService implement
)
);
/**
* We want a length limit so we don't blow past the indexing limit in the case of a long description string.
* This is an {@code OperatorDynamic} setting so that if the truncation hampers troubleshooting efforts,
* the operator could override it and retry the operation without necessarily restarting the cluster.
*/
public static final String DESCRIPTION_LENGTH_LIMIT_KEY = "fileSettings.descriptionLengthLimit";
static final Setting<Integer> DESCRIPTION_LENGTH_LIMIT = Setting.intSetting(
DESCRIPTION_LENGTH_LIMIT_KEY,
100,
1, // Need room for the ellipsis
Setting.Property.OperatorDynamic
);
private final Settings settings;
private boolean isActive = false;
private long changeCount = 0;
private long failureStreak = 0;
private String mostRecentFailure = null;
public FileSettingsHealthIndicatorService(Settings settings) {
this.settings = settings;
}
public synchronized void startOccurred() {
isActive = true;
failureStreak = 0;
@ -285,7 +305,16 @@ public class FileSettingsService extends MasterNodeFileWatchingService implement
public synchronized void failureOccurred(String description) {
++failureStreak;
mostRecentFailure = description;
mostRecentFailure = limitLength(description);
}
private String limitLength(String description) {
int descriptionLengthLimit = DESCRIPTION_LENGTH_LIMIT.get(settings);
if (description.length() > descriptionLengthLimit) {
return description.substring(0, descriptionLengthLimit - 1) + "…";
} else {
return description;
}
}
@Override

5
server/src/main/java/org/elasticsearch/search/SearchFeatures.java
View file

@ -25,9 +25,12 @@ public final class SearchFeatures implements FeatureSpecification {
}
public static final NodeFeature RETRIEVER_RESCORER_ENABLED = new NodeFeature("search.retriever.rescorer.enabled");
public static final NodeFeature COMPLETION_FIELD_SUPPORTS_DUPLICATE_SUGGESTIONS = new NodeFeature(
"search.completion_field.duplicate.support"
);
@Override
public Set<NodeFeature> getTestFeatures() {
return Set.of(RETRIEVER_RESCORER_ENABLED);
return Set.of(RETRIEVER_RESCORER_ENABLED, COMPLETION_FIELD_SUPPORTS_DUPLICATE_SUGGESTIONS);
}
}

42
server/src/test/java/org/elasticsearch/TransportVersionTests.java
View file

@ -9,7 +9,6 @@
package org.elasticsearch;
import org.elasticsearch.core.FixForMultiProject;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.TransportVersionUtils;
@ -17,7 +16,6 @@ import java.lang.reflect.Modifier;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -200,44 +198,4 @@ public class TransportVersionTests extends ESTestCase {
assertEquals("2000099", TransportVersion.fromId(2_00_00_99).toString());
assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString());
}
/**
* Until 9.0 bumps its transport version to 9_000_00_0, all transport changes must be backported to 8.x.
* This test ensures transport versions are dense, so that we have confidence backports have not been missed.
* Note that it does not ensure patches are not missed, but it should catch the majority of misordered
* or missing transport versions.
*/
@FixForMultiProject
public void testDenseTransportVersions() {
Set<Integer> missingVersions = new TreeSet<>();
TransportVersion previous = null;
for (var tv : TransportVersion.getAllVersions()) {
if (tv.before(TransportVersions.V_8_16_0)) {
continue;
}
if (tv.equals(TransportVersions.MULTI_PROJECT)) {
continue; // Temporary work around. Remove this when Multi-Project merges to main (ES-7708)
}
if (previous == null) {
previous = tv;
continue;
}
if (previous.id() + 1000 < tv.id()) {
int nextId = previous.id();
do {
nextId = (nextId + 1000) / 1000 * 1000;
missingVersions.add(nextId);
} while (nextId + 1000 < tv.id());
}
previous = tv;
}
if (missingVersions.isEmpty() == false) {
StringBuilder msg = new StringBuilder("Missing transport versions:\n");
for (Integer id : missingVersions) {
msg.append(" " + id + "\n");
}
fail(msg.toString());
}
}
}

44
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java
View file

@ -20,7 +20,6 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.test.TransportVersionUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -37,7 +36,6 @@ import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseT
import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomMappingHashToIndices;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.nullValue;
public class FieldCapabilitiesNodeResponseTests extends AbstractWireSerializingTestCase<FieldCapabilitiesNodeResponse> {
@ -145,48 +143,6 @@ public class FieldCapabilitiesNodeResponseTests extends AbstractWireSerializingT
}
}
public void testSerializeNodeResponseBetweenOldNodes() throws IOException {
final TransportVersion minCompactVersion = TransportVersions.MINIMUM_COMPATIBLE;
assertTrue("Remove this test once minCompactVersion >= 8.2.0", minCompactVersion.before(TransportVersions.V_8_2_0));
List<FieldCapabilitiesIndexResponse> indexResponses = CollectionUtils.concatLists(
randomIndexResponsesWithMappingHash(randomMappingHashToIndices()),
randomIndexResponsesWithoutMappingHash()
);
Randomness.shuffle(indexResponses);
FieldCapabilitiesNodeResponse inResponse = randomNodeResponse(indexResponses);
TransportVersion version = TransportVersionUtils.randomVersionBetween(
random(),
minCompactVersion,
TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_2_0)
);
final FieldCapabilitiesNodeResponse outResponse = copyInstance(inResponse, version);
assertThat(outResponse.getFailures().keySet(), equalTo(inResponse.getFailures().keySet()));
assertThat(outResponse.getUnmatchedShardIds(), equalTo(inResponse.getUnmatchedShardIds()));
final List<FieldCapabilitiesIndexResponse> inList = inResponse.getIndexResponses();
final List<FieldCapabilitiesIndexResponse> outList = outResponse.getIndexResponses();
assertThat(outList, hasSize(inList.size()));
for (int i = 0; i < inList.size(); i++) {
assertThat("Responses between old nodes don't have mapping hash", outList.get(i).getIndexMappingHash(), nullValue());
assertThat(outList.get(i).getIndexName(), equalTo(inList.get(i).getIndexName()));
assertThat(outList.get(i).canMatch(), equalTo(inList.get(i).canMatch()));
Map<String, IndexFieldCapabilities> outCap = outList.get(i).get();
Map<String, IndexFieldCapabilities> inCap = inList.get(i).get();
if (version.onOrAfter(TransportVersions.V_8_0_0)) {
assertThat(outCap, equalTo(inCap));
} else {
// Exclude metric types which was introduced in 8.0
assertThat(outCap.keySet(), equalTo(inCap.keySet()));
for (String field : outCap.keySet()) {
assertThat(outCap.get(field).name(), equalTo(inCap.get(field).name()));
assertThat(outCap.get(field).type(), equalTo(inCap.get(field).type()));
assertThat(outCap.get(field).isSearchable(), equalTo(inCap.get(field).isSearchable()));
assertThat(outCap.get(field).isAggregatable(), equalTo(inCap.get(field).isAggregatable()));
assertThat(outCap.get(field).meta(), equalTo(inCap.get(field).meta()));
}
}
}
}
private static FieldCapabilitiesNodeResponse randomNodeResponse(List<FieldCapabilitiesIndexResponse> indexResponses) {
int numUnmatched = randomIntBetween(0, 3);
final Set<ShardId> unmatchedShardIds = new HashSet<>();

45
server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
View file

@ -40,7 +40,6 @@ import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseT
import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomMappingHashToIndices;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.nullValue;
public class FieldCapabilitiesResponseTests extends AbstractWireSerializingTestCase<FieldCapabilitiesResponse> {
@ -198,48 +197,4 @@ public class FieldCapabilitiesResponseTests extends AbstractWireSerializingTestC
}
}
}
public void testSerializeCCSResponseBetweenOldClusters() throws IOException {
TransportVersion minCompactVersion = TransportVersions.MINIMUM_COMPATIBLE;
assertTrue("Remove this test once minCompactVersion >= 8.2.0", minCompactVersion.before(TransportVersions.V_8_2_0));
List<FieldCapabilitiesIndexResponse> indexResponses = CollectionUtils.concatLists(
randomIndexResponsesWithMappingHash(randomMappingHashToIndices()),
randomIndexResponsesWithoutMappingHash()
);
Randomness.shuffle(indexResponses);
FieldCapabilitiesResponse inResponse = randomCCSResponse(indexResponses);
TransportVersion version = TransportVersionUtils.randomVersionBetween(
random(),
minCompactVersion,
TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_2_0)
);
final FieldCapabilitiesResponse outResponse = copyInstance(inResponse, version);
assertThat(
outResponse.getFailures().stream().flatMap(f -> Arrays.stream(f.getIndices())).toList(),
equalTo(inResponse.getFailures().stream().flatMap(f -> Arrays.stream(f.getIndices())).toList())
);
final List<FieldCapabilitiesIndexResponse> inList = inResponse.getIndexResponses();
final List<FieldCapabilitiesIndexResponse> outList = outResponse.getIndexResponses();
assertThat(outList, hasSize(inList.size()));
for (int i = 0; i < inList.size(); i++) {
assertThat("Responses between old clusters don't have mapping hash", outList.get(i).getIndexMappingHash(), nullValue());
assertThat(outList.get(i).getIndexName(), equalTo(inList.get(i).getIndexName()));
assertThat(outList.get(i).canMatch(), equalTo(inList.get(i).canMatch()));
Map<String, IndexFieldCapabilities> outCap = outList.get(i).get();
Map<String, IndexFieldCapabilities> inCap = inList.get(i).get();
if (version.onOrAfter(TransportVersions.V_8_0_0)) {
assertThat(outCap, equalTo(inCap));
} else {
// Exclude metric types which was introduced in 8.0
assertThat(outCap.keySet(), equalTo(inCap.keySet()));
for (String field : outCap.keySet()) {
assertThat(outCap.get(field).name(), equalTo(inCap.get(field).name()));
assertThat(outCap.get(field).type(), equalTo(inCap.get(field).type()));
assertThat(outCap.get(field).isSearchable(), equalTo(inCap.get(field).isSearchable()));
assertThat(outCap.get(field).isAggregatable(), equalTo(inCap.get(field).isAggregatable()));
assertThat(outCap.get(field).meta(), equalTo(inCap.get(field).meta()));
}
}
}
}
}

1
server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
View file

@ -220,7 +220,6 @@ public class SearchQueryThenFetchAsyncActionTests extends ESTestCase {
assertFalse(canReturnNullResponse.get());
assertThat(numWithTopDocs.get(), equalTo(0));
} else {
assertTrue(canReturnNullResponse.get());
if (withCollapse) {
assertThat(numWithTopDocs.get(), equalTo(0));
} else {

2
server/src/test/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableActionTests.java
View file

@ -333,7 +333,7 @@ public class TransportBroadcastUnpromotableActionTests extends ESTestCase {
// We were able to mark shards as stale, so the request finishes successfully
assertThat(safeAwait(broadcastUnpromotableRequest(wrongRoutingTable, true)), equalTo(ActionResponse.Empty.INSTANCE));
for (var shardRouting : wrongRoutingTable.unpromotableShards()) {
for (var shardRouting : wrongRoutingTable.assignedUnpromotableShards()) {
Mockito.verify(shardStateAction)
.remoteShardFailed(
eq(shardRouting.shardId()),

6
server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java
View file

@ -162,7 +162,7 @@ public class PostWriteRefreshTests extends IndexShardTestCase {
new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "message"),
ShardRouting.Role.SEARCH_ONLY
);
when(routingTable.allUnpromotableShards()).thenReturn(List.of(shardRouting));
when(routingTable.unpromotableShards()).thenReturn(List.of(shardRouting));
when(routingTable.shardId()).thenReturn(shardId);
WriteRequest.RefreshPolicy policy = randomFrom(WriteRequest.RefreshPolicy.IMMEDIATE, WriteRequest.RefreshPolicy.WAIT_UNTIL);
postWriteRefresh.refreshShard(policy, primary, result.getTranslogLocation(), f, postWriteRefreshTimeout);
@ -238,9 +238,9 @@ public class PostWriteRefreshTests extends IndexShardTestCase {
);
// Randomly test scenarios with and without unpromotables
if (randomBoolean()) {
when(routingTable.allUnpromotableShards()).thenReturn(Collections.emptyList());
when(routingTable.unpromotableShards()).thenReturn(Collections.emptyList());
} else {
when(routingTable.allUnpromotableShards()).thenReturn(List.of(shardRouting));
when(routingTable.unpromotableShards()).thenReturn(List.of(shardRouting));
}
WriteRequest.RefreshPolicy policy = WriteRequest.RefreshPolicy.WAIT_UNTIL;
postWriteRefresh.refreshShard(policy, primary, null, f, postWriteRefreshTimeout);

6
server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
View file

@ -612,11 +612,7 @@ public class ShardStateActionTests extends ESTestCase {
final String allocationId = randomRealisticUnicodeOfCodepointLengthBetween(10, 100);
final long primaryTerm = randomIntBetween(0, 100);
final String message = randomRealisticUnicodeOfCodepointLengthBetween(10, 100);
final TransportVersion version = randomFrom(
getFirstVersion(),
getPreviousVersion(TransportVersions.MINIMUM_COMPATIBLE),
getPreviousVersion(TransportVersions.V_8_15_0)
);
final TransportVersion version = randomFrom(getFirstVersion(), getPreviousVersion(TransportVersions.V_8_15_0));
final ShardLongFieldRange timestampRange = ShardLongFieldRangeWireTests.randomRange();
final ShardLongFieldRange eventIngestedRange = ShardLongFieldRangeWireTests.randomRange();
var startedShardEntry = new StartedShardEntry(shardId, allocationId, primaryTerm, message, timestampRange, eventIngestedRange);

36
server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
View file

@ -31,8 +31,6 @@ import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.test.NodeRoles.nonRemoteClusterClientNode;
import static org.elasticsearch.test.NodeRoles.remoteClusterClientNode;
import static org.elasticsearch.test.TransportVersionUtils.getPreviousVersion;
import static org.elasticsearch.test.TransportVersionUtils.randomVersionBetween;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -274,39 +272,5 @@ public class DiscoveryNodeTests extends ESTestCase {
}
}
}
{
var oldVersion = randomVersionBetween(
random(),
TransportVersions.MINIMUM_COMPATIBLE,
getPreviousVersion(TransportVersions.NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION)
);
try (var out = new BytesStreamOutput()) {
out.setTransportVersion(oldVersion);
node.writeTo(out);
try (var in = StreamInput.wrap(out.bytes().array())) {
in.setTransportVersion(oldVersion);
var deserialized = new DiscoveryNode(in);
assertThat(deserialized.getId(), equalTo(node.getId()));
assertThat(deserialized.getAddress(), equalTo(node.getAddress()));
assertThat(deserialized.getMinIndexVersion(), equalTo(node.getMinIndexVersion()));
assertThat(deserialized.getMaxIndexVersion(), equalTo(node.getMaxIndexVersion()));
assertThat(deserialized.getMinReadOnlyIndexVersion(), equalTo(node.getMinIndexVersion()));
assertThat(
deserialized.getVersionInformation(),
equalTo(
new VersionInformation(
node.getBuildVersion(),
node.getMinIndexVersion(),
node.getMinIndexVersion(),
node.getMaxIndexVersion()
)
)
);
}
}
}
}
}

49
server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
View file

@ -303,6 +303,55 @@ public class CompletionFieldMapperTests extends MapperTestCase {
);
}
public void testDuplicateSuggestionsWithContexts() throws IOException {
DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "completion");
b.startArray("contexts");
{
b.startObject();
b.field("name", "place");
b.field("type", "category");
b.endObject();
}
b.endArray();
}));
ParsedDocument parsedDocument = defaultMapper.parse(source(b -> {
b.startArray("field");
{
b.startObject();
{
b.array("input", "timmy", "starbucks");
b.startObject("contexts").array("place", "cafe", "food").endObject();
b.field("weight", 10);
}
b.endObject();
b.startObject();
{
b.array("input", "timmy", "starbucks");
b.startObject("contexts").array("place", "restaurant").endObject();
b.field("weight", 1);
}
b.endObject();
}
b.endArray();
}));
List<IndexableField> indexedFields = parsedDocument.rootDoc().getFields("field");
assertThat(indexedFields, hasSize(4));
assertThat(
indexedFields,
containsInAnyOrder(
contextSuggestField("timmy"),
contextSuggestField("timmy"),
contextSuggestField("starbucks"),
contextSuggestField("starbucks")
)
);
}
public void testCompletionWithContextAndSubCompletion() throws Exception {
DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
b.field("type", "completion");

7
server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
View file

@ -22,7 +22,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.TransportVersionUtils;
import org.elasticsearch.test.index.IndexVersionUtils;
import org.elasticsearch.xcontent.XContentBuilder;
import org.hamcrest.CoreMatchers;
@ -327,11 +326,7 @@ public class MappingParserTests extends MapperServiceTestCase {
IndexVersions.MINIMUM_READONLY_COMPATIBLE,
IndexVersions.V_8_5_0
);
TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween(
random(),
TransportVersions.MINIMUM_COMPATIBLE,
TransportVersions.V_8_5_0
);
TransportVersion transportVersion = TransportVersions.V_8_5_0;
{
XContentBuilder builder = mapping(b -> b.startObject(" ").field("type", randomFieldType()).endObject());
MappingParser mappingParser = createMappingParser(Settings.EMPTY, version, transportVersion);

69
server/src/test/java/org/elasticsearch/indices/cluster/ShardCloseExecutorTests.java Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.indices.cluster;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.test.ESTestCase;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger;
public class ShardCloseExecutorTests extends ESTestCase {
public void testThrottling() {
// This defaults to the number of CPUs of the machine running the tests which could be either side of 10.
final var defaultProcessors = EsExecutors.NODE_PROCESSORS_SETTING.get(Settings.EMPTY).roundUp();
ensureThrottling(Math.min(10, defaultProcessors), Settings.EMPTY);
if (10 < defaultProcessors) {
ensureThrottling(
10,
Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), between(10, defaultProcessors - 1)).build()
);
} // else we cannot run this check, the machine running the tests doesn't have enough CPUs
if (1 < defaultProcessors) {
final var fewProcessors = between(1, Math.min(10, defaultProcessors - 1));
ensureThrottling(fewProcessors, Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), fewProcessors).build());
} // else we cannot run this check, the machine running the tests has less than 2 whole CPUs (and we already tested the 1 case)
// but in any case we can override the throttle regardless of its default value
final var override = between(1, defaultProcessors * 2);
ensureThrottling(
override,
Settings.builder().put(IndicesClusterStateService.CONCURRENT_SHARD_CLOSE_LIMIT.getKey(), override).build()
);
}
private static void ensureThrottling(int expectedLimit, Settings settings) {
final var tasksToRun = new ArrayList<Runnable>(expectedLimit + 1);
final var executor = new IndicesClusterStateService.ShardCloseExecutor(settings, tasksToRun::add);
final var runCount = new AtomicInteger();
// enqueue one more task than the throttling limit
for (int i = 0; i < expectedLimit + 1; i++) {
executor.execute(runCount::incrementAndGet);
}
// check that we submitted tasks up to the expected limit, holding back the final task behind the throttle for now
assertEquals(expectedLimit, tasksToRun.size());
// now execute all the tasks one by one
for (int i = 0; i < expectedLimit + 1; i++) {
assertEquals(i, runCount.get());
tasksToRun.get(i).run();
assertEquals(i + 1, runCount.get());
// executing the first task enqueues the final task
assertEquals(expectedLimit + 1, tasksToRun.size());
}
}
}

28
server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsHealthIndicatorServiceTests.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.reservedstate.service;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.health.HealthIndicatorDetails;
import org.elasticsearch.health.HealthIndicatorResult;
import org.elasticsearch.health.SimpleHealthIndicatorDetails;
@ -21,6 +22,7 @@ import java.util.Map;
import static org.elasticsearch.health.HealthStatus.GREEN;
import static org.elasticsearch.health.HealthStatus.YELLOW;
import static org.elasticsearch.reservedstate.service.FileSettingsService.FileSettingsHealthIndicatorService.DESCRIPTION_LENGTH_LIMIT_KEY;
import static org.elasticsearch.reservedstate.service.FileSettingsService.FileSettingsHealthIndicatorService.FAILURE_SYMPTOM;
import static org.elasticsearch.reservedstate.service.FileSettingsService.FileSettingsHealthIndicatorService.INACTIVE_SYMPTOM;
import static org.elasticsearch.reservedstate.service.FileSettingsService.FileSettingsHealthIndicatorService.NO_CHANGES_SYMPTOM;
@ -37,7 +39,7 @@ public class FileSettingsHealthIndicatorServiceTests extends ESTestCase {
@Before
public void initialize() {
healthIndicatorService = new FileSettingsHealthIndicatorService();
healthIndicatorService = new FileSettingsHealthIndicatorService(Settings.EMPTY);
}
public void testInitiallyGreen() {}
@ -101,4 +103,28 @@ public class FileSettingsHealthIndicatorServiceTests extends ESTestCase {
healthIndicatorService.calculate(false, null)
);
}
public void testDescriptionIsTruncated() {
checkTruncatedDescription(9, "123456789", "123456789");
checkTruncatedDescription(8, "123456789", "1234567…");
checkTruncatedDescription(1, "12", "…");
}
private void checkTruncatedDescription(int lengthLimit, String description, String expectedTruncatedDescription) {
var service = new FileSettingsHealthIndicatorService(Settings.builder().put(DESCRIPTION_LENGTH_LIMIT_KEY, lengthLimit).build());
service.startOccurred();
service.changeOccurred();
service.failureOccurred(description);
assertEquals(
new HealthIndicatorResult(
"file_settings",
YELLOW,
FAILURE_SYMPTOM,
new SimpleHealthIndicatorDetails(Map.of("failure_streak", 1L, "most_recent_failure", expectedTruncatedDescription)),
STALE_SETTINGS_IMPACT,
List.of()
),
service.calculate(false, null)
);
}
}

2
server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
View file

@ -139,7 +139,7 @@ public class FileSettingsServiceTests extends ESTestCase {
List.of()
)
);
healthIndicatorService = spy(new FileSettingsHealthIndicatorService());
healthIndicatorService = spy(new FileSettingsHealthIndicatorService(Settings.EMPTY));
fileSettingsService = spy(new FileSettingsService(clusterService, controller, env, healthIndicatorService));
watchedFile = fileSettingsService.watchedFile();
}

88
test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
View file

@ -628,7 +628,7 @@ public class HeapAttackIT extends ESRestTestCase {
}
public void testLookupExplosion() throws IOException {
int sensorDataCount = 7500;
int sensorDataCount = 500;
int lookupEntries = 10000;
Map<?, ?> map = lookupExplosion(sensorDataCount, lookupEntries);
assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries))));
@ -636,19 +636,74 @@ public class HeapAttackIT extends ESRestTestCase {
public void testLookupExplosionManyMatches() throws IOException {
assertCircuitBreaks(() -> {
Map<?, ?> result = lookupExplosion(8500, 10000);
Map<?, ?> result = lookupExplosion(1500, 10000);
logger.error("should have failed but got {}", result);
});
}
public void testLookupExplosionNoFetch() throws IOException {
int sensorDataCount = 7500;
int lookupEntries = 10000;
Map<?, ?> map = lookupExplosionNoFetch(sensorDataCount, lookupEntries);
assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries))));
}
public void testLookupExplosionNoFetchManyMatches() throws IOException {
assertCircuitBreaks(() -> {
Map<?, ?> result = lookupExplosionNoFetch(8500, 10000);
logger.error("should have failed but got {}", result);
});
}
public void testLookupExplosionBigString() throws IOException {
int sensorDataCount = 150;
int lookupEntries = 1;
Map<?, ?> map = lookupExplosionBigString(sensorDataCount, lookupEntries);
assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries))));
}
public void testLookupExplosionBigStringManyMatches() throws IOException {
assertCircuitBreaks(() -> {
Map<?, ?> result = lookupExplosionBigString(500, 1);
logger.error("should have failed but got {}", result);
});
}
private Map<?, ?> lookupExplosion(int sensorDataCount, int lookupEntries) throws IOException {
initSensorData(sensorDataCount, 1);
initSensorLookup(lookupEntries, 1, i -> "73.9857 40.7484");
lookupExplosionData(sensorDataCount, lookupEntries);
StringBuilder query = startQuery();
query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(location)\"}");
return responseAsMap(query(query.toString(), null));
}
private Map<?, ?> lookupExplosionNoFetch(int sensorDataCount, int lookupEntries) throws IOException {
lookupExplosionData(sensorDataCount, lookupEntries);
StringBuilder query = startQuery();
query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(*)\"}");
return responseAsMap(query(query.toString(), null));
}
private void lookupExplosionData(int sensorDataCount, int lookupEntries) throws IOException {
initSensorData(sensorDataCount, 1);
initSensorLookup(lookupEntries, 1, i -> "73.9857 40.7484");
}
private Map<?, ?> lookupExplosionBigString(int sensorDataCount, int lookupEntries) throws IOException {
initSensorData(sensorDataCount, 1);
initSensorLookupString(lookupEntries, 1, i -> {
int target = Math.toIntExact(ByteSizeValue.ofMb(1).getBytes());
StringBuilder str = new StringBuilder(Math.toIntExact(ByteSizeValue.ofMb(2).getBytes()));
while (str.length() < target) {
str.append("Lorem ipsum dolor sit amet, consectetur adipiscing elit.");
}
logger.info("big string is {} characters", str.length());
return str.toString();
});
StringBuilder query = startQuery();
query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(string)\"}");
return responseAsMap(query(query.toString(), null));
}
public void testEnrichExplosion() throws IOException {
int sensorDataCount = 1000;
int lookupEntries = 100;
@ -834,6 +889,31 @@ public class HeapAttackIT extends ESRestTestCase {
initIndex("sensor_lookup", data.toString());
}
private void initSensorLookupString(int lookupEntries, int sensorCount, IntFunction<String> string) throws IOException {
logger.info("loading sensor lookup with huge strings");
createIndex("sensor_lookup", Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOOKUP.getName()).build(), """
{
"properties": {
"id": { "type": "long" },
"string": { "type": "text" }
}
}""");
int docsPerBulk = 10;
StringBuilder data = new StringBuilder();
for (int i = 0; i < lookupEntries; i++) {
int sensor = i % sensorCount;
data.append(String.format(Locale.ROOT, """
{"create":{}}
{"id": %d, "string": "%s"}
""", sensor, string.apply(sensor)));
if (i % docsPerBulk == docsPerBulk - 1) {
bulk("sensor_lookup", data.toString());
data.setLength(0);
}
}
initIndex("sensor_lookup", data.toString());
}
private void initSensorEnrich(int lookupEntries, int sensorCount, IntFunction<String> location) throws IOException {
initSensorLookup(lookupEntries, sensorCount, location);
logger.info("loading sensor enrich");

9
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View file

@ -2682,6 +2682,15 @@ public abstract class ESTestCase extends LuceneTestCase {
);
}
/**
* Checks a specific exception class with matched message is thrown by the given runnable, and returns it.
*/
public static <T extends Throwable> T expectThrows(Class<T> expectedType, Matcher<String> messageMatcher, ThrowingRunnable runnable) {
var e = expectThrows(expectedType, runnable);
assertThat(e.getMessage(), messageMatcher);
return e;
}
/**
* Same as {@link #runInParallel(int, IntConsumer)} but also attempts to start all tasks at the same time by blocking execution on a
* barrier until all threads are started and ready to execute their task.

13
test/framework/src/main/java/org/elasticsearch/test/LambdaMatchers.java
View file

@ -23,7 +23,7 @@ import java.util.function.Predicate;
public class LambdaMatchers {
private static class TransformMatcher<T, U> extends BaseMatcher<T> {
private static class TransformMatcher<T, U> extends TypeSafeMatcher<T> {
private final Matcher<U> matcher;
private final Function<T, U> transform;
@ -33,24 +33,21 @@ public class LambdaMatchers {
}
@Override
@SuppressWarnings("unchecked")
public boolean matches(Object actual) {
protected boolean matchesSafely(T item) {
U u;
try {
u = transform.apply((T) actual);
u = transform.apply(item);
} catch (ClassCastException e) {
throw new AssertionError(e);
}
return matcher.matches(u);
}
@Override
@SuppressWarnings("unchecked")
public void describeMismatch(Object item, Description description) {
protected void describeMismatchSafely(T item, Description description) {
U u;
try {
u = transform.apply((T) item);
u = transform.apply(item);
} catch (ClassCastException e) {
description.appendValue(item).appendText(" is not of the correct type (").appendText(e.getMessage()).appendText(")");
return;

3
test/framework/src/test/java/org/elasticsearch/test/LambdaMatchersTests.java
View file

@ -19,11 +19,13 @@ import static org.elasticsearch.test.LambdaMatchers.transformedArrayItemsMatch;
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
import static org.elasticsearch.test.LambdaMatchers.trueWith;
import static org.hamcrest.Matchers.anything;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.emptyString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
public class LambdaMatchersTests extends ESTestCase {
@ -56,6 +58,7 @@ public class LambdaMatchersTests extends ESTestCase {
assertThat(new A("1"), transformedMatch(a -> a.str, equalTo("1")));
assertThat(new B("1"), transformedMatch((A a) -> a.str, equalTo("1")));
assertMismatch((A) null, transformedMatch(A::toString, anything()), is("was null"));
assertMismatch(new A("1"), transformedMatch(a -> a.str, emptyString()), equalTo("transformed value was \"1\""));
}

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CoordinatedInferenceActionRequestTests.java
View file

@ -120,7 +120,7 @@ public class CoordinatedInferenceActionRequestTests extends AbstractBWCWireSeria
instance.setPrefixType(TrainedModelPrefixStrings.PrefixType.NONE);
}
return new CoordinatedInferenceAction.Request(
var newInstance = new CoordinatedInferenceAction.Request(
instance.getModelId(),
instance.getInputs(),
instance.getTaskSettings(),
@ -131,5 +131,7 @@ public class CoordinatedInferenceActionRequestTests extends AbstractBWCWireSeria
instance.getHighPriority(),
instance.getRequestModelType()
);
newInstance.setPrefixType(instance.getPrefixType());
return newInstance;
}
}

3
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
View file

@ -32,6 +32,7 @@ public class MetadataAttribute extends TypedAttribute {
public static final String TIMESTAMP_FIELD = "@timestamp";
public static final String TSID_FIELD = "_tsid";
public static final String SCORE = "_score";
public static final String INDEX = "_index";
static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
Attribute.class,
@ -42,7 +43,7 @@ public class MetadataAttribute extends TypedAttribute {
private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
"_version",
tuple(DataType.LONG, false), // _version field is not searchable
"_index",
INDEX,
tuple(DataType.KEYWORD, true),
IdFieldMapper.NAME,
tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default

10
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Holder.java
View file

@ -26,6 +26,16 @@ public class Holder<T> {
this.value = value;
}
/**
* Sets a value in the holder, but only if none has already been set.
* @param value the new value to set.
*/
public void setIfAbsent(T value) {
if (this.value == null) {
this.value = value;
}
}
public T get() {
return value;
}

27
x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
View file

@ -773,6 +773,33 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
}
}
public void testErrorMessageForMissingParams() throws IOException {
ResponseException re = expectThrows(
ResponseException.class,
() -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1").params("[]"))
);
assertThat(
EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""),
containsString("line 1:23: Unknown query parameter [n1]")
);
re = expectThrows(
ResponseException.class,
() -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1 and y == ?n2").params("[{\"n\" : \"v\"}]"))
);
assertThat(EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""), containsString("""
line 1:23: Unknown query parameter [n1], did you mean [n]?; line 1:36: Unknown query parameter [n2], did you mean [n]?"""));
re = expectThrows(
ResponseException.class,
() -> runEsql(requestObjectBuilder().query("from idx | where x == ?n1 and y == ?n2").params("[{\"n1\" : \"v1\"}]"))
);
assertThat(
EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""),
containsString("line 1:36: Unknown query parameter [n2], did you mean [n1]")
);
}
public void testErrorMessageForLiteralDateMathOverflow() throws IOException {
List<String> dateMathOverflowExpressions = List.of(
"2147483647 day + 1 day",

16
x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
View file

@ -11,6 +11,8 @@ import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.breaker.CircuitBreaker;
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
@ -31,9 +33,11 @@ import org.elasticsearch.geo.GeometryTestUtils;
import org.elasticsearch.geo.ShapeTestUtils;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.tasks.TaskCancelledException;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.transport.RemoteTransportException;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
@ -72,8 +76,8 @@ import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
import org.elasticsearch.xpack.esql.session.Configuration;
import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
import org.elasticsearch.xpack.esql.stats.SearchStats;
import org.elasticsearch.xpack.esql.telemetry.Metrics;
import org.elasticsearch.xpack.versionfield.Version;
@ -140,6 +144,7 @@ import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassificatio
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.mock;
public final class EsqlTestUtils {
@ -360,7 +365,14 @@ public final class EsqlTestUtils {
public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L));
public static final QueryBuilderResolver MOCK_QUERY_BUILDER_RESOLVER = new MockQueryBuilderResolver();
public static final TransportActionServices MOCK_TRANSPORT_ACTION_SERVICES = new TransportActionServices(
mock(TransportService.class),
mock(SearchService.class),
null,
mock(ClusterService.class),
mock(IndexNameExpressionResolver.class),
null
);
private EsqlTestUtils() {}

30
x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/MockQueryBuilderResolver.java
View file

@ -1,30 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
import org.elasticsearch.xpack.esql.session.Result;
import java.util.function.BiConsumer;
public class MockQueryBuilderResolver extends QueryBuilderResolver {
public MockQueryBuilderResolver() {
super(null, null, null, null);
}
@Override
public void resolveQueryBuilders(
LogicalPlan plan,
ActionListener<Result> listener,
BiConsumer<LogicalPlan, ActionListener<Result>> callback
) {
callback.accept(plan, listener);
}
}

2
x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-remote.csv-spec
View file

@ -39,7 +39,7 @@ max:integer |_index:keyword
;
metaIndexAliasedInAggs
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: metadata_fields_remote_test
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i | SORT _i;

6
x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata.csv-spec
View file

@ -40,7 +40,7 @@ max:integer |_index:keyword
;
metaIndexSorted
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | sort _index, emp_no desc | keep emp_no, _index | limit 2;
@ -50,7 +50,7 @@ emp_no:integer |_index:keyword
;
metaIndexWithInPredicate
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | where _index in ("employees", "foobar") | sort emp_no desc | keep emp_no, _index | limit 2;
@ -60,7 +60,7 @@ emp_no:integer |_index:keyword
;
metaIndexAliasedInAggs
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;

54
x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
View file

@ -133,7 +133,7 @@ mc:l | count:l
multiIndexIpString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_remove_fields
@ -162,7 +162,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexIpStringRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_remove_fields
@ -191,7 +191,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexIpStringRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_str METADATA _index
@ -219,7 +219,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexWhereIpString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_str METADATA _index
@ -237,7 +237,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected
multiIndexWhereIpStringLike
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_str METADATA _index
@ -445,7 +445,7 @@ count:long | message:keyword
multiIndexMissingIpToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_missing_field
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@ -480,7 +480,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450
multiIndexMissingIpToIp
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_missing_field
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@ -515,7 +515,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_long METADATA _index
@ -543,7 +543,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsLongRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_long METADATA _index
@ -573,7 +573,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsNanosRename
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_nanos METADATA _index
@ -602,7 +602,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsNanosRenameToNanos
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_nanos METADATA _index
@ -631,7 +631,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 34502
multiIndex sort millis and nanos as nanos
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_nanos METADATA _index
@ -660,7 +660,7 @@ sample_data | 2023-10-23T12:15:03.360000000Z | 172.21.2.162 | 34502
multiIndex sort millis and nanos as millis
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_nanos METADATA _index
@ -691,7 +691,7 @@ multiIndexTsNanosRenameToNanosWithFiltering
required_capability: to_date_nanos
required_capability: date_nanos_binary_comparison
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_nanos METADATA _index
@ -716,7 +716,7 @@ sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 12323
multiIndexTsLongRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_long METADATA _index
@ -744,7 +744,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexWhereTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
FROM sample_data, sample_data_ts_long METADATA _index
@ -979,7 +979,7 @@ count:long | message:keyword
multiIndexIpStringTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1022,7 +1022,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexIpStringTsLongDropped
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: to_date_nanos
FROM sample_data* METADATA _index
@ -1064,7 +1064,7 @@ sample_data_ts_nanos | 8268153 | Connection error
multiIndexIpStringTsLongRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1107,7 +1107,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexIpStringTsLongRenameDropped
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: to_date_nanos
FROM sample_data* METADATA _index
@ -1149,7 +1149,7 @@ sample_data_ts_nanos | 8268153 | Connection error
multiIndexIpStringTsLongRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1192,7 +1192,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexWhereIpStringTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1226,7 +1226,7 @@ count:long | message:keyword
multiIndexWhereIpStringLikeTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1260,7 +1260,7 @@ count:long | message:keyword
multiIndexMultiColumnTypesRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1279,7 +1279,7 @@ null | null | 8268153 | Connectio
multiIndexMultiColumnTypesRenameAndKeep
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1299,7 +1299,7 @@ sample_data_ts_nanos | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015123456
multiIndexMultiColumnTypesRenameAndDrop
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos
@ -1591,7 +1591,7 @@ FROM sample_data, sample_data_ts_long
shortIntegerWidening
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_numeric_widening

2
x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java
View file

@ -246,7 +246,7 @@ public class MatchFunctionIT extends AbstractEsqlIntegTestCase {
var error = expectThrows(ElasticsearchException.class, () -> run(query));
assertThat(
error.getMessage(),
containsString("[MATCH] function cannot operate on [\"a brown fox\"], which is not a field from an index mapping")
containsString("line 2:15: [MATCH] function cannot operate on [content], which is not a field from an index mapping")
);
}

2
x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
View file

@ -230,7 +230,7 @@ public class MatchOperatorIT extends AbstractEsqlIntegTestCase {
var error = expectThrows(ElasticsearchException.class, () -> run(query));
assertThat(
error.getMessage(),
containsString("[:] operator cannot operate on [\"a brown fox\"], which is not a field from an index mapping")
containsString("line 2:9: [:] operator cannot operate on [content], which is not a field from an index mapping")
);
}

7
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file

@ -121,12 +121,17 @@ public class EsqlCapabilities {
* Cast string literals to a desired data type for IN predicate and more types for BinaryComparison.
*/
STRING_LITERAL_AUTO_CASTING_EXTENDED,
/**
* Support for metadata fields.
*/
METADATA_FIELDS,
/**
* Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently
* supported.
*/
INDEX_METADATA_FIELD,
/**
* Support for timespan units abbreviations
*/

6
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
View file

@ -21,10 +21,10 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
import org.elasticsearch.xpack.esql.session.Configuration;
import org.elasticsearch.xpack.esql.session.EsqlSession;
import org.elasticsearch.xpack.esql.session.IndexResolver;
import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
import org.elasticsearch.xpack.esql.session.Result;
import org.elasticsearch.xpack.esql.telemetry.Metrics;
import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
@ -62,7 +62,7 @@ public class PlanExecutor {
EsqlExecutionInfo executionInfo,
IndicesExpressionGrouper indicesExpressionGrouper,
EsqlSession.PlanRunner planRunner,
QueryBuilderResolver queryBuilderResolver,
TransportActionServices services,
ActionListener<Result> listener
) {
final PlanTelemetry planTelemetry = new PlanTelemetry(functionRegistry);
@ -78,7 +78,7 @@ public class PlanExecutor {
verifier,
planTelemetry,
indicesExpressionGrouper,
queryBuilderResolver
services
);
QueryMetric clientId = QueryMetric.fromString("rest");
metrics.total(clientId);

8
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
View file

@ -7,7 +7,7 @@
package org.elasticsearch.xpack.esql.expression.function.fulltext;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.compute.lucene.LuceneQueryExpressionEvaluator;
import org.elasticsearch.compute.lucene.LuceneQueryExpressionEvaluator.ShardConfig;
import org.elasticsearch.compute.operator.EvalOperator;
@ -110,11 +110,7 @@ public abstract class FullTextFunction extends Function implements TranslationAw
*/
public Object queryAsObject() {
Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */);
if (queryAsObject instanceof BytesRef bytesRef) {
return bytesRef.utf8ToString();
}
return queryAsObject;
return BytesRefs.toString(queryAsObject);
}
@Override

65
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
View file

@ -14,7 +14,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
import org.elasticsearch.xpack.esql.common.Failure;
import org.elasticsearch.xpack.esql.common.Failures;
import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
@ -30,6 +30,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
import org.elasticsearch.xpack.esql.core.util.Check;
import org.elasticsearch.xpack.esql.core.util.NumericUtils;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@ -38,6 +39,7 @@ import org.elasticsearch.xpack.esql.expression.function.OptionalArgument;
import org.elasticsearch.xpack.esql.expression.function.Param;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import org.elasticsearch.xpack.esql.querydsl.query.MatchQuery;
import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
@ -48,6 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiConsumer;
import static java.util.Map.entry;
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
@ -88,7 +91,7 @@ import static org.elasticsearch.xpack.esql.expression.predicate.operator.compari
/**
* Full text function that performs a {@link org.elasticsearch.xpack.esql.querydsl.query.MatchQuery} .
*/
public class Match extends FullTextFunction implements OptionalArgument, PostOptimizationVerificationAware {
public class Match extends FullTextFunction implements OptionalArgument, PostAnalysisPlanVerificationAware {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::readFrom);
public static final Set<DataType> FIELD_DATA_TYPES = Set.of(
@ -429,23 +432,23 @@ public class Match extends FullTextFunction implements OptionalArgument, PostOpt
}
@Override
public void postOptimizationVerification(Failures failures) {
Expression fieldExpression = field();
// Field may be converted to other data type (field_name :: data_type), so we need to check the original field
if (fieldExpression instanceof AbstractConvertFunction convertFunction) {
fieldExpression = convertFunction.field();
}
if (fieldExpression instanceof FieldAttribute == false) {
failures.add(
Failure.fail(
field,
"[{}] {} cannot operate on [{}], which is not a field from an index mapping",
functionName(),
functionType(),
field.sourceText()
)
);
}
public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
return (plan, failures) -> {
super.postAnalysisPlanVerification().accept(plan, failures);
plan.forEachExpression(Match.class, m -> {
if (m.fieldAsFieldAttribute() == null) {
failures.add(
Failure.fail(
m.field(),
"[{}] {} cannot operate on [{}], which is not a field from an index mapping",
functionName(),
functionType(),
m.field().sourceText()
)
);
}
});
};
}
@Override
@ -476,22 +479,24 @@ public class Match extends FullTextFunction implements OptionalArgument, PostOpt
@Override
protected Query translate(TranslatorHandler handler) {
var fieldAttribute = fieldAsFieldAttribute();
Check.notNull(fieldAttribute, "Match must have a field attribute as the first argument");
String fieldName = fieldAttribute.name();
if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) {
// If we have multiple field types, we allow the query to be done, but getting the underlying field name
fieldName = multiTypeEsField.getName();
}
// Make query lenient so mixed field types can be queried when a field type is incompatible with the value provided
return new MatchQuery(source(), fieldName, queryAsObject(), matchQueryOptions());
}
private FieldAttribute fieldAsFieldAttribute() {
Expression fieldExpression = field;
// Field may be converted to other data type (field_name :: data_type), so we need to check the original field
if (fieldExpression instanceof AbstractConvertFunction convertFunction) {
fieldExpression = convertFunction.field();
}
if (fieldExpression instanceof FieldAttribute fieldAttribute) {
String fieldName = fieldAttribute.name();
if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) {
// If we have multiple field types, we allow the query to be done, but getting the underlying field name
fieldName = multiTypeEsField.getName();
}
// Make query lenient so mixed field types can be queried when a field type is incompatible with the value provided
return new MatchQuery(source(), fieldName, queryAsObject(), matchQueryOptions());
}
throw new IllegalArgumentException("Match must have a field attribute as the first argument");
return fieldExpression instanceof FieldAttribute fieldAttribute ? fieldAttribute : null;
}
@Override

95
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.fulltext;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ResolvedIndices;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.Rewriteable;
import org.elasticsearch.xpack.esql.core.util.Holder;
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
import org.elasticsearch.xpack.esql.session.IndexResolver;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
* Some {@link FullTextFunction} implementations such as {@link org.elasticsearch.xpack.esql.expression.function.fulltext.Match}
* will be translated to a {@link QueryBuilder} that require a rewrite phase on the coordinator.
* {@link QueryBuilderResolver#resolveQueryBuilders(LogicalPlan, TransportActionServices, ActionListener)} will rewrite the plan by
* replacing {@link FullTextFunction} expression with new ones that hold rewritten {@link QueryBuilder}s.
*/
public final class QueryBuilderResolver {
private QueryBuilderResolver() {}
public static void resolveQueryBuilders(LogicalPlan plan, TransportActionServices services, ActionListener<LogicalPlan> listener) {
var hasFullTextFunctions = plan.anyMatch(p -> {
Holder<Boolean> hasFullTextFunction = new Holder<>(false);
p.forEachExpression(FullTextFunction.class, unused -> hasFullTextFunction.set(true));
return hasFullTextFunction.get();
});
if (hasFullTextFunctions) {
Rewriteable.rewriteAndFetch(
new FullTextFunctionsRewritable(plan),
queryRewriteContext(services, indexNames(plan)),
listener.delegateFailureAndWrap((l, r) -> l.onResponse(r.plan))
);
} else {
listener.onResponse(plan);
}
}
private static QueryRewriteContext queryRewriteContext(TransportActionServices services, Set<String> indexNames) {
ResolvedIndices resolvedIndices = ResolvedIndices.resolveWithIndexNamesAndOptions(
indexNames.toArray(String[]::new),
IndexResolver.FIELD_CAPS_INDICES_OPTIONS,
services.clusterService().state().metadata().getProject(),
services.indexNameExpressionResolver(),
services.transportService().getRemoteClusterService(),
System.currentTimeMillis()
);
return services.searchService().getRewriteContext(System::currentTimeMillis, resolvedIndices, null);
}
private static Set<String> indexNames(LogicalPlan plan) {
Set<String> indexNames = new HashSet<>();
plan.forEachDown(EsRelation.class, esRelation -> indexNames.addAll(esRelation.concreteIndices()));
return indexNames;
}
private record FullTextFunctionsRewritable(LogicalPlan plan) implements Rewriteable<QueryBuilderResolver.FullTextFunctionsRewritable> {
@Override
public FullTextFunctionsRewritable rewrite(QueryRewriteContext ctx) throws IOException {
Holder<IOException> exceptionHolder = new Holder<>();
Holder<Boolean> updated = new Holder<>(false);
LogicalPlan newPlan = plan.transformExpressionsDown(FullTextFunction.class, f -> {
QueryBuilder builder = f.queryBuilder(), initial = builder;
builder = builder == null ? f.asQuery(TranslatorHandler.TRANSLATOR_HANDLER).asBuilder() : builder;
try {
builder = builder.rewrite(ctx);
} catch (IOException e) {
exceptionHolder.setIfAbsent(e);
}
var rewritten = builder != initial;
updated.set(updated.get() || rewritten);
return rewritten ? f.replaceQueryBuilder(builder) : f;
});
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
return updated.get() ? new FullTextFunctionsRewritable(newPlan) : this;
}
}
}

35
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java
View file

@ -12,7 +12,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware;
import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware;
import org.elasticsearch.xpack.esql.common.Failure;
import org.elasticsearch.xpack.esql.common.Failures;
import org.elasticsearch.xpack.esql.core.expression.Expression;
@ -26,10 +26,12 @@ import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import java.io.IOException;
import java.util.List;
import java.util.function.BiConsumer;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
@ -39,7 +41,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isStr
/**
* Full text function that performs a {@link TermQuery} .
*/
public class Term extends FullTextFunction implements PostOptimizationVerificationAware {
public class Term extends FullTextFunction implements PostAnalysisPlanVerificationAware {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Term", Term::readFrom);
@ -104,18 +106,23 @@ public class Term extends FullTextFunction implements PostOptimizationVerificati
}
@Override
public void postOptimizationVerification(Failures failures) {
if (field instanceof FieldAttribute == false) {
failures.add(
Failure.fail(
field,
"[{}] {} cannot operate on [{}], which is not a field from an index mapping",
functionName(),
functionType(),
field.sourceText()
)
);
}
public BiConsumer<LogicalPlan, Failures> postAnalysisPlanVerification() {
return (plan, failures) -> {
super.postAnalysisPlanVerification().accept(plan, failures);
plan.forEachExpression(Term.class, t -> {
if (t.field() instanceof FieldAttribute == false) { // TODO: is a conversion possible, similar to Match's case?
failures.add(
Failure.fail(
t.field(),
"[{}] {} cannot operate on [{}], which is not a field from an index mapping",
t.functionName(),
t.functionType(),
t.field().sourceText()
)
);
}
});
};
}
@Override

12
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
View file

@ -108,17 +108,7 @@ public class LogicalPlanBuilder extends ExpressionBuilder {
if (errors.hasNext() == false) {
return p;
} else {
StringBuilder message = new StringBuilder();
int i = 0;
while (errors.hasNext()) {
if (i > 0) {
message.append("; ");
}
message.append(errors.next().getMessage());
i++;
}
throw new ParsingException(message.toString());
throw ParsingException.combineParsingExceptions(errors);
}
}

38
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParsingException.java
View file

@ -9,6 +9,8 @@ package org.elasticsearch.xpack.esql.parser;
import org.elasticsearch.xpack.esql.EsqlClientException;
import org.elasticsearch.xpack.esql.core.tree.Source;
import java.util.Iterator;
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
public class ParsingException extends EsqlClientException {
@ -21,6 +23,10 @@ public class ParsingException extends EsqlClientException {
this.charPositionInLine = charPositionInLine + 1;
}
/**
* To be used only if the exception cannot be associated with a specific position in the query.
* Error message will start with {@code line -1:-1:} instead of using specific location.
*/
public ParsingException(String message, Object... args) {
this(Source.EMPTY, message, args);
}
@ -37,6 +43,38 @@ public class ParsingException extends EsqlClientException {
this.charPositionInLine = source.source().getColumnNumber();
}
private ParsingException(int line, int charPositionInLine, String message, Object... args) {
super(message, args);
this.line = line;
this.charPositionInLine = charPositionInLine;
}
/**
* Combine multiple {@code ParsingException} into one, this is used by {@code LogicalPlanBuilder} to
* consolidate multiple named parameters related {@code ParsingException}.
*/
public static ParsingException combineParsingExceptions(Iterator<ParsingException> parsingExceptions) {
StringBuilder message = new StringBuilder();
int i = 0;
int line = -1;
int charPositionInLine = -1;
while (parsingExceptions.hasNext()) {
ParsingException e = parsingExceptions.next();
if (i > 0) {
message.append("; ");
message.append(e.getMessage());
} else {
// line and column numbers are the associated with the first error
line = e.getLineNumber();
charPositionInLine = e.getColumnNumber();
message.append(e.getErrorMessage());
}
i++;
}
return new ParsingException(line, charPositionInLine, message.toString());
}
public int getLineNumber() {
return line;
}

40
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/premapper/PreMapper.java Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.planner.premapper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryBuilderResolver;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
/**
* The class is responsible for invoking any premapping steps that need to be applied to the logical plan,
* before this is being mapped to a physical one.
*/
public class PreMapper {
private final TransportActionServices services;
public PreMapper(TransportActionServices services) {
this.services = services;
}
/**
* Invokes any premapping steps that need to be applied to the logical plan, before this is being mapped to a physical one.
*/
public void preMapper(LogicalPlan plan, ActionListener<LogicalPlan> listener) {
queryRewrite(plan, listener.delegateFailureAndWrap((l, p) -> {
p.setOptimized();
l.onResponse(p);
}));
}
private void queryRewrite(LogicalPlan plan, ActionListener<LogicalPlan> listener) {
QueryBuilderResolver.resolveQueryBuilders(plan, services, listener);
}
}

24
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportActionServices.java Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.plugin;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.compute.operator.exchange.ExchangeService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.usage.UsageService;
public record TransportActionServices(
TransportService transportService,
SearchService searchService,
ExchangeService exchangeService,
ClusterService clusterService,
IndexNameExpressionResolver indexNameExpressionResolver,
UsageService usageService
) {}

15
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
View file

@ -53,7 +53,6 @@ import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService;
import org.elasticsearch.xpack.esql.execution.PlanExecutor;
import org.elasticsearch.xpack.esql.session.Configuration;
import org.elasticsearch.xpack.esql.session.EsqlSession.PlanRunner;
import org.elasticsearch.xpack.esql.session.QueryBuilderResolver;
import org.elasticsearch.xpack.esql.session.Result;
import java.io.IOException;
@ -81,8 +80,8 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
private final LookupFromIndexService lookupFromIndexService;
private final AsyncTaskManagementService<EsqlQueryRequest, EsqlQueryResponse, EsqlQueryTask> asyncTaskManagementService;
private final RemoteClusterService remoteClusterService;
private final QueryBuilderResolver queryBuilderResolver;
private final UsageService usageService;
private final TransportActionServices services;
// Listeners for active async queries, key being the async task execution ID
private final Map<String, EsqlQueryListener> asyncListeners = ConcurrentCollections.newConcurrentMap();
@ -153,8 +152,16 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
bigArrays
);
this.remoteClusterService = transportService.getRemoteClusterService();
this.queryBuilderResolver = new QueryBuilderResolver(searchService, clusterService, transportService, indexNameExpressionResolver);
this.usageService = usageService;
this.services = new TransportActionServices(
transportService,
searchService,
exchangeService,
clusterService,
indexNameExpressionResolver,
usageService
);
}
@Override
@ -258,7 +265,7 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
executionInfo,
remoteClusterService,
planRunner,
queryBuilderResolver,
services,
ActionListener.wrap(result -> {
recordCCSTelemetry(task, executionInfo, request, null);
listener.onResponse(toResponse(task, request, configuration, result));

24
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
View file

@ -73,6 +73,8 @@ import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
import org.elasticsearch.xpack.esql.planner.premapper.PreMapper;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
import java.util.ArrayList;
@ -109,12 +111,12 @@ public class EsqlSession {
private final Verifier verifier;
private final EsqlFunctionRegistry functionRegistry;
private final LogicalPlanOptimizer logicalPlanOptimizer;
private final PreMapper preMapper;
private final Mapper mapper;
private final PhysicalPlanOptimizer physicalPlanOptimizer;
private final PlanTelemetry planTelemetry;
private final IndicesExpressionGrouper indicesExpressionGrouper;
private final QueryBuilderResolver queryBuilderResolver;
public EsqlSession(
String sessionId,
@ -128,7 +130,7 @@ public class EsqlSession {
Verifier verifier,
PlanTelemetry planTelemetry,
IndicesExpressionGrouper indicesExpressionGrouper,
QueryBuilderResolver queryBuilderResolver
TransportActionServices services
) {
this.sessionId = sessionId;
this.configuration = configuration;
@ -142,7 +144,7 @@ public class EsqlSession {
this.physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(configuration));
this.planTelemetry = planTelemetry;
this.indicesExpressionGrouper = indicesExpressionGrouper;
this.queryBuilderResolver = queryBuilderResolver;
this.preMapper = new PreMapper(services);
}
public String sessionId() {
@ -162,16 +164,12 @@ public class EsqlSession {
new EsqlSessionCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
@Override
public void onResponse(LogicalPlan analyzedPlan) {
try {
var optimizedPlan = optimizedPlan(analyzedPlan);
queryBuilderResolver.resolveQueryBuilders(
optimizedPlan,
listener,
(newPlan, next) -> executeOptimizedPlan(request, executionInfo, planRunner, newPlan, next)
);
} catch (Exception e) {
listener.onFailure(e);
}
preMapper.preMapper(
analyzedPlan,
listener.delegateFailureAndWrap(
(l, p) -> executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(p), l)
)
);
}
}
);

3
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
View file

@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.TimeSeriesParams;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.DateEsField;
import org.elasticsearch.xpack.esql.core.type.EsField;
@ -50,7 +51,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
public class IndexResolver {
public static final Set<String> ALL_FIELDS = Set.of("*");
public static final Set<String> INDEX_METADATA_FIELD = Set.of("_index");
public static final Set<String> INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX);
public static final String UNMAPPED = "unmapped";
public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
View file

@ -516,7 +516,7 @@ public class CsvTests extends ESTestCase {
TEST_VERIFIER,
new PlanTelemetry(functionRegistry),
null,
EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER
EsqlTestUtils.MOCK_TRANSPORT_ACTION_SERVICES
);
TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets);

23
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
View file

@ -1186,9 +1186,9 @@ public class VerifierTests extends ESTestCase {
public void testMatchInsideEval() throws Exception {
assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot());
assertEquals(
"1:36: [:] operator is only supported in WHERE commands",
"1:36: [:] operator is only supported in WHERE commands\n"
+ "line 1:36: [:] operator cannot operate on [title], which is not a field from an index mapping",
error("row title = \"brown fox\" | eval x = title:\"fox\" ")
);
}
@ -1217,6 +1217,25 @@ public class VerifierTests extends ESTestCase {
assertEquals("1:24: [:] operator cannot be used after LIMIT", error("from test | limit 10 | where first_name : \"Anna\""));
}
// These should pass eventually once we lift some restrictions on match function
public void testMatchWithNonIndexedColumnCurrentlyUnsupported() {
assertEquals(
"1:67: [MATCH] function cannot operate on [initial], which is not a field from an index mapping",
error("from test | eval initial = substring(first_name, 1) | where match(initial, \"A\")")
);
assertEquals(
"1:67: [MATCH] function cannot operate on [text], which is not a field from an index mapping",
error("from test | eval text=concat(first_name, last_name) | where match(text, \"cat\")")
);
}
public void testMatchFunctionIsNotNullable() {
assertEquals(
"1:48: [MATCH] function cannot operate on [text::keyword], which is not a field from an index mapping",
error("row n = null | eval text = n + 5 | where match(text::keyword, \"Anna\")")
);
}
public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception {
// Source commands
assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")"));

23
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
View file

@ -61,7 +61,7 @@ public abstract class AbstractAggregationTestCase extends AbstractFunctionTestCa
* Use if possible, as this method may get updated with new checks in the future.
* </p>
*/
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
List<TestCaseSupplier> suppliers,
boolean entirelyNullPreservesType,
PositionalErrorMessageSupplier positionalErrorMessageSupplier
@ -74,13 +74,24 @@ public abstract class AbstractAggregationTestCase extends AbstractFunctionTestCa
);
}
// TODO: Remove and migrate everything to the method with all the parameters
/**
* @deprecated Use {@link #parameterSuppliersFromTypedDataWithDefaultChecks(List, boolean, PositionalErrorMessageSupplier)} instead.
* This method doesn't add all the default checks.
* Converts a list of test cases into a list of parameter suppliers.
* Also, adds a default set of extra test cases.
* <p>
* Use if possible, as this method may get updated with new checks in the future.
* </p>
*
* @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)}
*/
@Deprecated
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(List<TestCaseSupplier> suppliers) {
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(
// TODO remove after removing parameterSuppliersFromTypedDataWithDefaultChecks rename this to that.
List<TestCaseSupplier> suppliers,
boolean entirelyNullPreservesType
) {
return parameterSuppliersFromTypedData(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)));
}
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(List<TestCaseSupplier> suppliers) {
return parameterSuppliersFromTypedData(withNoRowsExpectingNull(randomizeBytesRefsOffset(suppliers)));
}

51
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
View file

@ -51,33 +51,6 @@ import static org.hamcrest.Matchers.sameInstance;
* which can be automatically tested against several scenarios (null handling, concurrency, etc).
*/
public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTestCase {
/**
* Converts a list of test cases into a list of parameter suppliers.
* Also, adds a default set of extra test cases.
* <p>
* Use if possible, as this method may get updated with new checks in the future.
* </p>
*
* @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)}
* @deprecated use {@link #parameterSuppliersFromTypedDataWithDefaultChecksNoErrors}
* and make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase}.
* It's a <strong>long</strong> faster.
*/
@Deprecated
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
boolean entirelyNullPreservesType,
List<TestCaseSupplier> suppliers,
PositionalErrorMessageSupplier positionalErrorMessageSupplier
) {
return parameterSuppliersFromTypedData(
errorsForCasesWithoutExamples(
anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)),
positionalErrorMessageSupplier
)
);
}
/**
* Converts a list of test cases into a list of parameter suppliers.
* Also, adds a default set of extra test cases.
@ -113,30 +86,6 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
return parameterSuppliersFromTypedData(anyNullIsNull(randomizeBytesRefsOffset(suppliers), nullsExpectedType, evaluatorToString));
}
/**
* Converts a list of test cases into a list of parameter suppliers.
* Also, adds a default set of extra test cases.
* <p>
* Use if possible, as this method may get updated with new checks in the future.
* </p>
*
* @param nullsExpectedType See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)}
* @param evaluatorToString See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)}
*/
protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
ExpectedType nullsExpectedType,
ExpectedEvaluatorToString evaluatorToString,
List<TestCaseSupplier> suppliers,
PositionalErrorMessageSupplier positionalErrorMessageSupplier
) {
return parameterSuppliersFromTypedData(
errorsForCasesWithoutExamples(
anyNullIsNull(randomizeBytesRefsOffset(suppliers), nullsExpectedType, evaluatorToString),
positionalErrorMessageSupplier
)
);
}
public final void testEvaluate() {
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
boolean readFloating = randomBoolean();

37
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgErrorTests.java Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.aggregate;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.hamcrest.Matcher;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.equalTo;
public class AvgErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
@Override
protected List<TestCaseSupplier> cases() {
return paramsToSuppliers(AvgTests.parameters());
}
@Override
protected Expression build(Source source, List<Expression> args) {
return new Avg(source, args.get(0));
}
@Override
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric except unsigned_long or counter types"));
}
}

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
View file

@ -53,7 +53,7 @@ public class AvgTests extends AbstractAggregationTestCase {
)
);
return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, true, (v, p) -> "numeric except unsigned_long or counter types");
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true);
}
@Override

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
View file

@ -93,7 +93,6 @@ public class CountDistinctTests extends AbstractAggregationTestCase {
}
// "No rows" expects 0 here instead of null
// return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
}

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java
View file

@ -82,7 +82,6 @@ public class CountTests extends AbstractAggregationTestCase {
}
// "No rows" expects 0 here instead of null
// return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
}

39
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.aggregate;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.hamcrest.Matcher;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.equalTo;
public class MaxErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
@Override
protected List<TestCaseSupplier> cases() {
return paramsToSuppliers(MaxTests.parameters());
}
@Override
protected Expression build(Source source, List<Expression> args) {
return new Max(source, args.get(0));
}
@Override
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
return equalTo(
typeErrorMessage(false, validPerPosition, signature, (v, p) -> "representable except unsigned_long and spatial types")
);
}
}

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
View file

@ -157,11 +157,7 @@ public class MaxTests extends AbstractAggregationTestCase {
)
);
return parameterSuppliersFromTypedDataWithDefaultChecks(
suppliers,
false,
(v, p) -> "representable except unsigned_long and spatial types"
);
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, false);
}
@Override

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviationTests.java
View file

@ -39,7 +39,7 @@ public class MedianAbsoluteDeviationTests extends AbstractAggregationTestCase {
MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
).flatMap(List::stream).map(MedianAbsoluteDeviationTests::makeSupplier).toList();
return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true);
}
@Override

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianTests.java
View file

@ -73,7 +73,7 @@ public class MedianTests extends AbstractAggregationTestCase {
)
);
return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true);
}
@Override

39
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.aggregate;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.hamcrest.Matcher;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.equalTo;
public class MinErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
@Override
protected List<TestCaseSupplier> cases() {
return paramsToSuppliers(MinTests.parameters());
}
@Override
protected Expression build(Source source, List<Expression> args) {
return new Min(source, args.get(0));
}
@Override
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
return equalTo(
typeErrorMessage(false, validPerPosition, signature, (v, p) -> "representable except unsigned_long and spatial types")
);
}
}

6
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
View file

@ -157,11 +157,7 @@ public class MinTests extends AbstractAggregationTestCase {
)
);
return parameterSuppliersFromTypedDataWithDefaultChecks(
suppliers,
false,
(v, p) -> "representable except unsigned_long and spatial types"
);
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, false);
}
@Override

37
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileErrorTests.java Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.aggregate;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.hamcrest.Matcher;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.equalTo;
public class PercentileErrorTests extends ErrorsForCasesWithoutExamplesTestCase {
@Override
protected List<TestCaseSupplier> cases() {
return paramsToSuppliers(PercentileTests.parameters());
}
@Override
protected Expression build(Source source, List<Expression> args) {
return new Percentile(source, args.get(0), args.get(1));
}
@Override
protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerPosition, List<DataType> signature) {
return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "numeric except unsigned_long"));
}
}

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java
View file

@ -53,7 +53,7 @@ public class PercentileTests extends AbstractAggregationTestCase {
}
}
return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, false, (v, p) -> "numeric except unsigned_long");
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, false);
}
@Override

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
View file

@ -47,7 +47,6 @@ public class SpatialCentroidTests extends AbstractAggregationTestCase {
).flatMap(List::stream).map(SpatialCentroidTests::makeSupplier).toList();
// The withNoRowsExpectingNull() cases don't work here, as this aggregator doesn't return nulls.
// return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
}

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialExtentTests.java
View file

@ -48,7 +48,6 @@ public class SpatialExtentTests extends AbstractAggregationTestCase {
).flatMap(List::stream).map(SpatialExtentTests::makeSupplier).toList();
// The withNoRowsExpectingNull() cases don't work here, as this aggregator doesn't return nulls.
// return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
}

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java
View file

@ -41,7 +41,7 @@ public class StdDevTests extends AbstractAggregationTestCase {
MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
).flatMap(List::stream).map(StdDevTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true);
}
@Override

Some files were not shown because too many files have changed in this diff Show more