From 1c7d17c295ee1db6c0fa872a50aa93d04ad243a5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 24 Feb 2025 17:48:07 -0800 Subject: [PATCH 01/31] Remove unnecessary temp dir access (#123314) All modules have read/write access to the temp dir. This commit removes unnecessarily adding the temp dir explicitly to the server policy. --- .../initialization/EntitlementInitialization.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 96bf8d3ba6ab..b4dd40001afc 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -176,7 +176,6 @@ public class EntitlementInitialization { new FilesEntitlement( List.of( // Base ES directories - FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE), FileData.ofPath(bootstrapArgs.configDir(), READ), FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE), @@ -211,11 +210,7 @@ public class EntitlementInitialization { new LoadNativeLibrariesEntitlement(), new ManageThreadsEntitlement(), new FilesEntitlement( - List.of( - FileData.ofPath(bootstrapArgs.configDir(), READ), - FileData.ofPath(bootstrapArgs.tempDir(), READ), - FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE) - ) + List.of(FileData.ofPath(bootstrapArgs.configDir(), READ), FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)) ) ) ), From 688542b515f290c42b75dfeffdfd89b6fee50a10 Mon Sep 17 00:00:00 2001 From: Stanislav Malyshev Date: Mon, 24 Feb 2025 21:48:18 -0700 Subject: [PATCH 02/31] Increase await timeout for testCancelFailedSearchWhenPartialResultDisallowed (#123084) --- muted-tests.yml | 3 --- .../java/org/elasticsearch/search/SearchCancellationIT.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index a6a5e1a1f8fa..c5a962741eb0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -257,9 +257,6 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously} issue: https://github.com/elastic/elasticsearch/issues/122102 -- class: org.elasticsearch.search.SearchCancellationIT - method: testCancelFailedSearchWhenPartialResultDisallowed - issue: https://github.com/elastic/elasticsearch/issues/121719 - class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT issue: https://github.com/elastic/elasticsearch/issues/121716 - class: org.elasticsearch.smoketest.SmokeTestMonitoringWithSecurityIT diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java index 0cc1c89b36d1..8701c88f9d41 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java @@ -267,7 +267,7 @@ public class SearchCancellationIT extends AbstractSearchCancellationTestCase { if (letOneShardProceed.compareAndSet(false, true)) { // Let one shard continue. } else { - safeAwait(shardTaskLatch); // Block the other shards. + safeAwait(shardTaskLatch, TimeValue.timeValueSeconds(30)); // Block the other shards. } }); } From 4a6343ad0b8d9212ad0a5606b546c075ff33bdb9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 25 Feb 2025 07:39:14 +0000 Subject: [PATCH 03/31] Update minimum transport version in `main` (#123254) Versions 9.1.0 onwards will not be wire-compatible with versions before 8.19.0. This commit sets the minimum transport version to reject handshakes from earlier versions. --- server/src/main/java/org/elasticsearch/TransportVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 35cd890af7fc..05afa77e4391 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -264,7 +264,7 @@ public class TransportVersions { * Reference to the earliest compatible transport version to this version of the codebase. * This should be the transport version used by the highest minor version of the previous major. */ - public static final TransportVersion MINIMUM_COMPATIBLE = BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1; + public static final TransportVersion MINIMUM_COMPATIBLE = INITIAL_ELASTICSEARCH_8_19; /** * Reference to the minimum transport version that can be used with CCS. From 5b0591e04a03d776031d1b4391e56e0332b7f362 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Feb 2025 00:05:36 -0800 Subject: [PATCH 04/31] Consider entitlement lib as system module (#123315) * Consider entitlement lib as system module Entitlements sometimes needs to perform sensitive operations, particularly within the FileAccessTree. This commit expands the trivially allowed check to include entitlements as one of the system modules alongside the jdk. One consequence is that the self test must be moved outside entitlements. * [CI] Auto commit changes from spotless * remove old method call --------- Co-authored-by: elasticsearchmachine Co-authored-by: Elastic Machine --- .../bootstrap/EntitlementBootstrap.java | 49 ------------------- .../runtime/policy/PolicyManager.java | 16 +++--- .../bootstrap/Elasticsearch.java | 32 ++++++++++++ 3 files changed, 38 insertions(+), 59 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 4f37362d9325..8610d9f3be66 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -14,16 +14,13 @@ import com.sun.tools.attach.AgentLoadException; import com.sun.tools.attach.AttachNotSupportedException; import com.sun.tools.attach.VirtualMachine; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; -import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -114,7 +111,6 @@ public class EntitlementBootstrap { ); exportInitializationToAgent(); loadAgent(findAgentJar()); - selfTest(); } @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically") @@ -160,50 +156,5 @@ public class EntitlementBootstrap { } } - /** - * Attempt a few sensitive operations to ensure that some are permitted and some are forbidden. - *

- * - * This serves two purposes: - * - *

    - *
  1. - * a smoke test to make sure the entitlements system is not completely broken, and - *
    2. - *
  2. - * an early test of certain important operations so they don't fail later on at an awkward time. - *
    3. - *
- * - * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing - */ - private static void selfTest() { - ensureCannotStartProcess(ProcessBuilder::start); - // Try again with reflection - ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess); - } - - private static void ensureCannotStartProcess(CheckedConsumer startProcess) { - try { - // The command doesn't matter; it doesn't even need to exist - startProcess.accept(new ProcessBuilder("")); - } catch (NotEntitledException e) { - logger.debug("Success: Entitlement protection correctly prevented process creation"); - return; - } catch (Exception e) { - throw new IllegalStateException("Failed entitlement protection self-test", e); - } - throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); - } - - private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { - try { - var start = ProcessBuilder.class.getMethod("start"); - start.invoke(pb); - } catch (InvocationTargetException e) { - throw (Exception) e.getCause(); - } - } - private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 2aafcfc594ab..b6296fe5d471 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -12,7 +12,6 @@ package org.elasticsearch.entitlement.runtime.policy; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; -import org.elasticsearch.entitlement.bridge.EntitlementChecker; import org.elasticsearch.entitlement.instrumentation.InstrumentationService; import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; @@ -126,11 +125,12 @@ public class PolicyManager { .stream() .map(ModuleReference::descriptor) .collect(Collectors.toUnmodifiableSet()); - return ModuleLayer.boot() - .modules() - .stream() - .filter(m -> systemModulesDescriptors.contains(m.getDescriptor())) - .collect(Collectors.toUnmodifiableSet()); + return Stream.concat( + // entitlements is a "system" module, we can do anything from it + Stream.of(PolicyManager.class.getModule()), + // anything in the boot layer is also part of the system + ModuleLayer.boot().modules().stream().filter(m -> systemModulesDescriptors.contains(m.getDescriptor())) + ).collect(Collectors.toUnmodifiableSet()); } /** @@ -564,10 +564,6 @@ public class PolicyManager { logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName()); return true; } - if (EntitlementChecker.class.isAssignableFrom(requestingClass)) { - logger.debug("Entitlement trivially allowed for EntitlementChecker class"); - return true; - } logger.trace("Entitlement not trivially allowed"); return false; } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 3b67b86eff01..37dfd46db581 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -29,9 +29,11 @@ import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.entitlement.runtime.policy.PolicyParserUtils; import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement; @@ -54,6 +56,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.lang.invoke.MethodHandles; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.security.Permission; @@ -254,6 +257,7 @@ class Elasticsearch { nodeEnv.logsDir(), nodeEnv.tmpDir() ); + entitlementSelfTest(); } else { assert RuntimeVersionFeature.isSecurityManagerAvailable(); // no need to explicitly enable native access for legacy code @@ -270,6 +274,34 @@ class Elasticsearch { bootstrap.setPluginsLoader(pluginsLoader); } + // check entitlements were loaded correctly. note this must be outside the entitlements lib. + private static void entitlementSelfTest() { + ensureCannotStartProcess(ProcessBuilder::start); + // Try again with reflection + ensureCannotStartProcess(Elasticsearch::reflectiveStartProcess); + } + + private static void ensureCannotStartProcess(CheckedConsumer startProcess) { + try { + // The command doesn't matter; it doesn't even need to exist + startProcess.accept(new ProcessBuilder("")); + } catch (NotEntitledException e) { + return; + } catch (Exception e) { + throw new IllegalStateException("Failed entitlement protection self-test", e); + } + throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); + } + + private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { + try { + var start = ProcessBuilder.class.getMethod("start"); + start.invoke(pb); + } catch (InvocationTargetException e) { + throw (Exception) e.getCause(); + } + } + private static void ensureInitialized(Class... classes) { for (final var clazz : classes) { try { From 4bd1f81ef90a283a9271f8d079ecf2196b541ca9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 25 Feb 2025 09:30:42 +0100 Subject: [PATCH 05/31] Move eclipse specific extention property into elasticsearch eclipse plugin (#123320) --- .../internal/conventions/EclipseConventionPlugin.java | 10 ++++++++++ build.gradle | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java index 58b183fac315..48465cb08cc7 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java @@ -15,6 +15,7 @@ import org.gradle.api.GradleException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Transformer; +import org.gradle.api.invocation.Gradle; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.Copy; @@ -38,6 +39,15 @@ public class EclipseConventionPlugin implements Plugin { @Override public void apply(Project project) { project.getPlugins().apply(EclipsePlugin.class); + Gradle gradle = project.getGradle(); + + boolean isEclipse = project.getProviders().systemProperty("eclipse.launcher").isPresent() || // Gradle launched from Eclipse + project.getProviders().systemProperty("eclipse.application").isPresent() || // Gradle launched from the Eclipse compiler server + gradle.getStartParameter().getTaskNames().contains("eclipse") || // Gradle launched from the command line to do eclipse stuff + gradle.getStartParameter().getTaskNames().contains("cleanEclipse"); + // for eclipse ide specific hacks... + project.getExtensions().add("isEclipse", isEclipse); + EclipseModel eclipseModel = project.getExtensions().getByType(EclipseModel.class); EclipseProject eclipseProject = eclipseModel.getProject(); diff --git a/build.gradle b/build.gradle index 440032675213..1f8f11fe0511 100644 --- a/build.gradle +++ b/build.gradle @@ -247,15 +247,6 @@ allprojects { } } - // injecting groovy property variables into all projects - project.ext { - // for ide hacks... - isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE - providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server - gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff - gradle.startParameter.taskNames.contains('cleanEclipse') - } - ext.bwc_tests_enabled = bwc_tests_enabled // eclipse configuration From 9aaf07242c0e76dcb4739a1d3f8b5a9f26b2e55c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Feb 2025 01:18:56 -0800 Subject: [PATCH 06/31] Add pidfile access for server (#123313) --- .../bootstrap/EntitlementBootstrap.java | 10 +++- .../EntitlementInitialization.java | 58 ++++++++++--------- .../bootstrap/Elasticsearch.java | 3 +- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 8610d9f3be66..06e985913c9b 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -41,7 +41,8 @@ public class EntitlementBootstrap { Path configDir, Path libDir, Path logsDir, - Path tempDir + Path tempDir, + Path pidFile ) { public BootstrapArgs { requireNonNull(pluginPolicies); @@ -80,6 +81,7 @@ public class EntitlementBootstrap { * @param libDir the lib directory for Elasticsearch * @param tempDir the temp directory for Elasticsearch * @param logsDir the log directory for Elasticsearch + * @param pidFile path to a pid file for Elasticsearch, or {@code null} if one was not specified */ public static void bootstrap( Map pluginPolicies, @@ -91,7 +93,8 @@ public class EntitlementBootstrap { Path configDir, Path libDir, Path logsDir, - Path tempDir + Path tempDir, + Path pidFile ) { logger.debug("Loading entitlement agent"); if (EntitlementBootstrap.bootstrapArgs != null) { @@ -107,7 +110,8 @@ public class EntitlementBootstrap { configDir, libDir, logsDir, - tempDir + tempDir, + pidFile ); exportInitializationToAgent(); loadAgent(findAgentJar()); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index b4dd40001afc..e1749eb2a4bb 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -148,6 +148,36 @@ public class EntitlementInitialization { ); List serverScopes = new ArrayList<>(); + List serverModuleFileDatas = new ArrayList<>(); + Collections.addAll( + serverModuleFileDatas, + // Base ES directories + FileData.ofPath(bootstrapArgs.configDir(), READ), + FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), + FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE), + FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE), + + // OS release on Linux + FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/usr/lib/os-release"), READ).withPlatform(LINUX), + // read max virtual memory areas + FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/proc/meminfo"), READ).withPlatform(LINUX), + // load averages on Linux + FileData.ofPath(Path.of("/proc/loadavg"), READ).withPlatform(LINUX), + // control group stats on Linux. cgroup v2 stats are in an unpredicable + // location under `/sys/fs/cgroup`, so unfortunately we have to allow + // read access to the entire directory hierarchy. + FileData.ofPath(Path.of("/proc/self/cgroup"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ).withPlatform(LINUX), + // // io stats on Linux + FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX), + FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX) + ); + if (bootstrapArgs.pidFile() != null) { + serverModuleFileDatas.add(FileData.ofPath(bootstrapArgs.pidFile(), READ_WRITE)); + } Collections.addAll( serverScopes, new Scope( @@ -173,33 +203,7 @@ public class EntitlementInitialization { new OutboundNetworkEntitlement(), new LoadNativeLibrariesEntitlement(), new ManageThreadsEntitlement(), - new FilesEntitlement( - List.of( - // Base ES directories - FileData.ofPath(bootstrapArgs.configDir(), READ), - FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE), - FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE), - FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE), - - // OS release on Linux - FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX), - FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX), - FileData.ofPath(Path.of("/usr/lib/os-release"), READ).withPlatform(LINUX), - // read max virtual memory areas - FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ).withPlatform(LINUX), - FileData.ofPath(Path.of("/proc/meminfo"), READ).withPlatform(LINUX), - // load averages on Linux - FileData.ofPath(Path.of("/proc/loadavg"), READ).withPlatform(LINUX), - // control group stats on Linux. cgroup v2 stats are in an unpredicable - // location under `/sys/fs/cgroup`, so unfortunately we have to allow - // read access to the entire directory hierarchy. - FileData.ofPath(Path.of("/proc/self/cgroup"), READ).withPlatform(LINUX), - FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ).withPlatform(LINUX), - // // io stats on Linux - FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX), - FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX) - ) - ) + new FilesEntitlement(serverModuleFileDatas) ) ), new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())), diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 37dfd46db581..05072ae940ae 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -255,7 +255,8 @@ class Elasticsearch { nodeEnv.configDir(), nodeEnv.libDir(), nodeEnv.logsDir(), - nodeEnv.tmpDir() + nodeEnv.tmpDir(), + args.pidFile() ); entitlementSelfTest(); } else { From 12fcdd8633f03159a8f1505d2f828ed0bba06709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 25 Feb 2025 10:19:15 +0100 Subject: [PATCH 07/31] Some missing entitlements preventing serverless to start (#123271) --- .../entitlement/initialization/EntitlementInitialization.java | 4 +++- .../security/src/main/plugin-metadata/entitlement-policy.yaml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index e1749eb2a4bb..0fef2c27df66 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -255,7 +255,9 @@ public class EntitlementInitialization { new FilesEntitlement( List.of( FileData.ofPath(Path.of("/co/elastic/apm/agent/"), READ), - FileData.ofPath(Path.of("/agent/co/elastic/apm/agent/"), READ) + FileData.ofPath(Path.of("/agent/co/elastic/apm/agent/"), READ), + FileData.ofPath(Path.of("/proc/meminfo"), READ), + FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ) ) ) ); diff --git a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml index 99dd7d5c1380..a6f29cb2ad7e 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/security/src/main/plugin-metadata/entitlement-policy.yaml @@ -37,3 +37,6 @@ org.opensaml.saml.impl: - relative_path: metadata.xml relative_to: config mode: read + - relative_path: "saml/" + relative_to: config + mode: read From ec82c24a8722e7c4b50392567bafef7d96051849 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 25 Feb 2025 11:38:51 +0100 Subject: [PATCH 08/31] Add support to VALUES aggregation for spatial types (#122886) The original work at https://github.com/elastic/elasticsearch/pull/106065 did not support geospatial types with this comment: > I made this work for everything but geo_point and cartesian_point because I'm not 100% sure how to integrate with those. We can grab those in a follow up. The geospatial types should be possible to collect using the VALUES aggregation with similar behavior to the `ST_COLLECT` OGC function, based on the Elasticsearch convention that treats multi-value geospatial fields as behaving similarly to any geometry collection. So this implementation is a trivial addition to the existing values types support. --- docs/changelog/122886.yaml | 6 ++ .../functions/kibana/definition/values.json | 48 ++++++++++++++ x-pack/plugin/build.gradle | 2 + .../src/main/resources/spatial.csv-spec | 64 +++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 ++ .../esql/expression/EsqlTypeResolutions.java | 11 ---- .../expression/function/aggregate/Values.java | 43 ++++++++++--- .../function/AbstractFunctionTestCase.java | 8 ++- .../function/aggregate/ValuesErrorTests.java | 2 +- .../function/aggregate/ValuesTests.java | 7 +- .../rest-api-spec/test/esql/130_spatial.yml | 29 +++++---- 11 files changed, 189 insertions(+), 36 deletions(-) create mode 100644 docs/changelog/122886.yaml diff --git a/docs/changelog/122886.yaml b/docs/changelog/122886.yaml new file mode 100644 index 000000000000..7306a21d1470 --- /dev/null +++ b/docs/changelog/122886.yaml @@ -0,0 +1,6 @@ +pr: 122886 +summary: Add support to VALUES aggregation for spatial types +area: ES|QL +type: bug +issues: + - 122413 diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json index 95ac402bb242..0ac74c61cf73 100644 --- a/docs/reference/esql/functions/kibana/definition/values.json +++ b/docs/reference/esql/functions/kibana/definition/values.json @@ -16,6 +16,30 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_point" + }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, { "params" : [ { @@ -52,6 +76,30 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_point" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, { "params" : [ { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index d3052cb191a0..784eed9bf0fa 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -102,6 +102,8 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/180_match_operator/match with disjunctions", "Disjunctions in full text functions work now") + task.skipTest("esql/130_spatial/values unsupported for geo_point", "Spatial types are now supported in VALUES aggregation") + task.skipTest("esql/130_spatial/values unsupported for geo_point status code", "Spatial types are now supported in VALUES aggregation") // Expected deprecation warning to compat yaml tests: task.addAllowedWarningRegex(".*rollup functionality will be removed in Elasticsearch.*") task.skipTest("esql/40_tsdb/from doc with aggregate_metric_double", "TODO: support for subset of metric fields") diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 8718112979ce..b5c588a1b486 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -144,6 +144,70 @@ c:long | x:double | y:double 19 | null | null ; +values +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| STATS locations=VALUES(location) +| EVAL locations = MV_SORT(TO_STRING(locations)) +; + +locations:keyword +[POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (127.495916124681 36.7220227766673), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (75.9570722403652 30.8503598561702), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)] +; + +valuesGrouped +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| EVAL first_letter = SUBSTRING(abbrev, 0, 1) +| STATS locations=VALUES(location) BY first_letter +| EVAL locations = MV_SORT(TO_STRING(locations)) +| SORT first_letter +| KEEP first_letter, locations +; + +first_letter:keyword | locations:keyword +A | POINT (48.7471065435931 31.3431585560757) +B | POINT (83.5504532124038 53.3633850813046) +C | [POINT (127.495916124681 36.7220227766673), POINT (61.5122589740201 55.2977919496055)] +G | POINT (78.2172186546348 26.285487697937) +H | POINT (42.97109630194 14.7552534413725) +I | [POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.8092915005895 22.727749187571), POINT (76.8017261105242 30.6707248949667), POINT (81.7317271462187 25.443522027821), POINT (85.3235970368767 23.3177245989962)] +K | POINT (126.810839481226 35.1400051390198) +L | [POINT (72.9878190922305 31.3627435480862), POINT (75.9570722403652 30.8503598561702)] +M | POINT (112.711418617258 -7.92998002840567) +O | [POINT (35.3018728575279 47.8732635579023), POINT (73.0320498392002 33.5614146278861), POINT (73.3163595376585 54.9576482934059), POINT (82.6671524525865 55.0095847136264)] +P | POINT (101.446569298441 0.464600872998505) +R | POINT (145.243980298582 14.1717712971216) +S | [POINT (73.4084964764375 61.3401672194481), POINT (75.9330597710755 17.625415183635)] +T | [POINT (128.637537699933 35.8999277969087), POINT (63.0279333519181 25.988794590011), POINT (78.7089578747476 10.7603571306554)] +U | [POINT (129.355731047528 35.5928957527107), POINT (66.9487311480949 30.249043186181)] +V | [POINT (75.7584828456005 31.4329422397715), POINT (79.452002687657 28.4218087161144)] +W | POINT (105.176060419161 -5.242566777132) +Z | POINT (60.900708564915 29.4752941956573) +; + +valuesGroupedByOrdinals +required_capability: agg_values_spatial + +FROM airports +| WHERE scalerank == 9 +| STATS locations=VALUES(location) BY type +| EVAL locations = MV_SORT(TO_STRING(locations)) +| SORT type +| KEEP type, locations +; + +type:keyword | locations:keyword +major | [POINT (127.495916124681 36.7220227766673), POINT (76.8017261105242 30.6707248949667)] +mid | [POINT (101.446569298441 0.464600872998505), POINT (105.176060419161 -5.242566777132), POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (128.637537699933 35.8999277969087), POINT (129.355731047528 35.5928957527107), POINT (145.243980298582 14.1717712971216), POINT (35.3018728575279 47.8732635579023), POINT (42.97109630194 14.7552534413725), POINT (48.7471065435931 31.3431585560757), POINT (60.900708564915 29.4752941956573), POINT (61.5122589740201 55.2977919496055), POINT (63.0279333519181 25.988794590011), POINT (66.9487311480949 30.249043186181), POINT (72.9878190922305 31.3627435480862), POINT (73.3163595376585 54.9576482934059), POINT (73.4084964764375 61.3401672194481), POINT (73.8105674924689 19.9660205672806), POINT (75.3958432922005 19.8672969621082), POINT (75.7584828456005 31.4329422397715), POINT (75.8092915005895 22.727749187571), POINT (75.9330597710755 17.625415183635), POINT (78.2172186546348 26.285487697937), POINT (78.7089578747476 10.7603571306554), POINT (82.6671524525865 55.0095847136264), POINT (83.5504532124038 53.3633850813046), POINT (85.3235970368767 23.3177245989962)] +military | [POINT (112.711418617258 -7.92998002840567), POINT (126.810839481226 35.1400051390198), POINT (35.3018728575279 47.8732635579023), POINT (72.9878190922305 31.3627435480862), POINT (75.7584828456005 31.4329422397715), POINT (76.8017261105242 30.6707248949667), POINT (78.2172186546348 26.285487697937), POINT (79.452002687657 28.4218087161144), POINT (81.7317271462187 25.443522027821)] +small | [POINT (73.0320498392002 33.5614146278861), POINT (75.9570722403652 30.8503598561702)] +; + ############################################### # Tests for ST_CENTROID_AGG on GEO_POINT type diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5f88f9f34827..06764454b142 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -81,6 +81,11 @@ public class EsqlCapabilities { */ AGG_VALUES, + /** + * Expand the {@code VALUES} agg to cover spatial types. + */ + AGG_VALUES_SPATIAL, + /** * Does ESQL support async queries. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java index b97374d179a4..23f875e4a059 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java @@ -63,12 +63,6 @@ public class EsqlTypeResolutions { GEO_SHAPE.typeName(), CARTESIAN_SHAPE.typeName() }; private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() }; - private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types() - .stream() - .filter(DataType::isRepresentable) - .filter(t -> DataType.isSpatial(t) == false) - .map(DataType::esType) - .toArray(String[]::new); public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES); @@ -77,9 +71,4 @@ public class EsqlTypeResolutions { public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES); } - - public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { - return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES); - } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index 4dbe0e93b501..6ffee21411ac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -50,11 +50,28 @@ public class Values extends AggregateFunction implements ToAggregator { Map.entry(DataType.SEMANTIC_TEXT, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.GEO_POINT, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.CARTESIAN_POINT, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.GEO_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.CARTESIAN_SHAPE, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new) ); @FunctionInfo( - returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" }, + returnType = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "version" }, preview = true, description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. " + "If you need the values returned in order use <>.", @@ -74,7 +91,21 @@ public class Values extends AggregateFunction implements ToAggregator { Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" } + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "date_nanos", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" } ) Expression v ) { this(source, v, Literal.TRUE); @@ -115,13 +146,7 @@ public class Values extends AggregateFunction implements ToAggregator { @Override protected TypeResolution resolveType() { - return TypeResolutions.isType( - field(), - SUPPLIERS::containsKey, - sourceText(), - DEFAULT, - "any type except unsigned_long and spatial types" - ); + return TypeResolutions.isType(field(), SUPPLIERS::containsKey, sourceText(), DEFAULT, "any type except unsigned_long"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 9ed140558a47..a6d8e3c2e057 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -858,7 +858,9 @@ public abstract class AbstractFunctionTestCase extends ESTestCase { @AfterClass public static void renderSignature() throws IOException { - if (System.getProperty("generateDocs") == null) { + // Temporarily turn off docs generation during docs freeze + // TODO: Only turn this back on once this generates the correct MD files + if (System.getProperty("generateDocs") == null || true) { return; } String name = functionName(); @@ -933,7 +935,9 @@ public abstract class AbstractFunctionTestCase extends ESTestCase { } protected static void renderDocs(String name) throws IOException { - if (System.getProperty("generateDocs") == null) { + // Temporarily turn off docs generation during docs freeze + // TODO: Only turn this back on once this generates the correct MD files + if (System.getProperty("generateDocs") == null || true) { return; } if (binaryOperator(name) != null || unaryOperator(name) != null || searchOperator(name) != null || likeOrInOperator(name)) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java index f9dafc954b6f..d34fe1df2982 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java @@ -32,6 +32,6 @@ public class ValuesErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long and spatial types")); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except unsigned_long")); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java index 80e6a7fc09d5..57795ee9270d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java @@ -52,7 +52,12 @@ public class ValuesTests extends AbstractAggregationTestCase { // Lower values for strings, as they take more space and may trigger the circuit breaker MultiRowTestCaseSupplier.stringCases(1, 20, DataType.KEYWORD), MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT), - MultiRowTestCaseSupplier.stringCases(1, 20, DataType.SEMANTIC_TEXT) + MultiRowTestCaseSupplier.stringCases(1, 20, DataType.SEMANTIC_TEXT), + // For spatial types, we can have many rows for points, but reduce rows for shapes to avoid circuit breaker + MultiRowTestCaseSupplier.geoPointCases(1, 1000, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.cartesianPointCases(1, 1000, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.geoShapeCasesWithoutCircle(1, 100, MultiRowTestCaseSupplier.IncludingAltitude.NO), + MultiRowTestCaseSupplier.cartesianShapeCasesWithoutCircle(1, 100, MultiRowTestCaseSupplier.IncludingAltitude.NO) ).flatMap(List::stream).map(ValuesTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, false); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml index ad3adbf93ada..e63262081728 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml @@ -3,7 +3,7 @@ setup: - requires: cluster_features: ["gte_v8.14.0"] reason: "Mixed cluster tests don't work with the changed error message from sort" - test_runner_features: allowed_warnings_regex + test_runner_features: [ capabilities, allowed_warnings_regex ] - do: indices.create: @@ -148,20 +148,25 @@ geo_point unsortable with limit from row: query: 'ROW wkt = ["POINT(42.9711 -14.7553)", "POINT(75.8093 22.7277)"] | MV_EXPAND wkt | EVAL pt = TO_GEOPOINT(wkt) | limit 5 | sort pt' --- -values unsupported for geo_point: +values supported for geo_point: + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ agg_values_spatial ] + reason: "Spatial types added to values aggregation in 8.19.0" - do: - catch: '/.+argument of \[VALUES\(location\)\] must be .+/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM geo_points | STATS VALUES(location)' - ---- -values unsupported for geo_point status code: - - do: - catch: bad_request - esql.query: - body: - query: 'FROM geo_points | STATS VALUES(location)' + query: 'FROM geo_points | STATS locations = VALUES(location) | EVAL locations = MV_SORT(TO_STRING(locations))' + - length: { columns: 1 } + - match: { columns.0.name: locations } + - match: { columns.0.type: keyword } + - length: { values: 1 } + - match: { values.0.0: ["POINT (-1.0 1.0)", "POINT (1.0 -1.0)"] } --- cartesian_point: From 4f6a0e6840042133f937b0ddfd160e97ac2b68a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 25 Feb 2025 13:39:58 +0100 Subject: [PATCH 09/31] [Entitlements] Follows links during FileAccessTree creation (#123357) --- .../runtime/policy/FileAccessTree.java | 48 +++++++++++++++---- .../runtime/policy/FileAccessTreeTests.java | 44 +++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java index d0eded74556b..336a00643e97 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java @@ -10,18 +10,27 @@ package org.elasticsearch.entitlement.runtime.policy; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; +import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.function.BiConsumer; import static org.elasticsearch.core.PathUtils.getDefaultFileSystem; public final class FileAccessTree { + private static final Logger logger = LogManager.getLogger(FileAccessTree.class); private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator(); private final String[] readPaths; @@ -30,6 +39,27 @@ public final class FileAccessTree { private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) { List readPaths = new ArrayList<>(); List writePaths = new ArrayList<>(); + BiConsumer addPath = (path, mode) -> { + var normalized = normalizePath(path); + if (mode == Mode.READ_WRITE) { + writePaths.add(normalized); + } + readPaths.add(normalized); + }; + BiConsumer addPathAndMaybeLink = (path, mode) -> { + addPath.accept(path, mode); + // also try to follow symlinks. Lucene does this and writes to the target path. + if (Files.exists(path)) { + try { + Path realPath = path.toRealPath(); + if (realPath.equals(path) == false) { + addPath.accept(realPath, mode); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }; for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) { var platform = fileData.platform(); if (platform != null && platform.isCurrent() == false) { @@ -38,18 +68,20 @@ public final class FileAccessTree { var mode = fileData.mode(); var paths = fileData.resolvePaths(pathLookup); paths.forEach(path -> { - var normalized = normalizePath(path); - if (mode == FilesEntitlement.Mode.READ_WRITE) { - writePaths.add(normalized); + if (path == null) { + // TODO: null paths shouldn't be allowed, but they can occur due to repo paths + return; } - readPaths.add(normalized); + addPathAndMaybeLink.accept(path, mode); }); } - // everything has access to the temp dir - String tempDir = normalizePath(pathLookup.tempDir()); - readPaths.add(tempDir); - writePaths.add(tempDir); + // everything has access to the temp dir and the jdk + addPathAndMaybeLink.accept(pathLookup.tempDir(), Mode.READ_WRITE); + + // TODO: watcher uses javax.activation which looks for known mime types configuration, should this be global or explicit in watcher? + Path jdk = Paths.get(System.getProperty("java.home")); + addPathAndMaybeLink.accept(jdk.resolve("conf"), Mode.READ); readPaths.sort(PATH_ORDER); writePaths.sort(PATH_ORDER); diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java index 71ec497b9ec1..98fd98b75719 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java @@ -10,11 +10,15 @@ package org.elasticsearch.entitlement.runtime.policy; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -23,6 +27,7 @@ import java.util.Map; import static org.elasticsearch.core.PathUtils.getDefaultFileSystem; import static org.hamcrest.Matchers.is; +@ESTestCase.WithoutSecurityManager public class FileAccessTreeTests extends ESTestCase { static Path root; @@ -211,6 +216,45 @@ public class FileAccessTreeTests extends ESTestCase { assertThat(tree.canRead(path("m/n")), is(true)); } + public void testJdkAccess() { + Path jdkDir = Paths.get(System.getProperty("java.home")); + var confDir = jdkDir.resolve("conf"); + var tree = accessTree(FilesEntitlement.EMPTY); + + assertThat(tree.canRead(confDir), is(true)); + assertThat(tree.canWrite(confDir), is(false)); + assertThat(tree.canRead(jdkDir), is(false)); + } + + @SuppressForbidden(reason = "don't care about the directory location in tests") + public void testFollowLinks() throws IOException { + Path baseSourceDir = Files.createTempDirectory("fileaccess_source"); + Path source1Dir = baseSourceDir.resolve("source1"); + Files.createDirectory(source1Dir); + Path source2Dir = baseSourceDir.resolve("source2"); + Files.createDirectory(source2Dir); + + Path baseTargetDir = Files.createTempDirectory("fileaccess_target"); + Path readTarget = baseTargetDir.resolve("read_link"); + Path writeTarget = baseTargetDir.resolve("write_link"); + Files.createSymbolicLink(readTarget, source1Dir); + Files.createSymbolicLink(writeTarget, source2Dir); + var tree = accessTree(entitlement(readTarget.toString(), "read", writeTarget.toString(), "read_write")); + + assertThat(tree.canRead(baseSourceDir), is(false)); + assertThat(tree.canRead(baseTargetDir), is(false)); + + assertThat(tree.canRead(readTarget), is(true)); + assertThat(tree.canWrite(readTarget), is(false)); + assertThat(tree.canRead(source1Dir), is(true)); + assertThat(tree.canWrite(source1Dir), is(false)); + + assertThat(tree.canRead(writeTarget), is(true)); + assertThat(tree.canWrite(writeTarget), is(true)); + assertThat(tree.canRead(source2Dir), is(true)); + assertThat(tree.canWrite(source2Dir), is(true)); + } + public void testTempDirAccess() { var tree = FileAccessTree.of(FilesEntitlement.EMPTY, TEST_PATH_LOOKUP); assertThat(tree.canRead(TEST_PATH_LOOKUP.tempDir()), is(true)); From d0db4cd085f830fa26391a267883ba2325a5e518 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 25 Feb 2025 12:50:43 +0000 Subject: [PATCH 10/31] Reduce licence checks in `LicensedWriteLoadForecaster` (#123346) Rather than checking the license (updating the usage map) on every single shard, just do it once at the start of a computation that needs to forecast write loads. Closes #123247 --- docs/changelog/123346.yaml | 6 ++ .../TransportGetDesiredBalanceAction.java | 1 + .../rollover/MetadataRolloverService.java | 1 + ...deAllocationStatsAndWeightsCalculator.java | 4 + .../allocation/WriteLoadForecaster.java | 5 ++ .../allocator/BalancedShardsAllocator.java | 5 ++ .../cluster/ESAllocationTestCase.java | 3 + .../WriteLoadForecasterIT.java | 2 + .../LicensedWriteLoadForecaster.java | 49 +++++++++++-- .../LicensedWriteLoadForecasterTests.java | 73 ++++++++++++++++++- 10 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/123346.yaml diff --git a/docs/changelog/123346.yaml b/docs/changelog/123346.yaml new file mode 100644 index 000000000000..42c6fbf6931a --- /dev/null +++ b/docs/changelog/123346.yaml @@ -0,0 +1,6 @@ +pr: 123346 +summary: Reduce license checks in `LicensedWriteLoadForecaster` +area: CRUD +type: bug +issues: + - 123247 diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java index 454fdad7cccc..d01fd702f79b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java @@ -92,6 +92,7 @@ public class TransportGetDesiredBalanceAction extends TransportMasterNodeReadAct return; } var clusterInfo = clusterInfoService.getClusterInfo(); + writeLoadForecaster.refreshLicense(); listener.onResponse( new DesiredBalanceResponse( desiredBalanceShardsAllocator.getStats(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 0c22a17bb1f6..63a5a792db3c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -430,6 +430,7 @@ public class MetadataRolloverService { ); } + writeLoadForecaster.refreshLicense(); metadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, metadataBuilder); metadataBuilder = withShardSizeForecastForWriteIndex(dataStreamName, metadataBuilder); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java index f793ab8f9eb7..d2a13b14bb4b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java @@ -70,6 +70,10 @@ public class NodeAllocationStatsAndWeightsCalculator { ClusterInfo clusterInfo, @Nullable DesiredBalance desiredBalance ) { + if (metadata.indices().isEmpty() == false) { + // must not use licensed features when just starting up + writeLoadForecaster.refreshLicense(); + } var weightFunction = new WeightFunction(shardBalanceFactor, indexBalanceFactor, writeLoadBalanceFactor, diskUsageBalanceFactor); var avgShardsPerNode = WeightFunction.avgShardPerNode(metadata, routingNodes); var avgWriteLoadPerNode = WeightFunction.avgWriteLoadPerNode(writeLoadForecaster, metadata, routingNodes); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java index e7ca51eee815..7bebedd9fdde 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadForecaster.java @@ -21,6 +21,8 @@ public interface WriteLoadForecaster { OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata); + void refreshLicense(); + class DefaultWriteLoadForecaster implements WriteLoadForecaster { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { @@ -31,5 +33,8 @@ public interface WriteLoadForecaster { public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return OptionalDouble.empty(); } + + @Override + public void refreshLicense() {} } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 0ce160523463..a39c4cce6c40 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -145,6 +145,11 @@ public class BalancedShardsAllocator implements ShardsAllocator { @Override public void allocate(RoutingAllocation allocation) { + if (allocation.metadata().indices().isEmpty() == false) { + // must not use licensed features when just starting up + writeLoadForecaster.refreshLicense(); + } + assert allocation.ignoreDisable() == false; if (allocation.routingNodes().size() == 0) { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index 132dd4b11946..1106aa198884 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -89,6 +89,9 @@ public abstract class ESAllocationTestCase extends ESTestCase { public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { return indexMetadata.getForecastedWriteLoad(); } + + @Override + public void refreshLicense() {} }; public static MockAllocationService createAllocationService() { diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index b37b026b853e..5c174d1bddef 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -84,6 +84,7 @@ public class WriteLoadForecasterIT extends ESIntegTestCase { assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); @@ -131,6 +132,7 @@ public class WriteLoadForecasterIT extends ESIntegTestCase { assertAllPreviousForecastsAreClearedAfterRollover(dataStream, metadata); setHasValidLicense(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndexMetadata); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(equalTo(false))); diff --git a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java index d4a85ce859b2..45c5abdc61fd 100644 --- a/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java +++ b/x-pack/plugin/write-load-forecaster/src/main/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecaster.java @@ -19,8 +19,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.threadpool.ThreadPool; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.List; import java.util.Objects; import java.util.OptionalDouble; @@ -30,6 +34,9 @@ import java.util.function.BooleanSupplier; import static org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterPlugin.OVERRIDE_WRITE_LOAD_FORECAST_SETTING; class LicensedWriteLoadForecaster implements WriteLoadForecaster { + + private static final Logger logger = LogManager.getLogger(LicensedWriteLoadForecaster.class); + public static final Setting MAX_INDEX_AGE_SETTING = Setting.timeSetting( "write_load_forecaster.max_index_age", TimeValue.timeValueDays(7), @@ -37,23 +44,26 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { Setting.Property.NodeScope, Setting.Property.Dynamic ); - private final BooleanSupplier hasValidLicense; + private final BooleanSupplier hasValidLicenseSupplier; private final ThreadPool threadPool; private volatile TimeValue maxIndexAge; + @SuppressWarnings("unused") // modified via VH_HAS_VALID_LICENSE_FIELD + private volatile boolean hasValidLicense; + LicensedWriteLoadForecaster( - BooleanSupplier hasValidLicense, + BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings ) { - this(hasValidLicense, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); + this(hasValidLicenseSupplier, threadPool, MAX_INDEX_AGE_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_INDEX_AGE_SETTING, this::setMaxIndexAgeSetting); } // exposed for tests only - LicensedWriteLoadForecaster(BooleanSupplier hasValidLicense, ThreadPool threadPool, TimeValue maxIndexAge) { - this.hasValidLicense = hasValidLicense; + LicensedWriteLoadForecaster(BooleanSupplier hasValidLicenseSupplier, ThreadPool threadPool, TimeValue maxIndexAge) { + this.hasValidLicenseSupplier = hasValidLicenseSupplier; this.threadPool = threadPool; this.maxIndexAge = maxIndexAge; } @@ -64,7 +74,7 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { @Override public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return metadata; } @@ -143,7 +153,7 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { @Override @SuppressForbidden(reason = "This is the only place where IndexMetadata#getForecastedWriteLoad is allowed to be used") public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) { - if (hasValidLicense.getAsBoolean() == false) { + if (hasValidLicense == false) { return OptionalDouble.empty(); } @@ -154,4 +164,29 @@ class LicensedWriteLoadForecaster implements WriteLoadForecaster { return indexMetadata.getForecastedWriteLoad(); } + + /** + * Used to atomically {@code getAndSet()} the {@link #hasValidLicense} field. This is better than an + * {@link java.util.concurrent.atomic.AtomicBoolean} because it takes one less pointer dereference on each read. + */ + private static final VarHandle VH_HAS_VALID_LICENSE_FIELD; + + static { + try { + VH_HAS_VALID_LICENSE_FIELD = MethodHandles.lookup() + .in(LicensedWriteLoadForecaster.class) + .findVarHandle(LicensedWriteLoadForecaster.class, "hasValidLicense", boolean.class); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override + public void refreshLicense() { + final var newValue = hasValidLicenseSupplier.getAsBoolean(); + final var oldValue = (boolean) VH_HAS_VALID_LICENSE_FIELD.getAndSet(this, newValue); + if (newValue != oldValue) { + logger.info("license state changed, now [{}]", newValue ? "valid" : "not valid"); + } + } } diff --git a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java index 790af0a20157..162e84b2562c 100644 --- a/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java +++ b/x-pack/plugin/write-load-forecaster/src/test/java/org/elasticsearch/xpack/writeloadforecaster/LicensedWriteLoadForecasterTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.writeloadforecaster; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; @@ -19,6 +21,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -30,9 +33,12 @@ import java.util.Map; import java.util.OptionalDouble; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.writeloadforecaster.LicensedWriteLoadForecaster.forecastIndexWriteLoad; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -53,7 +59,13 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { public void testWriteLoadForecastIsAddedToWriteIndex() { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); - final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + final AtomicInteger licenseCheckCount = new AtomicInteger(); + final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> { + licenseCheckCount.incrementAndGet(); + return hasValidLicense.get(); + }, threadPool, maxIndexAge); + + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -95,8 +107,12 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { assertThat(forecastedWriteLoad.isPresent(), is(true)); assertThat(forecastedWriteLoad.getAsDouble(), is(greaterThan(0.0))); + assertThat(licenseCheckCount.get(), equalTo(1)); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(licenseCheckCount.get(), equalTo(2)); + final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); } @@ -136,6 +152,7 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { metadataBuilder.put(dataStream); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(() -> true, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder updatedMetadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex( dataStream.getName(), @@ -154,6 +171,7 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { final TimeValue maxIndexAge = TimeValue.timeValueDays(7); final AtomicBoolean hasValidLicense = new AtomicBoolean(true); final WriteLoadForecaster writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, maxIndexAge); + writeLoadForecaster.refreshLicense(); final Metadata.Builder metadataBuilder = Metadata.builder(); final String dataStreamName = "logs-es"; @@ -197,6 +215,7 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { assertThat(forecastedWriteLoad.getAsDouble(), is(equalTo(0.6))); hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); final OptionalDouble forecastedWriteLoadAfterLicenseChange = writeLoadForecaster.getForecastedWriteLoad(writeIndex); assertThat(forecastedWriteLoadAfterLicenseChange.isPresent(), is(false)); @@ -327,4 +346,56 @@ public class LicensedWriteLoadForecasterTests extends ESTestCase { .setIndexMode(IndexMode.STANDARD) .build(); } + + public void testLicenseStateLogging() { + + final var seenMessages = new ArrayList(); + + final var collectingLoggingAssertion = new MockLog.SeenEventExpectation( + "seen event", + LicensedWriteLoadForecaster.class.getCanonicalName(), + Level.INFO, + "*" + ) { + @Override + public boolean innerMatch(LogEvent event) { + final var message = event.getMessage().getFormattedMessage(); + if (message.startsWith("license state changed, now [")) { + seenMessages.add(message); + return true; + } + + return false; + } + }; + + MockLog.assertThatLogger(() -> { + final var hasValidLicense = new AtomicBoolean(); + final var writeLoadForecaster = new LicensedWriteLoadForecaster(hasValidLicense::get, threadPool, randomTimeValue()); + assertThat(seenMessages, empty()); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, empty()); + + hasValidLicense.set(true); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]")); + + hasValidLicense.set(false); + writeLoadForecaster.refreshLicense(); + assertThat(seenMessages, contains("license state changed, now [valid]", "license state changed, now [not valid]")); + + hasValidLicense.set(true); + ESTestCase.startInParallel(between(1, 10), ignored -> writeLoadForecaster.refreshLicense()); + assertThat( + seenMessages, + contains( + "license state changed, now [valid]", + "license state changed, now [not valid]", + "license state changed, now [valid]" + ) + ); + }, LicensedWriteLoadForecaster.class, collectingLoggingAssertion); + } } From 2f0e1dabf7f59f058647f11aca7a5705e8d2aa70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 25 Feb 2025 13:55:25 +0100 Subject: [PATCH 11/31] Missing log4j policy (#123360) --- .../initialization/EntitlementInitialization.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 0fef2c27df66..d3d09e0f84ae 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -222,7 +222,10 @@ public class EntitlementInitialization { "org.apache.lucene.misc", List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))) ), - new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())), + new Scope( + "org.apache.logging.log4j.core", + List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE)))) + ), new Scope( "org.elasticsearch.nativeaccess", List.of( From 6c55099784b9f2953905541d987d94a5104c2ec8 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 25 Feb 2025 14:42:41 +0100 Subject: [PATCH 12/31] Store arrays offsets for ip fields natively with synthetic source (#122999) Follow up of #113757 and adds support to natively store array offsets for ip fields instead of falling back to ignored source. --- docs/changelog/122999.yaml | 5 + .../elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/FieldArrayContext.java | 47 ++- .../index/mapper/IpFieldMapper.java | 84 ++++- .../index/mapper/KeywordFieldMapper.java | 41 +-- ...etsDocValuesSyntheticFieldLoaderLayer.java | 32 +- ...eticSourceNativeArrayIntegrationTests.java | 91 ++++++ .../index/mapper/IpFieldMapperTests.java | 5 + .../index/mapper/IpFieldTypeTests.java | 20 +- .../mapper/IpOffsetDocValuesLoaderTests.java | 41 +++ .../KeywordOffsetDocValuesLoaderTests.java | 215 +------------ ...eticSourceNativeArrayIntegrationTests.java | 283 +---------------- .../NativeArrayIntegrationTestCase.java | 299 ++++++++++++++++++ .../mapper/OffsetDocValuesLoaderTestCase.java | 230 ++++++++++++++ 14 files changed, 858 insertions(+), 536 deletions(-) create mode 100644 docs/changelog/122999.yaml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java diff --git a/docs/changelog/122999.yaml b/docs/changelog/122999.yaml new file mode 100644 index 000000000000..a0134afc59a0 --- /dev/null +++ b/docs/changelog/122999.yaml @@ -0,0 +1,5 @@ +pr: 122999 +summary: Store arrays offsets for ip fields natively with synthetic source +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 2d1fe3133739..69c9d050d592 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -150,6 +150,7 @@ public class IndexVersions { public static final IndexVersion TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_011_0_00, Version.LUCENE_10_1_0); public static final IndexVersion TIME_SERIES_ID_DOC_VALUES_SPARSE_INDEX = def(9_012_0_00, Version.LUCENE_10_1_0); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(9_013_0_00, Version.LUCENE_10_1_0); + public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java index 523ac19524ee..ca293be5f511 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java @@ -13,6 +13,8 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.util.BitUtil; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import java.io.IOException; import java.util.ArrayList; @@ -23,9 +25,10 @@ import java.util.TreeMap; public class FieldArrayContext { + private static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets"; private final Map offsetsPerField = new HashMap<>(); - void recordOffset(String field, String value) { + void recordOffset(String field, Comparable value) { Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets()); int nextOffset = arrayOffsets.currentOffset++; var offsets = arrayOffsets.valueToOffsets.computeIfAbsent(value, s -> new ArrayList<>(2)); @@ -79,13 +82,53 @@ public class FieldArrayContext { return offsetToOrd; } + static String getOffsetsFieldName( + MapperBuilderContext context, + Mapper.SourceKeepMode indexSourceKeepMode, + boolean hasDocValues, + boolean isStored, + FieldMapper.Builder fieldMapperBuilder, + IndexVersion indexCreatedVersion, + IndexVersion minSupportedVersionMain + ) { + var sourceKeepMode = fieldMapperBuilder.sourceKeepMode.orElse(indexSourceKeepMode); + if (context.isSourceSynthetic() + && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS + && hasDocValues + && isStored == false + && fieldMapperBuilder.copyTo.copyToFields().isEmpty() + && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false + && indexVersionSupportStoringArraysNatively(indexCreatedVersion, minSupportedVersionMain)) { + // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets + // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and + // logging use cases + + // keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing + // (if field is stored then there is no point of doing this) + return context.buildFullName(fieldMapperBuilder.leafName() + FieldArrayContext.OFFSETS_FIELD_NAME_SUFFIX); + } else { + return null; + } + } + + private static boolean indexVersionSupportStoringArraysNatively( + IndexVersion indexCreatedVersion, + IndexVersion minSupportedVersionMain + ) { + return indexCreatedVersion.onOrAfter(minSupportedVersionMain) + || indexCreatedVersion.between( + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD_BACKPORT_8_X, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + ); + } + private static class Offsets { int currentOffset; // Need to use TreeMap here, so that we maintain the order in which each value (with offset) stored inserted, // (which is in the same order the document gets parsed) so we store offsets in right order. This is the same // order in what the values get stored in SortedSetDocValues. - final Map> valueToOffsets = new TreeMap<>(); + final Map, List> valueToOffsets = new TreeMap<>(); final List nullValueOffsets = new ArrayList<>(2); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 3ded3d2699b2..8e6224e5720c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -55,6 +55,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiFunction; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; import static org.elasticsearch.index.mapper.IpPrefixAutomatonUtil.buildIpPrefixAutomaton; /** @@ -92,8 +93,15 @@ public class IpFieldMapper extends FieldMapper { private final boolean ignoreMalformedByDefault; private final IndexVersion indexCreatedVersion; private final ScriptCompiler scriptCompiler; + private final SourceKeepMode indexSourceKeepMode; - public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) { + public Builder( + String name, + ScriptCompiler scriptCompiler, + boolean ignoreMalformedByDefault, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { super(name); this.scriptCompiler = Objects.requireNonNull(scriptCompiler); this.ignoreMalformedByDefault = ignoreMalformedByDefault; @@ -114,6 +122,7 @@ public class IpFieldMapper extends FieldMapper { ); } }); + this.indexSourceKeepMode = indexSourceKeepMode; } Builder nullValue(String nullValue) { @@ -184,6 +193,16 @@ public class IpFieldMapper extends FieldMapper { } hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); + + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + hasDocValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP + ); return new IpFieldMapper( leafName(), new IpFieldType( @@ -198,7 +217,8 @@ public class IpFieldMapper extends FieldMapper { ), builderParams(this, context), context.isSourceSynthetic(), - this + this, + offsetsFieldName ); } @@ -206,7 +226,7 @@ public class IpFieldMapper extends FieldMapper { public static final TypeParser PARSER = createTypeParserWithLegacySupport((n, c) -> { boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings()); - return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated()); + return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated(), c.getIndexSettings().sourceKeepMode()); }); public static final class IpFieldType extends SimpleMappedFieldType { @@ -501,13 +521,16 @@ public class IpFieldMapper extends FieldMapper { private final Script script; private final FieldValues scriptValues; private final ScriptCompiler scriptCompiler; + private final SourceKeepMode indexSourceKeepMode; + private final String offsetsFieldName; private IpFieldMapper( String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, boolean storeIgnored, - Builder builder + Builder builder, + String offsetsFieldName ) { super(simpleName, mappedFieldType, builderParams); this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault; @@ -523,6 +546,8 @@ public class IpFieldMapper extends FieldMapper { this.scriptCompiler = builder.scriptCompiler; this.dimension = builder.dimension.getValue(); this.storeIgnored = storeIgnored; + this.indexSourceKeepMode = builder.indexSourceKeepMode; + this.offsetsFieldName = offsetsFieldName; } @Override @@ -561,6 +586,14 @@ public class IpFieldMapper extends FieldMapper { if (address != null) { indexValue(context, address); } + if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) { + if (address != null) { + BytesRef sortableValue = new BytesRef(InetAddressPoint.encode(address)); + context.getOffSetContext().recordOffset(offsetsFieldName, sortableValue); + } else { + context.getOffSetContext().recordNull(offsetsFieldName); + } + } } private void indexValue(DocumentParserContext context, InetAddress address) { @@ -593,7 +626,9 @@ public class IpFieldMapper extends FieldMapper { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension( + dimension + ).init(this); } @Override @@ -610,19 +645,24 @@ public class IpFieldMapper extends FieldMapper { if (hasDocValues) { return new SyntheticSourceSupport.Native(() -> { var layers = new ArrayList(); - layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { - @Override - protected BytesRef convert(BytesRef value) { - byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); - return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); - } + if (offsetsFieldName != null) { + layers.add( + new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, IpFieldMapper::convert) + ); + } else { + layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { + @Override + protected BytesRef convert(BytesRef value) { + return IpFieldMapper.convert(value); + } - @Override - protected BytesRef preserve(BytesRef value) { - // No need to copy because convert has made a deep copy - return value; - } - }); + @Override + protected BytesRef preserve(BytesRef value) { + // No need to copy because convert has made a deep copy + return value; + } + }); + } if (ignoreMalformed) { layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); @@ -633,4 +673,14 @@ public class IpFieldMapper extends FieldMapper { return super.syntheticSourceSupport(); } + + static BytesRef convert(BytesRef value) { + byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); + return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); + } + + @Override + public String getOffsetFieldName() { + return offsetsFieldName; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 195ec5a27a72..b7e8e13e1683 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -85,6 +85,7 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -95,7 +96,6 @@ public final class KeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "keyword"; private static final String HOST_NAME = "host.name"; - public static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets"; public static class Defaults { public static final FieldType FIELD_TYPE; @@ -439,26 +439,15 @@ public final class KeywordFieldMapper extends FieldMapper { super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); - var sourceKeepMode = this.sourceKeepMode.orElse(indexSourceKeepMode); - String offsetsFieldName; - if (context.isSourceSynthetic() - && sourceKeepMode == SourceKeepMode.ARRAYS - && hasDocValues() - && fieldtype.stored() == false - && copyTo.copyToFields().isEmpty() - && multiFieldsBuilder.hasMultiFields() == false - && indexVersionSupportStoringArraysNatively()) { - // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets - // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and - // logging use cases - - // keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing - // (if field is stored then there is no point of doing this) - offsetsFieldName = context.buildFullName(leafName() + OFFSETS_FIELD_NAME_SUFFIX); - } else { - offsetsFieldName = null; - } - + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + hasDocValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD + ); return new KeywordFieldMapper( leafName(), fieldtype, @@ -472,14 +461,6 @@ public final class KeywordFieldMapper extends FieldMapper { ); } - private boolean indexVersionSupportStoringArraysNatively() { - return indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD) - || indexCreatedVersion.between( - IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD_BACKPORT_8_X, - IndexVersions.UPGRADE_TO_LUCENE_10_0_0 - ); - } - private FieldType resolveFieldType( final boolean useDocValuesSkipper, final IndexVersion indexCreatedVersion, @@ -1127,7 +1108,7 @@ public final class KeywordFieldMapper extends FieldMapper { } boolean indexed = indexValue(context, value); - if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.getRecordedSource() == false) { + if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) { if (indexed) { context.getOffSetContext().recordOffset(offsetsFieldName, value); } else if (value == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.java index 09a63eb6ab4a..e04cdd06280d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.java @@ -20,6 +20,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; +import java.util.function.Function; /** * Load {@code _source} fields from {@link SortedSetDocValues} and associated {@link BinaryDocValues}. The former contains the unique values @@ -30,11 +31,29 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co private final String name; private final String offsetsFieldName; + private final Function converter; private DocValuesWithOffsetsLoader docValues; + /** + * @param name The name of the field to synthesize + * @param offsetsFieldName The related offset field used to correctly synthesize the field if it is a leaf array + */ SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(String name, String offsetsFieldName) { + this(name, offsetsFieldName, Function.identity()); + } + + /** + * @param name The name of the field to synthesize + * @param offsetsFieldName The related offset field used to correctly synthesize the field if it is a leaf array + * @param converter This field value loader layer synthesizes the values read from doc values as utf8 string. If the doc value + * values aren't serializable as utf8 string then it is the responsibility of the converter to covert into a + * format that can be serialized as utf8 string. For example IP field mapper doc values can't directly be + * serialized as utf8 string. + */ + SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(String name, String offsetsFieldName, Function converter) { this.name = Objects.requireNonNull(name); this.offsetsFieldName = Objects.requireNonNull(offsetsFieldName); + this.converter = Objects.requireNonNull(converter); } @Override @@ -47,7 +66,7 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co SortedSetDocValues valueDocValues = DocValues.getSortedSet(leafReader, name); SortedDocValues offsetDocValues = DocValues.getSorted(leafReader, offsetsFieldName); - return docValues = new DocValuesWithOffsetsLoader(valueDocValues, offsetDocValues); + return docValues = new DocValuesWithOffsetsLoader(valueDocValues, offsetDocValues, converter); } @Override @@ -78,15 +97,21 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co static final class DocValuesWithOffsetsLoader implements DocValuesLoader { private final SortedDocValues offsetDocValues; private final SortedSetDocValues valueDocValues; + private final Function converter; private final ByteArrayStreamInput scratch = new ByteArrayStreamInput(); private boolean hasValue; private boolean hasOffset; private int[] offsetToOrd; - DocValuesWithOffsetsLoader(SortedSetDocValues valueDocValues, SortedDocValues offsetDocValues) { + DocValuesWithOffsetsLoader( + SortedSetDocValues valueDocValues, + SortedDocValues offsetDocValues, + Function converter + ) { this.valueDocValues = valueDocValues; this.offsetDocValues = offsetDocValues; + this.converter = converter; } @Override @@ -146,7 +171,7 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co long ord = ords[offset]; BytesRef c = valueDocValues.lookupOrd(ord); - // This is keyword specific and needs to be updated once support is added for other field types: + c = converter.apply(c); b.utf8Value(c.bytes, c.offset, c.length); } } else if (offsetToOrd != null) { @@ -158,6 +183,7 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co } else { for (int i = 0; i < valueDocValues.docValueCount(); i++) { BytesRef c = valueDocValues.lookupOrd(valueDocValues.nextOrd()); + c = converter.apply(c); b.utf8Value(c.bytes, c.offset, c.length); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 000000000000..2ad08ebb10aa --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.network.NetworkAddress; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + +public class IPSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "ip"; + } + + @Override + protected String getRandomValue() { + return NetworkAddress.format(randomIp(true)); + } + + public void testSynthesizeArray() throws Exception { + var arrayValues = new Object[][] { + new Object[] { "192.168.1.4", "192.168.1.3", null, "192.168.1.2", null, "192.168.1.1" }, + new Object[] { null, "192.168.1.2", null, "192.168.1.1" }, + new Object[] { null }, + new Object[] { null, null, null }, + new Object[] { "192.168.1.3", "192.168.1.2", "192.168.1.1" } }; + verifySyntheticArray(arrayValues); + } + + public void testSynthesizeArrayIgnoreMalformed() throws Exception { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", "ip") + .field("ignore_malformed", true) + .endObject() + .endObject() + .endObject(); + // Note values that would be ignored are added at the end of arrays, + // this makes testing easier as ignored values are always synthesized after regular values: + var arrayValues = new Object[][] { + new Object[] { null, "192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4", null, "malformed" }, + new Object[] { "192.168.1.1", "192.168.1.2", "malformed" }, + new Object[] { "192.168.1.1", "192.168.1.1", "malformed" }, + new Object[] { null, null, null, "malformed" }, + new Object[] { "192.168.1.3", "192.168.1.3", "192.168.1.1", "malformed" } }; + verifySyntheticArray(arrayValues, mapping, "_id", "field._ignore_malformed"); + } + + public void testSynthesizeObjectArray() throws Exception { + List> documents = new ArrayList<>(); + { + List document = new ArrayList<>(); + document.add(new Object[] { "192.168.1.3", "192.168.1.2", "192.168.1.1" }); + document.add(new Object[] { "192.168.1.110", "192.168.1.109", "192.168.1.111" }); + document.add(new Object[] { "192.168.1.2", "192.168.1.2", "192.168.1.1" }); + documents.add(document); + } + { + List document = new ArrayList<>(); + document.add(new Object[] { "192.168.1.9", "192.168.1.7", "192.168.1.5" }); + document.add(new Object[] { "192.168.1.2", "192.168.1.4", "192.168.1.6" }); + document.add(new Object[] { "192.168.1.7", "192.168.1.6", "192.168.1.5" }); + documents.add(document); + } + verifySyntheticObjectArray(documents); + } + + public void testSynthesizeArrayInObjectField() throws Exception { + List documents = new ArrayList<>(); + documents.add(new Object[] { "192.168.1.254", "192.168.1.253", "192.168.1.252" }); + documents.add(new Object[] { "192.168.1.112", "192.168.1.113", "192.168.1.114" }); + documents.add(new Object[] { "192.168.1.3", "192.168.1.2", "192.168.1.1" }); + documents.add(new Object[] { "192.168.1.9", "192.168.1.7", "192.168.1.5" }); + documents.add(new Object[] { "192.168.1.2", "192.168.1.4", "192.168.1.6" }); + documents.add(new Object[] { "192.168.1.7", "192.168.1.6", "192.168.1.5" }); + verifySyntheticArrayInObject(documents); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 1b8a2d68cd93..17ecc2c22db2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -439,4 +439,9 @@ public class IpFieldMapperTests extends MapperTestCase { protected Function loadBlockExpected() { return v -> InetAddresses.toAddrString(InetAddressPoint.decode(BytesRef.deepCopyOf((BytesRef) v).bytes)); } + + @Override + protected String randomSyntheticSourceKeep() { + return "all"; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java index b3064810e5ca..7dec761691ce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java @@ -349,16 +349,24 @@ public class IpFieldTypeTests extends FieldTypeTestCase { } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = new IpFieldMapper.Builder("field", ScriptCompiler.NONE, true, IndexVersion.current()).build( - MapperBuilderContext.root(false, false) - ).fieldType(); + MappedFieldType mapper = new IpFieldMapper.Builder( + "field", + ScriptCompiler.NONE, + true, + IndexVersion.current(), + Mapper.SourceKeepMode.NONE + ).build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1")); assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1")); assertEquals(List.of("::1"), fetchSourceValue(mapper, "0:0:0:0:0:0:0:1")); - MappedFieldType nullValueMapper = new IpFieldMapper.Builder("field", ScriptCompiler.NONE, true, IndexVersion.current()).nullValue( - "2001:db8:0:0:0:0:2:7" - ).build(MapperBuilderContext.root(false, false)).fieldType(); + MappedFieldType nullValueMapper = new IpFieldMapper.Builder( + "field", + ScriptCompiler.NONE, + true, + IndexVersion.current(), + Mapper.SourceKeepMode.NONE + ).nullValue("2001:db8:0:0:0:0:2:7").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java new file mode 100644 index 000000000000..dadfd22199ae --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.network.NetworkAddress; + +public class IpOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + public void testOffsetArray() throws Exception { + verifyOffsets("{\"field\":[\"192.168.1.1\",\"192.168.1.3\",\"192.168.1.2\",\"192.168.1.1\",\"192.168.1.9\",\"192.168.1.3\"]}"); + verifyOffsets("{\"field\":[\"192.168.1.4\",null,\"192.168.1.3\",\"192.168.1.2\",null,\"192.168.1.1\"]}"); + } + + public void testOffsetNestedArray() throws Exception { + verifyOffsets( + "{\"field\":[\"192.168.1.2\",[\"192.168.1.1\"],[\"192.168.1.0\"],null,\"192.168.1.0\"]}", + "{\"field\":[\"192.168.1.2\",\"192.168.1.1\",\"192.168.1.0\",null,\"192.168.1.0\"]}" + ); + verifyOffsets( + "{\"field\":[\"192.168.1.6\",[\"192.168.1.5\", [\"192.168.1.4\"]],[\"192.168.1.3\", [\"192.168.1.2\"]],null,\"192.168.1.1\"]}", + "{\"field\":[\"192.168.1.6\",\"192.168.1.5\",\"192.168.1.4\",\"192.168.1.3\",\"192.168.1.2\",null,\"192.168.1.1\"]}" + ); + } + + @Override + protected String getFieldTypeName() { + return "ip"; + } + + @Override + protected String randomValue() { + return NetworkAddress.format(randomIp(true)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java index 8300e8e8e461..55e935e11996 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java @@ -9,145 +9,7 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.DirectoryReader; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.DocValuesWithOffsetsLoader; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; - -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.nullValue; - -public class KeywordOffsetDocValuesLoaderTests extends MapperServiceTestCase { - - @Override - protected Settings getIndexSettings() { - return Settings.builder() - .put("index.mapping.source.mode", "synthetic") - .put("index.mapping.synthetic_source_keep", "arrays") - .build(); - } - - public void testOffsetArrayNoDocValues() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword", - "doc_values": false - } - } - } - } - """; - try (var mapperService = createMapperService(mapping)) { - var fieldMapper = mapperService.mappingLookup().getMapper("field"); - assertThat(fieldMapper.getOffsetFieldName(), nullValue()); - } - } - - public void testOffsetArrayStored() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword", - "store": true - } - } - } - } - """; - try (var mapperService = createMapperService(mapping)) { - var fieldMapper = mapperService.mappingLookup().getMapper("field"); - assertThat(fieldMapper.getOffsetFieldName(), nullValue()); - } - } - - public void testOffsetMultiFields() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword", - "fields": { - "sub": { - "type": "text" - } - } - } - } - } - } - """; - try (var mapperService = createMapperService(mapping)) { - var fieldMapper = mapperService.mappingLookup().getMapper("field"); - assertThat(fieldMapper.getOffsetFieldName(), nullValue()); - } - } - - public void testOffsetArrayNoSyntheticSource() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword" - } - } - } - } - """; - try (var mapperService = createMapperService(Settings.EMPTY, mapping)) { - var fieldMapper = mapperService.mappingLookup().getMapper("field"); - assertThat(fieldMapper.getOffsetFieldName(), nullValue()); - } - } - - public void testOffsetArrayNoSourceArrayKeep() throws Exception { - var settingsBuilder = Settings.builder().put("index.mapping.source.mode", "synthetic"); - String mapping; - if (randomBoolean()) { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword", - "synthetic_source_keep": "{{synthetic_source_keep}}" - } - } - } - } - """.replace("{{synthetic_source_keep}}", randomBoolean() ? "none" : "all"); - } else { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword" - } - } - } - } - """; - if (randomBoolean()) { - settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); - } - } - try (var mapperService = createMapperService(settingsBuilder.build(), mapping)) { - var fieldMapper = mapperService.mappingLookup().getMapper("field"); - assertThat(fieldMapper.getOffsetFieldName(), nullValue()); - } - } +public class KeywordOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { public void testOffsetArray() throws Exception { verifyOffsets("{\"field\":[\"z\",\"x\",\"y\",\"c\",\"b\",\"a\"]}"); @@ -162,76 +24,13 @@ public class KeywordOffsetDocValuesLoaderTests extends MapperServiceTestCase { ); } - public void testOffsetEmptyArray() throws Exception { - verifyOffsets("{\"field\":[]}"); + @Override + protected String getFieldTypeName() { + return "keyword"; } - public void testOffsetArrayWithNulls() throws Exception { - verifyOffsets("{\"field\":[null,null,null]}"); + @Override + protected String randomValue() { + return randomAlphanumericOfLength(2); } - - public void testOffsetArrayRandom() throws Exception { - StringBuilder values = new StringBuilder(); - int numValues = randomIntBetween(0, 256); - for (int i = 0; i < numValues; i++) { - if (randomInt(10) == 1) { - values.append("null"); - } else { - values.append('"').append(randomAlphanumericOfLength(2)).append('"'); - } - if (i != (numValues - 1)) { - values.append(','); - } - } - verifyOffsets("{\"field\":[" + values + "]}"); - } - - private void verifyOffsets(String source) throws IOException { - verifyOffsets(source, source); - } - - private void verifyOffsets(String source, String expectedSource) throws IOException { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "keyword" - } - } - } - } - """; - verifyOffsets(mapping, source, expectedSource); - } - - private void verifyOffsets(String mapping, String source, String expectedSource) throws IOException { - try (var mapperService = createMapperService(mapping)) { - var mapper = mapperService.documentMapper(); - - try (var directory = newDirectory()) { - var iw = indexWriterForSyntheticSource(directory); - var doc = mapper.parse(new SourceToParse("_id", new BytesArray(source), XContentType.JSON)); - doc.updateSeqID(0, 0); - doc.version().setLongValue(0); - iw.addDocuments(doc.docs()); - iw.close(); - try (var indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { - var layer = new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer("field", "field.offsets"); - var leafReader = indexReader.leaves().getFirst().reader(); - var loader = (DocValuesWithOffsetsLoader) layer.docValuesLoader(leafReader, new int[] { 0 }); - assertTrue(loader.advanceToDoc(0)); - assertTrue(loader.count() > 0); - XContentBuilder builder = jsonBuilder().startObject(); - builder.startArray("field"); - loader.write(builder); - builder.endArray().endObject(); - - var actual = Strings.toString(builder); - assertEquals(expectedSource, actual); - } - } - } - } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java index f0853c34b209..6f59f617ba25 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java @@ -9,38 +9,24 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.query.IdsQueryBuilder; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.hamcrest.Matchers; +import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Set; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.nullValue; -public class KeywordSyntheticSourceNativeArrayIntegrationTests extends ESSingleNodeTestCase { +public class KeywordSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "keyword"; + } + + @Override + protected String getRandomValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } public void testSynthesizeArray() throws Exception { var arrayValues = new Object[][] { @@ -52,16 +38,6 @@ public class KeywordSyntheticSourceNativeArrayIntegrationTests extends ESSingleN verifySyntheticArray(arrayValues); } - public void testSynthesizeEmptyArray() throws Exception { - var arrayValues = new Object[][] { new Object[] {} }; - verifySyntheticArray(arrayValues); - } - - public void testSynthesizeArrayRandom() throws Exception { - var arrayValues = new Object[][] { generateRandomStringArray(64, 8, false, true) }; - verifySyntheticArray(arrayValues); - } - public void testSynthesizeArrayIgnoreAbove() throws Exception { var mapping = jsonBuilder().startObject() .startObject("properties") @@ -79,7 +55,7 @@ public class KeywordSyntheticSourceNativeArrayIntegrationTests extends ESSingleN new Object[] { "123", "1234", "12345" }, new Object[] { null, null, null, "blabla" }, new Object[] { "1", "2", "3", "blabla" } }; - verifySyntheticArray(arrayValues, mapping, 4, "_id", "field._original"); + verifySyntheticArray(arrayValues, mapping, "_id", "field._original"); } public void testSynthesizeObjectArray() throws Exception { @@ -112,237 +88,4 @@ public class KeywordSyntheticSourceNativeArrayIntegrationTests extends ESSingleN verifySyntheticArrayInObject(documents); } - public void testSynthesizeArrayInObjectFieldRandom() throws Exception { - List documents = new ArrayList<>(); - int numDocs = randomIntBetween(8, 256); - for (int i = 0; i < numDocs; i++) { - documents.add(generateRandomStringArray(64, 8, false, true)); - } - verifySyntheticArrayInObject(documents); - } - - private void verifySyntheticArray(Object[][] arrays) throws IOException { - var mapping = jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", "keyword") - .endObject() - .endObject() - .endObject(); - verifySyntheticArray(arrays, mapping, null, "_id"); - } - - private void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, Integer ignoreAbove, String... expectedStoredFields) - throws IOException { - var indexService = createIndex( - "test-index", - Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - mapping - ); - for (int i = 0; i < arrays.length; i++) { - var array = arrays[i]; - - var indexRequest = new IndexRequest("test-index"); - indexRequest.id("my-id-" + i); - var source = jsonBuilder().startObject(); - if (array != null) { - source.startArray("field"); - for (Object arrayValue : array) { - source.value(arrayValue); - } - source.endArray(); - } else { - source.field("field").nullValue(); - } - indexRequest.source(source.endObject()); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client().index(indexRequest).actionGet(); - - var searchRequest = new SearchRequest("test-index"); - searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); - var searchResponse = client().search(searchRequest).actionGet(); - try { - var hit = searchResponse.getHits().getHits()[0]; - assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - assertThat(sourceAsMap, hasKey("field")); - var actualArray = (List) sourceAsMap.get("field"); - if (array == null) { - assertThat(actualArray, nullValue()); - } else if (array.length == 0) { - assertThat(actualArray, empty()); - } else { - assertThat(actualArray, Matchers.contains(array)); - } - } finally { - searchResponse.decRef(); - } - } - - try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { - var reader = searcher.getDirectoryReader(); - for (int i = 0; i < arrays.length; i++) { - var document = reader.storedFields().document(i); - // Verify that there is no ignored source: - Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains(expectedStoredFields)); - } else { - String[] copyExpectedStoredFields = new String[expectedStoredFields.length + 1]; - System.arraycopy(expectedStoredFields, 0, copyExpectedStoredFields, 0, expectedStoredFields.length); - copyExpectedStoredFields[copyExpectedStoredFields.length - 1] = "_recovery_source"; - assertThat(storedFieldNames, containsInAnyOrder(copyExpectedStoredFields)); - } - } - var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("field.offsets"); - assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); - } - } - - private void verifySyntheticObjectArray(List> documents) throws IOException { - var indexService = createIndex( - "test-index", - Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); - for (int i = 0; i < documents.size(); i++) { - var document = documents.get(i); - - var indexRequest = new IndexRequest("test-index"); - indexRequest.id("my-id-" + i); - var source = jsonBuilder().startObject(); - source.startArray("object"); - for (Object[] arrayValue : document) { - source.startObject(); - source.array("field", arrayValue); - source.endObject(); - } - source.endArray(); - indexRequest.source(source.endObject()); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client().index(indexRequest).actionGet(); - - var searchRequest = new SearchRequest("test-index"); - searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); - var searchResponse = client().search(searchRequest).actionGet(); - try { - var hit = searchResponse.getHits().getHits()[0]; - assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - var objectArray = (List) sourceAsMap.get("object"); - for (int j = 0; j < document.size(); j++) { - var expected = document.get(j); - List actual = (List) ((Map) objectArray.get(j)).get("field"); - assertThat(actual, Matchers.contains(expected)); - } - } finally { - searchResponse.decRef(); - } - } - - indexService.getShard(0).forceMerge(new ForceMergeRequest("test-index").maxNumSegments(1)); - try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { - var reader = searcher.getDirectoryReader(); - for (int i = 0; i < documents.size(); i++) { - var document = reader.storedFields().document(i); - // Verify that there is ignored source because of leaf array being wrapped by object array: - List storedFieldNames = document.getFields().stream().map(IndexableField::name).toList(); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains("_id", "_ignored_source")); - } else { - assertThat(storedFieldNames, containsInAnyOrder("_id", "_ignored_source", "_recovery_source")); - } - - // Verify that there is no offset field: - LeafReader leafReader = reader.leaves().get(0).reader(); - for (FieldInfo fieldInfo : leafReader.getFieldInfos()) { - String name = fieldInfo.getName(); - assertFalse("expected no field that contains [offsets] in name, but found [" + name + "]", name.contains("offsets")); - } - - var binaryDocValues = leafReader.getBinaryDocValues("object.field.offsets"); - assertThat(binaryDocValues, nullValue()); - } - } - } - - private void verifySyntheticArrayInObject(List documents) throws IOException { - var indexService = createIndex( - "test-index", - Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); - for (int i = 0; i < documents.size(); i++) { - var arrayValue = documents.get(i); - - var indexRequest = new IndexRequest("test-index"); - indexRequest.id("my-id-" + i); - var source = jsonBuilder().startObject(); - source.startObject("object"); - source.array("field", arrayValue); - source.endObject(); - indexRequest.source(source.endObject()); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client().index(indexRequest).actionGet(); - - var searchRequest = new SearchRequest("test-index"); - searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); - var searchResponse = client().search(searchRequest).actionGet(); - try { - var hit = searchResponse.getHits().getHits()[0]; - assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - var objectArray = (Map) sourceAsMap.get("object"); - - List actual = (List) objectArray.get("field"); - if (arrayValue == null) { - assertThat(actual, nullValue()); - } else if (arrayValue.length == 0) { - assertThat(actual, empty()); - } else { - assertThat(actual, Matchers.contains(arrayValue)); - } - } finally { - searchResponse.decRef(); - } - } - - indexService.getShard(0).forceMerge(new ForceMergeRequest("test-index").maxNumSegments(1)); - try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { - var reader = searcher.getDirectoryReader(); - for (int i = 0; i < documents.size(); i++) { - var document = reader.storedFields().document(i); - // Verify that there is no ignored source: - Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); - if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { - assertThat(storedFieldNames, contains("_id")); - } else { - assertThat(storedFieldNames, containsInAnyOrder("_id", "_recovery_source")); - } - } - var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("object.field.offsets"); - assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); - } - } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java new file mode 100644 index 000000000000..e074bf883ae1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -0,0 +1,299 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.IdsQueryBuilder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.nullValue; + +public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase { + + public void testSynthesizeEmptyArray() throws Exception { + var arrayValues = new Object[][] { new Object[] {} }; + verifySyntheticArray(arrayValues); + } + + public void testSynthesizeArrayRandom() throws Exception { + var arrayValues = new Object[randomInt(64)]; + for (int j = 0; j < arrayValues.length; j++) { + arrayValues[j] = NetworkAddress.format(randomIp(true)); + } + verifySyntheticArray(new Object[][] { arrayValues }); + } + + public void testSynthesizeArrayInObjectFieldRandom() throws Exception { + List documents = new ArrayList<>(); + int numDocs = randomIntBetween(8, 256); + for (int i = 0; i < numDocs; i++) { + Object[] document = new Object[randomInt(64)]; + for (int j = 0; j < document.length; j++) { + document[j] = getRandomValue(); + } + documents.add(document); + } + verifySyntheticArrayInObject(documents); + } + + protected abstract String getFieldTypeName(); + + protected abstract String getRandomValue(); + + protected void verifySyntheticArray(Object[][] arrays) throws IOException { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .endObject() + .endObject() + .endObject(); + verifySyntheticArray(arrays, mapping, "_id"); + } + + protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, String... expectedStoredFields) throws IOException { + var indexService = createIndex( + "test-index", + Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), + mapping + ); + for (int i = 0; i < arrays.length; i++) { + var array = arrays[i]; + + var indexRequest = new IndexRequest("test-index"); + indexRequest.id("my-id-" + i); + var source = jsonBuilder().startObject(); + if (array != null) { + source.startArray("field"); + for (Object arrayValue : array) { + source.value(arrayValue); + } + source.endArray(); + } else { + source.field("field").nullValue(); + } + indexRequest.source(source.endObject()); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + + var searchRequest = new SearchRequest("test-index"); + searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); + var searchResponse = client().search(searchRequest).actionGet(); + try { + var hit = searchResponse.getHits().getHits()[0]; + assertThat(hit.getId(), equalTo("my-id-" + i)); + var sourceAsMap = hit.getSourceAsMap(); + assertThat(sourceAsMap, hasKey("field")); + var actualArray = (List) sourceAsMap.get("field"); + if (array == null) { + assertThat(actualArray, nullValue()); + } else if (array.length == 0) { + assertThat(actualArray, empty()); + } else { + assertThat(actualArray, Matchers.contains(array)); + } + } finally { + searchResponse.decRef(); + } + } + + try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { + var reader = searcher.getDirectoryReader(); + for (int i = 0; i < arrays.length; i++) { + var document = reader.storedFields().document(i); + // Verify that there is no ignored source: + Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { + assertThat(storedFieldNames, contains(expectedStoredFields)); + } else { + var copyExpectedStoredFields = new String[expectedStoredFields.length + 1]; + System.arraycopy(expectedStoredFields, 0, copyExpectedStoredFields, 0, expectedStoredFields.length); + copyExpectedStoredFields[copyExpectedStoredFields.length - 1] = "_ignored_source"; + assertThat(storedFieldNames, containsInAnyOrder(copyExpectedStoredFields)); + } + } + var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("field.offsets"); + assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); + } + } + + protected void verifySyntheticObjectArray(List> documents) throws IOException { + var indexService = createIndex( + "test-index", + Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), + jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ); + for (int i = 0; i < documents.size(); i++) { + var document = documents.get(i); + + var indexRequest = new IndexRequest("test-index"); + indexRequest.id("my-id-" + i); + var source = jsonBuilder().startObject(); + source.startArray("object"); + for (Object[] arrayValue : document) { + source.startObject(); + source.array("field", arrayValue); + source.endObject(); + } + source.endArray(); + indexRequest.source(source.endObject()); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + + var searchRequest = new SearchRequest("test-index"); + searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); + var searchResponse = client().search(searchRequest).actionGet(); + try { + var hit = searchResponse.getHits().getHits()[0]; + assertThat(hit.getId(), equalTo("my-id-" + i)); + var sourceAsMap = hit.getSourceAsMap(); + var objectArray = (List) sourceAsMap.get("object"); + for (int j = 0; j < document.size(); j++) { + var expected = document.get(j); + List actual = (List) ((Map) objectArray.get(j)).get("field"); + assertThat(actual, Matchers.contains(expected)); + } + } finally { + searchResponse.decRef(); + } + } + + indexService.getShard(0).forceMerge(new ForceMergeRequest("test-index").maxNumSegments(1)); + try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { + var reader = searcher.getDirectoryReader(); + for (int i = 0; i < documents.size(); i++) { + var document = reader.storedFields().document(i); + // Verify that there is ignored source because of leaf array being wrapped by object array: + List storedFieldNames = document.getFields().stream().map(IndexableField::name).toList(); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { + assertThat(storedFieldNames, contains("_id", "_ignored_source")); + } else { + assertThat(storedFieldNames, containsInAnyOrder("_id", "_ignored_source", "_recovery_source")); + } + + // Verify that there is no offset field: + LeafReader leafReader = reader.leaves().get(0).reader(); + for (FieldInfo fieldInfo : leafReader.getFieldInfos()) { + String name = fieldInfo.getName(); + assertFalse("expected no field that contains [offsets] in name, but found [" + name + "]", name.contains("offsets")); + } + + var binaryDocValues = leafReader.getBinaryDocValues("object.field.offsets"); + assertThat(binaryDocValues, nullValue()); + } + } + } + + protected void verifySyntheticArrayInObject(List documents) throws IOException { + var indexService = createIndex( + "test-index", + Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), + jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ); + for (int i = 0; i < documents.size(); i++) { + var arrayValue = documents.get(i); + + var indexRequest = new IndexRequest("test-index"); + indexRequest.id("my-id-" + i); + var source = jsonBuilder().startObject(); + source.startObject("object"); + source.array("field", arrayValue); + source.endObject(); + indexRequest.source(source.endObject()); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + + var searchRequest = new SearchRequest("test-index"); + searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); + var searchResponse = client().search(searchRequest).actionGet(); + try { + var hit = searchResponse.getHits().getHits()[0]; + assertThat(hit.getId(), equalTo("my-id-" + i)); + var sourceAsMap = hit.getSourceAsMap(); + var objectArray = (Map) sourceAsMap.get("object"); + + List actual = (List) objectArray.get("field"); + if (arrayValue == null) { + assertThat(actual, nullValue()); + } else if (arrayValue.length == 0) { + assertThat(actual, empty()); + } else { + assertThat(actual, Matchers.contains(arrayValue)); + } + } finally { + searchResponse.decRef(); + } + } + + indexService.getShard(0).forceMerge(new ForceMergeRequest("test-index").maxNumSegments(1)); + try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { + var reader = searcher.getDirectoryReader(); + for (int i = 0; i < documents.size(); i++) { + var document = reader.storedFields().document(i); + // Verify that there is no ignored source: + Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); + if (IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE.isEnabled()) { + assertThat(storedFieldNames, contains("_id")); + } else { + assertThat(storedFieldNames, containsInAnyOrder("_id", "_recovery_source")); + } + } + var fieldInfo = FieldInfos.getMergedFieldInfos(reader).fieldInfo("object.field.offsets"); + assertThat(fieldInfo.getDocValuesType(), equalTo(DocValuesType.SORTED)); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java new file mode 100644 index 000000000000..bb12067aefd1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.nullValue; + +public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCase { + + @Override + protected Settings getIndexSettings() { + return Settings.builder() + .put("index.mapping.source.mode", "synthetic") + .put("index.mapping.synthetic_source_keep", "arrays") + .build(); + } + + public void testOffsetArrayNoDocValues() throws Exception { + String mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}", + "doc_values": false + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + try (var mapperService = createMapperService(mapping)) { + var fieldMapper = mapperService.mappingLookup().getMapper("field"); + assertThat(fieldMapper.getOffsetFieldName(), nullValue()); + } + } + + public void testOffsetArrayStored() throws Exception { + String mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}", + "store": true + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + ; + try (var mapperService = createMapperService(mapping)) { + var fieldMapper = mapperService.mappingLookup().getMapper("field"); + assertThat(fieldMapper.getOffsetFieldName(), nullValue()); + } + } + + public void testOffsetMultiFields() throws Exception { + String mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}", + "fields": { + "sub": { + "type": "text" + } + } + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + try (var mapperService = createMapperService(mapping)) { + var fieldMapper = mapperService.mappingLookup().getMapper("field"); + assertThat(fieldMapper.getOffsetFieldName(), nullValue()); + } + } + + public void testOffsetArrayNoSyntheticSource() throws Exception { + String mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}" + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + try (var mapperService = createMapperService(Settings.EMPTY, mapping)) { + var fieldMapper = mapperService.mappingLookup().getMapper("field"); + assertThat(fieldMapper.getOffsetFieldName(), nullValue()); + } + } + + public void testOffsetArrayNoSourceArrayKeep() throws Exception { + var settingsBuilder = Settings.builder().put("index.mapping.source.mode", "synthetic"); + String mapping; + if (randomBoolean()) { + mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}", + "synthetic_source_keep": "{{synthetic_source_keep}}" + } + } + } + } + """.replace("{{synthetic_source_keep}}", randomBoolean() ? "none" : "all").replace("{{type}}", getFieldTypeName()); + } else { + mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}" + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + if (randomBoolean()) { + settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); + } + } + try (var mapperService = createMapperService(settingsBuilder.build(), mapping)) { + var fieldMapper = mapperService.mappingLookup().getMapper("field"); + assertThat(fieldMapper.getOffsetFieldName(), nullValue()); + } + } + + public void testOffsetEmptyArray() throws Exception { + verifyOffsets("{\"field\":[]}"); + } + + public void testOffsetArrayWithNulls() throws Exception { + verifyOffsets("{\"field\":[null,null,null]}"); + verifyOffsets("{\"field\":[null,[null],null]}", "{\"field\":[null,null,null]}"); + } + + public void testOffsetArrayRandom() throws Exception { + StringBuilder values = new StringBuilder(); + int numValues = randomIntBetween(0, 256); + for (int i = 0; i < numValues; i++) { + if (randomInt(10) == 1) { + values.append("null"); + } else { + String randomValue = randomValue(); + values.append('"').append(randomValue).append('"'); + } + if (i != (numValues - 1)) { + values.append(','); + } + } + verifyOffsets("{\"field\":[" + values + "]}"); + } + + protected abstract String getFieldTypeName(); + + protected abstract String randomValue(); + + protected void verifyOffsets(String source) throws IOException { + verifyOffsets(source, source); + } + + protected void verifyOffsets(String source, String expectedSource) throws IOException { + String mapping = """ + { + "_doc": { + "properties": { + "field": { + "type": "{{type}}" + } + } + } + } + """.replace("{{type}}", getFieldTypeName()); + verifyOffsets(mapping, source, expectedSource); + } + + private void verifyOffsets(String mapping, String source, String expectedSource) throws IOException { + try (var mapperService = createMapperService(mapping)) { + var mapper = mapperService.documentMapper(); + + try (var directory = newDirectory()) { + var iw = indexWriterForSyntheticSource(directory); + var doc = mapper.parse(new SourceToParse("_id", new BytesArray(source), XContentType.JSON)); + doc.updateSeqID(0, 0); + doc.version().setLongValue(0); + iw.addDocuments(doc.docs()); + iw.close(); + try (var indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { + FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + var syntheticSourceLoader = fieldMapper.syntheticFieldLoader(); + var leafReader = indexReader.leaves().getFirst().reader(); + var docValueLoader = syntheticSourceLoader.docValuesLoader(leafReader, new int[] { 0 }); + assertTrue(docValueLoader.advanceToDoc(0)); + assertTrue(syntheticSourceLoader.hasValue()); + XContentBuilder builder = jsonBuilder().startObject(); + syntheticSourceLoader.write(builder); + builder.endObject(); + + var actual = Strings.toString(builder); + assertEquals(expectedSource, actual); + } + } + } + } + +} From f929270431114d6a6dc1034111b2a760ce75d2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Tue, 25 Feb 2025 15:09:30 +0100 Subject: [PATCH 13/31] Fix how we suppress logs for self-tests (#123361) --- .../bootstrap/EntitlementBootstrap.java | 12 +++-- .../EntitlementInitialization.java | 3 +- .../runtime/policy/PolicyManager.java | 14 ++--- .../runtime/policy/PolicyManagerTests.java | 42 ++++++++++----- .../bootstrap/Elasticsearch.java | 53 ++++++++++--------- 5 files changed, 75 insertions(+), 49 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 06e985913c9b..6a80ec75751c 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; @@ -42,7 +43,8 @@ public class EntitlementBootstrap { Path libDir, Path logsDir, Path tempDir, - Path pidFile + Path pidFile, + Set> suppressFailureLogClasses ) { public BootstrapArgs { requireNonNull(pluginPolicies); @@ -58,6 +60,7 @@ public class EntitlementBootstrap { requireNonNull(libDir); requireNonNull(logsDir); requireNonNull(tempDir); + requireNonNull(suppressFailureLogClasses); } } @@ -82,6 +85,7 @@ public class EntitlementBootstrap { * @param tempDir the temp directory for Elasticsearch * @param logsDir the log directory for Elasticsearch * @param pidFile path to a pid file for Elasticsearch, or {@code null} if one was not specified + * @param suppressFailureLogClasses classes for which we do not need or want to log Entitlements failures */ public static void bootstrap( Map pluginPolicies, @@ -94,7 +98,8 @@ public class EntitlementBootstrap { Path libDir, Path logsDir, Path tempDir, - Path pidFile + Path pidFile, + Set> suppressFailureLogClasses ) { logger.debug("Loading entitlement agent"); if (EntitlementBootstrap.bootstrapArgs != null) { @@ -111,7 +116,8 @@ public class EntitlementBootstrap { libDir, logsDir, tempDir, - pidFile + pidFile, + suppressFailureLogClasses ); exportInitializationToAgent(); loadAgent(findAgentJar()); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index d3d09e0f84ae..055660d54079 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -272,7 +272,8 @@ public class EntitlementInitialization { resolver, AGENTS_PACKAGE_NAME, ENTITLEMENTS_MODULE, - pathLookup + pathLookup, + bootstrapArgs.suppressFailureLogClasses() ); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index b6296fe5d471..cf3775474b79 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -11,7 +11,6 @@ package org.elasticsearch.entitlement.runtime.policy; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; import org.elasticsearch.entitlement.instrumentation.InstrumentationService; import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement; @@ -114,6 +113,7 @@ public class PolicyManager { private final Function, String> pluginResolver; private final PathLookup pathLookup; private final FileAccessTree defaultFileAccess; + private final Set> mutedClasses; public static final String ALL_UNNAMED = "ALL-UNNAMED"; @@ -150,7 +150,8 @@ public class PolicyManager { Function, String> pluginResolver, String apmAgentPackageName, Module entitlementsModule, - PathLookup pathLookup + PathLookup pathLookup, + Set> suppressFailureLogClasses ) { this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy)); this.apmAgentEntitlements = apmAgentEntitlements; @@ -162,6 +163,7 @@ public class PolicyManager { this.entitlementsModule = entitlementsModule; this.pathLookup = requireNonNull(pathLookup); this.defaultFileAccess = FileAccessTree.of(FilesEntitlement.EMPTY, pathLookup); + this.mutedClasses = suppressFailureLogClasses; for (var e : serverEntitlements.entrySet()) { validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue()); @@ -386,7 +388,7 @@ public class PolicyManager { checkFlagEntitlement(classEntitlements, OutboundNetworkEntitlement.class, requestingClass, callerClass); } - private static void checkFlagEntitlement( + private void checkFlagEntitlement( ModuleEntitlements classEntitlements, Class entitlementClass, Class requestingClass, @@ -446,10 +448,10 @@ public class PolicyManager { ); } - private static void notEntitled(String message, Class callerClass) { + private void notEntitled(String message, Class callerClass) { var exception = new NotEntitledException(message); - // don't log self tests in EntitlementBootstrap - if (EntitlementBootstrap.class.equals(callerClass) == false) { + // Don't emit a log for muted classes, e.g. classes containing self tests + if (mutedClasses.contains(callerClass) == false) { logger.warn(message, exception); } throw exception; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index c9fade1a4821..5a65ea81d0a0 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -87,7 +87,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -111,7 +112,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -131,7 +133,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -156,7 +159,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin2", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -174,7 +178,8 @@ public class PolicyManagerTests extends ESTestCase { c -> null, TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Tests do not run modular, so we cannot use a server class. @@ -204,7 +209,8 @@ public class PolicyManagerTests extends ESTestCase { c -> null, TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Tests do not run modular, so we cannot use a server class. @@ -230,7 +236,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "mock-plugin", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); var layer = createLayerForJar(jar, "org.example.plugin"); @@ -249,7 +256,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin2", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); // Any class from the current module (unnamed) will do @@ -308,7 +316,8 @@ public class PolicyManagerTests extends ESTestCase { c -> c.getPackageName().startsWith(TEST_AGENTS_PACKAGE_NAME) ? null : "test", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); ModuleEntitlements agentsEntitlements = policyManager.getEntitlements(TestAgent.class); assertThat(agentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); @@ -336,7 +345,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "test", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ) ); assertEquals( @@ -353,7 +363,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "test", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ) ); assertEquals( @@ -387,7 +398,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "plugin1", TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ) ); assertEquals( @@ -407,7 +419,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "test", // Insist that the class is in a plugin TEST_AGENTS_PACKAGE_NAME, NO_ENTITLEMENTS_MODULE, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); ModuleEntitlements notAgentsEntitlements = policyManager.getEntitlements(TestAgent.class); assertThat(notAgentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(false)); @@ -428,7 +441,8 @@ public class PolicyManagerTests extends ESTestCase { c -> "test", agentsPackageName, entitlementsModule, - TEST_PATH_LOOKUP + TEST_PATH_LOOKUP, + Set.of() ); } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 05072ae940ae..5ccb4af45ade 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -256,9 +256,10 @@ class Elasticsearch { nodeEnv.libDir(), nodeEnv.logsDir(), nodeEnv.tmpDir(), - args.pidFile() + args.pidFile(), + Set.of(EntitlementSelfTester.class) ); - entitlementSelfTest(); + EntitlementSelfTester.entitlementSelfTest(); } else { assert RuntimeVersionFeature.isSecurityManagerAvailable(); // no need to explicitly enable native access for legacy code @@ -275,31 +276,33 @@ class Elasticsearch { bootstrap.setPluginsLoader(pluginsLoader); } - // check entitlements were loaded correctly. note this must be outside the entitlements lib. - private static void entitlementSelfTest() { - ensureCannotStartProcess(ProcessBuilder::start); - // Try again with reflection - ensureCannotStartProcess(Elasticsearch::reflectiveStartProcess); - } - - private static void ensureCannotStartProcess(CheckedConsumer startProcess) { - try { - // The command doesn't matter; it doesn't even need to exist - startProcess.accept(new ProcessBuilder("")); - } catch (NotEntitledException e) { - return; - } catch (Exception e) { - throw new IllegalStateException("Failed entitlement protection self-test", e); + private static class EntitlementSelfTester { + // check entitlements were loaded correctly. note this must be outside the entitlements lib. + private static void entitlementSelfTest() { + ensureCannotStartProcess(ProcessBuilder::start); + // Try again with reflection + ensureCannotStartProcess(EntitlementSelfTester::reflectiveStartProcess); } - throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); - } - private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { - try { - var start = ProcessBuilder.class.getMethod("start"); - start.invoke(pb); - } catch (InvocationTargetException e) { - throw (Exception) e.getCause(); + private static void ensureCannotStartProcess(CheckedConsumer startProcess) { + try { + // The command doesn't matter; it doesn't even need to exist + startProcess.accept(new ProcessBuilder("")); + } catch (NotEntitledException e) { + return; + } catch (Exception e) { + throw new IllegalStateException("Failed entitlement protection self-test", e); + } + throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); + } + + private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception { + try { + var start = ProcessBuilder.class.getMethod("start"); + start.invoke(pb); + } catch (InvocationTargetException e) { + throw (Exception) e.getCause(); + } } } From e8438490ea9605445ca0bc2f4ecf31a880f709a8 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 25 Feb 2025 09:14:20 -0500 Subject: [PATCH 14/31] Updates to allow using Cohere binary embedding response in semantic search queries. (#121827) * wip * wip * [CI] Auto commit changes from spotless * updating tests * [CI] Auto commit changes from spotless * Update docs/changelog/121827.yaml * Updates after the refactor * [CI] Auto commit changes from spotless * Updating error message --------- Co-authored-by: elasticsearchmachine --- docs/changelog/121827.yaml | 6 ++ .../results/TextEmbeddingBitResults.java | 3 +- .../mapper/SemanticTextFieldMapper.java | 7 +- .../services/cohere/CohereService.java | 9 ++- .../EmbeddingRequestChunkerTests.java | 73 +++++++++++++++++++ .../results/TextEmbeddingBitResultsTests.java | 11 +++ .../TextEmbeddingByteResultsTests.java | 11 +++ .../results/TextEmbeddingResultsTests.java | 11 +++ .../services/cohere/CohereServiceTests.java | 18 ++++- 9 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/121827.yaml diff --git a/docs/changelog/121827.yaml b/docs/changelog/121827.yaml new file mode 100644 index 000000000000..11c9c201655a --- /dev/null +++ b/docs/changelog/121827.yaml @@ -0,0 +1,6 @@ +pr: 121827 +summary: Updates to allow using Cohere binary embedding response in semantic search + queries +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java index ba4a770b0484..a8f676bf41ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java @@ -55,7 +55,8 @@ public record TextEmbeddingBitResults(List e if (embeddings.isEmpty()) { throw new IllegalStateException("Embeddings list is empty"); } - return embeddings.getFirst().values().length; + // bit embeddings are encoded as bytes so convert this to bits + return Byte.SIZE * embeddings.getFirst().values().length; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 3bebd8086d79..0ecc76504107 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -709,9 +709,12 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie MlTextEmbeddingResults textEmbeddingResults = (MlTextEmbeddingResults) inferenceResults; float[] inference = textEmbeddingResults.getInferenceAsFloat(); - if (inference.length != modelSettings.dimensions()) { + var inferenceLength = modelSettings.elementType() == DenseVectorFieldMapper.ElementType.BIT + ? inference.length * Byte.SIZE + : inference.length; + if (inferenceLength != modelSettings.dimensions()) { throw new IllegalArgumentException( - generateDimensionCountMismatchMessage(inference.length, modelSettings.dimensions()) + generateDimensionCountMismatchMessage(inferenceLength, modelSettings.dimensions()) ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 08ab5f1ba86d..c951a008df0f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; +import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankModel; @@ -313,7 +314,7 @@ public class CohereService extends SenderService { if (model instanceof CohereEmbeddingsModel embeddingsModel) { var serviceSettings = embeddingsModel.getServiceSettings(); var similarityFromModel = serviceSettings.similarity(); - var similarityToUse = similarityFromModel == null ? defaultSimilarity() : similarityFromModel; + var similarityToUse = similarityFromModel == null ? defaultSimilarity(serviceSettings.getEmbeddingType()) : similarityFromModel; var updatedServiceSettings = new CohereEmbeddingsServiceSettings( new CohereServiceSettings( @@ -341,7 +342,11 @@ public class CohereService extends SenderService { * * @return The default similarity. */ - static SimilarityMeasure defaultSimilarity() { + static SimilarityMeasure defaultSimilarity(CohereEmbeddingType embeddingType) { + if (embeddingType == CohereEmbeddingType.BIT || embeddingType == CohereEmbeddingType.BINARY) { + return SimilarityMeasure.L2_NORM; + } + return SimilarityMeasure.DOT_PRODUCT; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index 23912b4000d0..7cdae8d51ce0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingBitResults; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; @@ -377,6 +378,78 @@ public class EmbeddingRequestChunkerTests extends ESTestCase { } } + public void testMergingListener_Bit() { + int batchSize = 5; + int chunkSize = 20; + int overlap = 0; + // passage will be chunked into batchSize + 1 parts + // and spread over 2 batch requests + int numberOfWordsInPassage = (chunkSize * batchSize) + 5; + + var passageBuilder = new StringBuilder(); + for (int i = 0; i < numberOfWordsInPassage; i++) { + passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace + } + List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); + + var finalListener = testListener(); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener); + assertThat(batches, hasSize(2)); + + // 4 inputs in 2 batches + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new TextEmbeddingByteResults.Embedding(new byte[] { randomByte() })); + } + batches.get(0).listener().onResponse(new TextEmbeddingBitResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < 4; i++) { // 4 requests in the 2nd batch + embeddings.add(new TextEmbeddingByteResults.Embedding(new byte[] { randomByte() })); + } + batches.get(1).listener().onResponse(new TextEmbeddingBitResults(embeddings)); + } + + assertNotNull(finalListener.results); + assertThat(finalListener.results, hasSize(4)); + { + var chunkedResult = finalListener.results.get(0); + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("1st small", chunkedByteResult.chunks().get(0).matchedText()); + } + { + // this is the large input split in multiple chunks + var chunkedResult = finalListener.results.get(1); + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(6)); + assertThat(chunkedByteResult.chunks().get(0).matchedText(), startsWith("passage_input0 ")); + assertThat(chunkedByteResult.chunks().get(1).matchedText(), startsWith(" passage_input20 ")); + assertThat(chunkedByteResult.chunks().get(2).matchedText(), startsWith(" passage_input40 ")); + assertThat(chunkedByteResult.chunks().get(3).matchedText(), startsWith(" passage_input60 ")); + assertThat(chunkedByteResult.chunks().get(4).matchedText(), startsWith(" passage_input80 ")); + assertThat(chunkedByteResult.chunks().get(5).matchedText(), startsWith(" passage_input100 ")); + } + { + var chunkedResult = finalListener.results.get(2); + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("2nd small", chunkedByteResult.chunks().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(3); + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("3rd small", chunkedByteResult.chunks().get(0).matchedText()); + } + } + public void testMergingListener_Sparse() { int batchSize = 4; int chunkSize = 10; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java index 8d3ead44f233..fb3203f633ff 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java @@ -105,6 +105,17 @@ public class TextEmbeddingBitResultsTests extends AbstractWireSerializingTestCas ); } + public void testGetFirstEmbeddingSize() { + var firstEmbeddingSize = new TextEmbeddingBitResults( + List.of( + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23, (byte) 24 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 25, (byte) 26 }) + ) + ).getFirstEmbeddingSize(); + + assertThat(firstEmbeddingSize, is(16)); + } + @Override protected Writeable.Reader instanceReader() { return TextEmbeddingBitResults::new; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java index aaa823fda072..945eadd67d1f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java @@ -104,6 +104,17 @@ public class TextEmbeddingByteResultsTests extends AbstractWireSerializingTestCa ); } + public void testGetFirstEmbeddingSize() { + var firstEmbeddingSize = new TextEmbeddingByteResults( + List.of( + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23, (byte) 24 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 25, (byte) 26 }) + ) + ).getFirstEmbeddingSize(); + + assertThat(firstEmbeddingSize, is(2)); + } + @Override protected Writeable.Reader instanceReader() { return TextEmbeddingByteResults::new; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java index 77cbeaea68c3..c56defb69382 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java @@ -105,6 +105,17 @@ public class TextEmbeddingResultsTests extends AbstractWireSerializingTestCase instanceReader() { return TextEmbeddingFloatResults::new; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 8549871d67ff..bd1dbc201f52 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -1100,6 +1100,7 @@ public class CohereServiceTests extends ESTestCase { try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { var embeddingSize = randomNonNegativeInt(); + var embeddingType = randomFrom(CohereEmbeddingType.values()); var model = CohereEmbeddingsModelTests.createModel( randomAlphaOfLength(10), randomAlphaOfLength(10), @@ -1107,13 +1108,15 @@ public class CohereServiceTests extends ESTestCase { randomNonNegativeInt(), randomNonNegativeInt(), randomAlphaOfLength(10), - randomFrom(CohereEmbeddingType.values()), + embeddingType, similarityMeasure ); Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); - SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? CohereService.defaultSimilarity() : similarityMeasure; + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null + ? CohereService.defaultSimilarity(embeddingType) + : similarityMeasure; assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); } @@ -1590,8 +1593,15 @@ public class CohereServiceTests extends ESTestCase { } } - public void testDefaultSimilarity() { - assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity()); + public void testDefaultSimilarity_BinaryEmbedding() { + assertEquals(SimilarityMeasure.L2_NORM, CohereService.defaultSimilarity(CohereEmbeddingType.BINARY)); + assertEquals(SimilarityMeasure.L2_NORM, CohereService.defaultSimilarity(CohereEmbeddingType.BIT)); + } + + public void testDefaultSimilarity_NotBinaryEmbedding() { + assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity(CohereEmbeddingType.FLOAT)); + assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity(CohereEmbeddingType.BYTE)); + assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity(CohereEmbeddingType.INT8)); } public void testInfer_StreamRequest() throws Exception { From ae6474db630a5686f8ff221a4b435a6920a184b8 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 25 Feb 2025 10:02:50 -0500 Subject: [PATCH 15/31] Deprecate Behavioral Analytics CRUD apis (#122960) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Deprecate Behavioral Analytics CRUD APIs * Add allowed warning for REST Compatibility tests * Update docs/changelog/122960.yaml * Update changelog * Update docs to add deprecation flags and fix failing tests * Update changelog * Update changelog again * Update docs formatting Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> * Skip asciidoc test --------- Co-authored-by: Efe Gürkan YALAMAN Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Co-authored-by: Efe Gürkan YALAMAN --- docs/changelog/122960.yaml | 10 + .../apis/delete-analytics-collection.asciidoc | 11 +- .../behavioral-analytics/apis/index.asciidoc | 1 + .../apis/list-analytics-collection.asciidoc | 25 +-- .../post-analytics-collection-event.asciidoc | 16 +- .../apis/put-analytics-collection.asciidoc | 10 +- .../behavioral-analytics-api.asciidoc | 3 + .../behavioral-analytics-cors.asciidoc | 4 +- ...avioral-analytics-event-reference.asciidoc | 2 + .../behavioral-analytics-event.asciidoc | 2 + .../behavioral-analytics-overview.asciidoc | 3 + .../behavioral-analytics-start.asciidoc | 2 + ...plication.delete_behavioral_analytics.json | 4 + ..._application.get_behavioral_analytics.json | 12 +- ...ation.post_behavioral_analytics_event.json | 4 + ..._application.put_behavioral_analytics.json | 4 + x-pack/plugin/ent-search/qa/rest/build.gradle | 8 + .../rest-api-spec/test/entsearch/20_usage.yml | 6 + .../10_behavioral_analytics_list.yml | 26 ++- .../analytics/20_behavioral_analytics_put.yml | 14 ++ .../30_behavioral_analytics_delete.yml | 18 ++ .../40_behavioral_analytics_event_post.yml | 178 ++++++++++++------ ..._search_application_search_with_apikey.yml | 56 +++--- .../xpack/application/EnterpriseSearch.java | 4 + .../analytics/AnalyticsCollection.java | 2 + .../AnalyticsCollectionResolver.java | 2 + .../analytics/AnalyticsCollectionService.java | 2 + .../analytics/AnalyticsConstants.java | 4 + .../AnalyticsEventIngestService.java | 2 + .../analytics/AnalyticsTemplateRegistry.java | 4 + .../DeleteAnalyticsCollectionAction.java | 4 + .../action/GetAnalyticsCollectionAction.java | 4 + .../action/PostAnalyticsEventAction.java | 4 + .../action/PutAnalyticsCollectionAction.java | 4 + .../RestDeleteAnalyticsCollectionAction.java | 4 + .../RestGetAnalyticsCollectionAction.java | 4 + .../action/RestPostAnalyticsEventAction.java | 4 + .../RestPutAnalyticsCollectionAction.java | 4 + ...nsportDeleteAnalyticsCollectionAction.java | 11 ++ ...TransportGetAnalyticsCollectionAction.java | 11 ++ .../TransportPostAnalyticsEventAction.java | 13 +- ...TransportPutAnalyticsCollectionAction.java | 15 +- .../analytics/event/AnalyticsEvent.java | 2 + .../event/AnalyticsEventFactory.java | 2 + .../parser/event/PageViewAnalyticsEvent.java | 4 + .../parser/event/SearchAnalyticsEvent.java | 4 + .../event/SearchClickAnalyticsEvent.java | 4 + .../field/DocumentAnalyticsEventField.java | 4 + .../parser/field/PageAnalyticsEventField.java | 4 + .../field/PaginationAnalyticsEventField.java | 4 + .../field/SearchAnalyticsEventField.java | 4 + .../SearchFiltersAnalyticsEventField.java | 4 + .../SearchResultAnalyticsEventField.java | 4 + .../field/SessionAnalyticsEventField.java | 4 + .../field/SortOrderAnalyticsEventField.java | 4 + .../parser/field/UserAnalyticsEventField.java | 4 + .../ingest/AnalyticsEventEmitter.java | 4 + .../ingest/AnalyticsEventIngestConfig.java | 2 + .../ingest/BulkProcessorFactory.java | 2 + 59 files changed, 441 insertions(+), 140 deletions(-) create mode 100644 docs/changelog/122960.yaml diff --git a/docs/changelog/122960.yaml b/docs/changelog/122960.yaml new file mode 100644 index 000000000000..5745c41b007b --- /dev/null +++ b/docs/changelog/122960.yaml @@ -0,0 +1,10 @@ +pr: 122960 +summary: Deprecate Behavioral Analytics CRUD apis +area: Search +type: deprecation +issues: [ ] +deprecation: + title: Deprecate Behavioral Analytics CRUD apis + area: Search + details: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. The APIs will still work for now, but will emit warning headers that the API has been deprecated. + impact: Behavioral Analytics has been deprecated as of 9.0.0 and will be removed in a future release. diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc index 19c1b5437ef0..1c6c39ea137e 100644 --- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc @@ -2,6 +2,7 @@ [[delete-analytics-collection]] === Delete Analytics Collection +deprecated:[9.0.0] beta::[] ++++ @@ -14,15 +15,6 @@ beta::[] For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. -- -//// -[source,console] ----- -PUT _application/analytics/my_analytics_collection ----- -// TESTSETUP - -//// - Removes a <> Collection and its associated data stream. [[delete-analytics-collection-request]] @@ -59,3 +51,4 @@ The following example deletes the Analytics Collection named `my_analytics_colle ---- DELETE _application/analytics/my_analytics_collection/ ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] diff --git a/docs/reference/behavioral-analytics/apis/index.asciidoc b/docs/reference/behavioral-analytics/apis/index.asciidoc index 6dc12599c229..1fdcd0f1afc9 100644 --- a/docs/reference/behavioral-analytics/apis/index.asciidoc +++ b/docs/reference/behavioral-analytics/apis/index.asciidoc @@ -1,6 +1,7 @@ [[behavioral-analytics-apis]] == Behavioral Analytics APIs +deprecated:[9.0.0] beta::[] ++++ diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc index 46ee8296f3eb..c0892529bb58 100644 --- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc @@ -2,6 +2,7 @@ [[list-analytics-collection]] === List Analytics Collections +deprecated:[9.0.0] beta::[] ++++ @@ -14,22 +15,6 @@ beta::[] For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. -- -//// -[source,console] ----- -PUT _application/analytics/my_analytics_collection -PUT _application/analytics/my_analytics_collection2 ----- -// TESTSETUP - -[source,console] ----- -DELETE _application/analytics/my_analytics_collection -DELETE _application/analytics/my_analytics_collection2 ----- -// TEARDOWN -//// - Returns information about <> Collections. [[list-analytics-collection-request]] @@ -46,8 +31,9 @@ Requires the `manage_behavioral_analytics` cluster privilege. ==== {api-path-parms-title} ``:: -(optional, string) -Criteria is used to find a matching analytics collection. This could be the name of the collection or a pattern to match multiple. If not specified, will return all analytics collections. +(optional, string) Criteria is used to find a matching analytics collection. +This could be the name of the collection or a pattern to match multiple. +If not specified, will return all analytics collections. [[list-analytics-collection-response-codes]] ==== {api-response-codes-title} @@ -66,6 +52,7 @@ The following example lists all configured Analytics Collections: ---- GET _application/analytics/ ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] A sample response: @@ -91,6 +78,7 @@ The following example returns the Analytics Collection that matches `my_analytic ---- GET _application/analytics/my_analytics_collection ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] A sample response: @@ -111,6 +99,7 @@ The following example returns all Analytics Collections prefixed with `my`: ---- GET _application/analytics/my* ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] A sample response: diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc index 60985cd50d3d..aad246872e92 100644 --- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc +++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc @@ -2,6 +2,7 @@ [[post-analytics-collection-event]] === Post Event to an Analytics Collection +deprecated:[9.0.0] beta::[] ++++ @@ -14,20 +15,6 @@ beta::[] For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. -- -//// -[source,console] ----- -PUT _application/analytics/my_analytics_collection ----- -// TESTSETUP - -[source,console] ----- -DELETE _application/analytics/my_analytics_collection ----- -// TEARDOWN -//// - Post an event to a <> Collection. [[post-analytics-collection-event-request]] @@ -105,3 +92,4 @@ POST _application/analytics/my_analytics_collection/event/search_click } } ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc index 412277afa207..0547630db954 100644 --- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc @@ -2,6 +2,7 @@ [[put-analytics-collection]] === Put Analytics Collection +deprecated:[9.0.0] beta::[] ++++ @@ -14,14 +15,6 @@ beta::[] For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. -- -//// -[source,console] ----- -DELETE _application/analytics/my_analytics_collection ----- -// TEARDOWN -//// - Creates a <> Collection. [[put-analytics-collection-request]] @@ -55,3 +48,4 @@ The following example creates a new Analytics Collection called `my_analytics_co ---- PUT _application/analytics/my_analytics_collection ---- +// TEST[skip:Behavioral Analytics APIs emit deprecation warnings and will not be updated] diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-api.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-api.asciidoc index 1c406b8c8dfd..715be30924fd 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-api.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-api.asciidoc @@ -1,9 +1,12 @@ [[behavioral-analytics-api]] === Behavioral Analytics API overview + ++++ API overview ++++ +deprecated:[9.0.0] + This page outlines all the APIs available for behavioral analytics and links to their documentation. [discrete] diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-cors.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-cors.asciidoc index d67e47d1b02f..3dd75e06e3a6 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-cors.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-cors.asciidoc @@ -4,6 +4,8 @@ Set up CORs ++++ +deprecated:[9.0.0] + Behavioral Analytics sends events directly to the {es} API. This means that the browser makes requests to the {es} API directly. {es} supports https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS[Cross-Origin Resource Sharing (CORS)^], but this feature is disabled by default. @@ -43,4 +45,4 @@ On Elastic Cloud, you can do this by {cloud}/ec-add-user-settings.html#ec-add-us ==== Proxy the request through a server that supports CORS If you are unable to enable CORS on {es}, you can proxy the request through a server that supports CORS. -This is more complicated, but is a viable option. \ No newline at end of file +This is more complicated, but is a viable option. diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event-reference.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event-reference.asciidoc index 42dbd2313c92..d5aa69f0f3ff 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event-reference.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event-reference.asciidoc @@ -4,6 +4,8 @@ Events reference ++++ +deprecated:[9.0.0] + Behavioral Analytics logs events using the {ecs-ref}/ecs-reference.html[Elastic Common Schema^], including a custom field set for analytics events. Refer to <> of the full data objects that are logged. diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event.asciidoc index 3511f761456c..9aff3b89cd50 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-event.asciidoc @@ -4,6 +4,8 @@ View events ++++ +deprecated:[9.0.0] + [TIP] ==== Refer to <> for a complete list of the fields logged by events. diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-overview.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-overview.asciidoc index 95306e442b13..47d6f1dd80fb 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-overview.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-overview.asciidoc @@ -1,6 +1,9 @@ [[behavioral-analytics-overview]] == Search analytics +deprecated:[9.0.0] + + Behavioral Analytics is an analytics event collection platform. Use these tools to analyze your users' searching and clicking behavior. Leverage this information to improve the relevance of your search results and identify gaps in your content. diff --git a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-start.asciidoc b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-start.asciidoc index f29a6f3a37fd..a32837788416 100644 --- a/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-start.asciidoc +++ b/docs/reference/search/search-your-data/behavioral-analytics/behavioral-analytics-start.asciidoc @@ -4,6 +4,8 @@ Get started ++++ +deprecated:[9.0.0] + You can manage your analytics in the {kib} UI. Go to *Search > Behavioral Analytics* to get started. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.delete_behavioral_analytics.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.delete_behavioral_analytics.json index 7f19b6debce8..77a99c4c5d83 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.delete_behavioral_analytics.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.delete_behavioral_analytics.json @@ -6,6 +6,10 @@ }, "stability": "experimental", "visibility": "public", + "deprecated": { + "version": "9.0.0", + "description": "Behavioral Analytics has been deprecated and will be removed in a future release." + }, "headers": { "accept": [ "application/json" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.get_behavioral_analytics.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.get_behavioral_analytics.json index 175f5e8202b1..c4d2edcf42fc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.get_behavioral_analytics.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.get_behavioral_analytics.json @@ -6,6 +6,10 @@ }, "stability": "experimental", "visibility": "public", + "deprecated": { + "version": "9.0.0", + "description": "Behavioral Analytics has been deprecated and will be removed in a future release." + }, "headers": { "accept": [ "application/json" @@ -24,10 +28,10 @@ "methods": [ "GET" ], - "parts":{ - "name":{ - "type":"list", - "description":"A comma-separated list of analytics collections to limit the returned information" + "parts": { + "name": { + "type": "list", + "description": "A comma-separated list of analytics collections to limit the returned information" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.post_behavioral_analytics_event.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.post_behavioral_analytics_event.json index 1364fa0d9ef3..b497c3b1314b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.post_behavioral_analytics_event.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.post_behavioral_analytics_event.json @@ -6,6 +6,10 @@ }, "stability": "experimental", "visibility": "public", + "deprecated": { + "version": "9.0.0", + "description": "Behavioral Analytics has been deprecated and will be removed in a future release." + }, "headers": { "accept": [ "application/json" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.put_behavioral_analytics.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.put_behavioral_analytics.json index 4929e6b6621d..811791dd586d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.put_behavioral_analytics.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_application.put_behavioral_analytics.json @@ -5,6 +5,10 @@ "description": "Creates a behavioral analytics collection." }, "stability": "experimental", + "deprecated": { + "version": "9.0.0", + "description": "Behavioral Analytics has been deprecated and will be removed in a future release." + }, "visibility": "public", "headers": { "accept": [ diff --git a/x-pack/plugin/ent-search/qa/rest/build.gradle b/x-pack/plugin/ent-search/qa/rest/build.gradle index 5b04a326f142..68255a5e7d43 100644 --- a/x-pack/plugin/ent-search/qa/rest/build.gradle +++ b/x-pack/plugin/ent-search/qa/rest/build.gradle @@ -37,3 +37,11 @@ testClusters.configureEach { artifacts { restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } + + +tasks.named("yamlRestCompatTestTransform").configure( + { task -> + // Behavioral Analytics is deprecated with 9.0.0. + task.addAllowedWarning("Behavioral Analytics is deprecated and will be removed in a future release.") + } +) diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml index 8271b1762660..eca45925b293 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/20_usage.yml @@ -27,6 +27,8 @@ teardown: --- "xpack usage includes Enterprise Search": + - requires: + test_runner_features: [ allowed_warnings ] - do: xpack.usage: { } @@ -79,6 +81,8 @@ teardown: query: "{{query_string}}" - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: test-analytics-collection @@ -113,6 +117,8 @@ teardown: } - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: test-analytics-collection diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml index 387348cf4064..94aea5bbd70f 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/10_behavioral_analytics_list.yml @@ -1,39 +1,55 @@ setup: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: my-test-analytics-collection - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: my-test-analytics-collection2 --- teardown: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection2 --- "Get Analytics Collection for a particular collection": - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.get_behavioral_analytics: name: my-test-analytics-collection - match: { - "my-test-analytics-collection": { - event_data_stream: { - name: "behavioral_analytics-events-my-test-analytics-collection" - } + "my-test-analytics-collection": { + event_data_stream: { + name: "behavioral_analytics-events-my-test-analytics-collection" } } + } --- "Get Analytics Collection list": - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.get_behavioral_analytics: name: @@ -56,6 +72,8 @@ teardown: "Get Analytics Collection - Resource does not exist": - do: catch: "missing" + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.get_behavioral_analytics: name: test-nonexistent-analytics-collection diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml index 7252de0b3b79..427a46d69ad9 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/20_behavioral_analytics_put.yml @@ -1,11 +1,19 @@ teardown: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: test-analytics-collection --- "Create Analytics Collection": + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: test-analytics-collection @@ -14,7 +22,11 @@ teardown: --- "Create Analytics Collection - analytics collection already exists": + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: test-analytics-collection @@ -22,6 +34,8 @@ teardown: - do: catch: bad_request + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: test-analytics-collection diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml index d6b158cd4a52..5b055bd952fe 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/30_behavioral_analytics_delete.yml @@ -1,18 +1,30 @@ setup: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: my-test-analytics-collection --- teardown: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection ignore: 404 --- "Delete Analytics Collection": + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection @@ -20,13 +32,19 @@ teardown: - do: catch: "missing" + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.get_behavioral_analytics: name: my-test-analytics-collection --- "Delete Analytics Collection - Analytics Collection does not exist": + - requires: + test_runner_features: [ allowed_warnings ] - do: catch: "missing" + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: test-nonexistent-analytics-collection diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml index f26bc2480516..396fc4e06b98 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/analytics/40_behavioral_analytics_event_post.yml @@ -1,11 +1,19 @@ setup: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: my-test-analytics-collection --- teardown: + - requires: + test_runner_features: [ allowed_warnings ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection @@ -13,30 +21,34 @@ teardown: # Page view event tests ######################################### --- "Post page_view analytics event": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" body: - session: - id: "123" - user: - id: "456" - page: - url: "https://www.elastic.co" + session: + id: "123" + user: + id: "456" + page: + url: "https://www.elastic.co" --- "Post page_view analytics event - Missing page.url": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -49,11 +61,13 @@ teardown: --- "Post page_view analytics event - With document": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -70,11 +84,13 @@ teardown: --- "Post page_view analytics event - With page title": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -89,11 +105,13 @@ teardown: --- "Post page_view analytics event - With referrer": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -108,14 +126,16 @@ teardown: --- "Post page_view analytics event - debug and session information": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: X-Forwarded-For: 192.23.12.12 User-Agent: Mozilla/5.0 Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -152,11 +172,13 @@ teardown: # Search event tests ############################################ --- "Post search analytics event": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -170,12 +192,14 @@ teardown: --- "Post search analytics event – Missing search query": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -188,11 +212,13 @@ teardown: --- "Post search analytics event - With sort order": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -208,11 +234,13 @@ teardown: --- "Post search analytics event - With sort name and direction": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -229,11 +257,13 @@ teardown: --- "Post search analytics event - With pagination": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -250,11 +280,13 @@ teardown: --- "Post search analytics event - With search application": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -269,10 +301,12 @@ teardown: --- "Post search analytics event - With search results": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection @@ -287,7 +321,7 @@ teardown: results: total_results: 150 items: - - document : + - document: id: doc-1 - document: id: doc-2 @@ -302,11 +336,13 @@ teardown: --- "Post search analytics event - With filters": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -324,14 +360,16 @@ teardown: --- "Post search analytics event - debug and session information": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: X-Forwarded-For: 192.23.12.12 User-Agent: Mozilla/5.0 Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search" @@ -393,11 +431,13 @@ teardown: # Search click event tests ####################################### --- "Post search_click analytics event": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -415,14 +455,16 @@ teardown: --- "Post search_click analytics event - debug and session information": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: X-Forwarded-For: 192.23.12.12 User-Agent: Mozilla/5.0 Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -455,11 +497,13 @@ teardown: --- "Post search_click analytics event - Page Only": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -475,11 +519,13 @@ teardown: --- "Post search_click analytics event - Document Only": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -496,12 +542,14 @@ teardown: --- "Post search_click analytics event – Missing search query": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -516,12 +564,14 @@ teardown: --- "Post search_click analytics event – Missing page url and document": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "search_click" @@ -537,12 +587,14 @@ teardown: # Generic errors tests ############################################### --- "Post analytics event - Analytics collection does not exist": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "missing" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: test-nonexistent-analytics-collection event_type: "page_view" @@ -556,12 +608,14 @@ teardown: --- "Post analytics event - Event type does not exist": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "nonexistent-event-type" @@ -577,12 +631,14 @@ teardown: --- "Post page_view analytics event - Missing session.id": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -595,12 +651,14 @@ teardown: --- "Post page_view analytics event - Missing user.id": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "page_view" @@ -613,12 +671,14 @@ teardown: --- "Post analytics event - Unknown event field": - - skip: - features: headers + - requires: + test_runner_features: [ allowed_warnings, headers ] - do: catch: "bad_request" headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.post_behavioral_analytics_event: collection_name: my-test-analytics-collection event_type: "nonexistent-event-type" diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml index b11b5fb4f3b6..ae91fefa8cf8 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/56_search_application_search_with_apikey.yml @@ -1,4 +1,6 @@ setup: + - requires: + test_runner_features: [ allowed_warnings ] - do: indices.create: index: test-search-index1 @@ -27,6 +29,8 @@ setup: number_of_replicas: 0 - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put: name: test-search-application body: @@ -51,6 +55,8 @@ setup: type: string - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put: name: test-search-application-1 body: @@ -110,10 +116,14 @@ setup: refresh: true - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.put_behavioral_analytics: name: my-test-analytics-collection --- teardown: + - requires: + test_runner_features: [ allowed_warnings ] - do: search_application.delete: name: test-search-application @@ -145,18 +155,20 @@ teardown: ignore: 404 - do: + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.delete_behavioral_analytics: name: my-test-analytics-collection --- "Query Search Application with API key": - - skip: - features: headers + - requires: + test_runner_features: [ headers, allowed_warnings ] - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user security.create_api_key: - body: > + body: > { "name": "search-application-api-key", "role_descriptors": { @@ -176,10 +188,10 @@ teardown: } - match: { name: "search-application-api-key" } - - set: { encoded: api_key_encoded } + - set: { encoded: api_key_encoded } - set: { id: api_key_id } -# Query Search Application with default parameters: + # Query Search Application with default parameters: - do: headers: Authorization: ApiKey ${api_key_encoded} @@ -191,7 +203,7 @@ teardown: - match: { hits.hits.1._id: "doc2" } -# Query Search Application overriding part of the parameters: + # Query Search Application overriding part of the parameters: - do: headers: Authorization: ApiKey ${api_key_encoded} @@ -205,7 +217,7 @@ teardown: - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "doc1" } -# Query Search Application overriding all parameters: + # Query Search Application overriding all parameters: - do: headers: Authorization: ApiKey ${api_key_encoded} @@ -220,7 +232,7 @@ teardown: - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "doc2" } -# Query Search Application with list of parameters: + # Query Search Application with list of parameters: - do: headers: Authorization: ApiKey ${api_key_encoded} @@ -241,7 +253,7 @@ teardown: - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "doc2" } -# Query Search Application with invalid parameter validation: + # Query Search Application with invalid parameter validation: - do: catch: "bad_request" headers: @@ -253,7 +265,7 @@ teardown: field_name: field3 field_value: 35 -# Query Search Application without required parameter: + # Query Search Application without required parameter: - do: catch: "bad_request" headers: @@ -264,7 +276,7 @@ teardown: params: field_value: test -# Query Search Application - not found: + # Query Search Application - not found: - do: catch: forbidden headers: @@ -276,11 +288,13 @@ teardown: field_name: field3 field_value: value3 -# Get Analytics Collection should be rejected due to a workflow restriction + # Get Analytics Collection should be rejected due to a workflow restriction - do: catch: forbidden headers: Authorization: ApiKey ${api_key_encoded} + allowed_warnings: + - "Behavioral Analytics is deprecated and will be removed in a future release." search_application.get_behavioral_analytics: name: - match: { status: 403 } @@ -288,7 +302,7 @@ teardown: - match: { error.root_cause.0.type: role_restriction_exception } - match: { error.root_cause.0.reason: "access restricted by workflow" } -# Get API key should not be allowed + # Get API key should not be allowed - do: catch: forbidden headers: @@ -300,18 +314,18 @@ teardown: - match: { error.root_cause.0.type: role_restriction_exception } - match: { error.root_cause.0.reason: "access restricted by workflow" } -# Authenticate with API key should not be allowed + # Authenticate with API key should not be allowed - do: catch: forbidden headers: Authorization: ApiKey ${api_key_encoded} - security.authenticate: {} + security.authenticate: { } - match: { status: 403 } - match: { error.type: security_exception } - match: { error.root_cause.0.type: role_restriction_exception } - match: { error.root_cause.0.reason: "access restricted by workflow" } -# Direct index search should be rejected due to a workflow restriction + # Direct index search should be rejected due to a workflow restriction - do: catch: forbidden headers: @@ -327,11 +341,11 @@ teardown: - match: { error.root_cause.0.type: role_restriction_exception } - match: { error.root_cause.0.reason: "access restricted by workflow" } -# Creating an API key which can only search 'test-search-application-1' + # Creating an API key which can only search 'test-search-application-1' - do: headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" } # user security.create_api_key: - body: > + body: > { "name": "search-application-1-api-key", "role_descriptors": { @@ -350,10 +364,10 @@ teardown: } - match: { name: "search-application-1-api-key" } - - set: { encoded: api_key_encoded_1 } + - set: { encoded: api_key_encoded_1 } - set: { id: api_key_id_1 } -# Query Search Application 'test-search-application' should be denied since API key (api_key_encoded_1) does not have required index privilege + # Query Search Application 'test-search-application' should be denied since API key (api_key_encoded_1) does not have required index privilege - do: catch: forbidden headers: @@ -364,7 +378,7 @@ teardown: - match: { error.type: security_exception } - match: { error.reason: "action [indices:data/read/xpack/application/search_application/search] is unauthorized for API key id [${api_key_id_1}] of user [entsearch-user] on indices [test-search-application], this action is granted by the index privileges [read,all]" } -# Query Search Application 'test-search-application-1' with new API key (api_key_encoded_1) should be allowed: + # Query Search Application 'test-search-application-1' with new API key (api_key_encoded_1) should be allowed: - do: headers: Authorization: ApiKey ${api_key_encoded_1} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index df1c76ccf770..db6ee3a621d8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -233,6 +233,10 @@ public class EnterpriseSearch extends Plugin implements ActionPlugin, SystemInde private final boolean enabled; + // NOTE: Behavioral Analytics is deprecated in 9.0 but not 8.x. + public static final String BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE = + "Behavioral Analytics is deprecated and will be removed in a future release."; + public EnterpriseSearch(Settings settings) { this.enabled = XPackSettings.ENTERPRISE_SEARCH_ENABLED.get(settings); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollection.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollection.java index 96ea6dfaec53..b0c000a3ffa6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollection.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollection.java @@ -28,7 +28,9 @@ import static org.elasticsearch.xpack.application.analytics.AnalyticsConstants.E /** * The {@link AnalyticsCollection} model. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsCollection implements Writeable, ToXContentObject { private static final ObjectParser PARSER = ObjectParser.fromBuilder( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java index 405c29b00586..d6df46c1882b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java @@ -28,7 +28,9 @@ import static org.elasticsearch.xpack.application.analytics.AnalyticsConstants.E /** * A service that allows the resolution of {@link AnalyticsCollection} by name. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsCollectionResolver { private final IndexNameExpressionResolver indexNameExpressionResolver; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java index fc43783cfd8a..322b3bdad036 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java @@ -33,7 +33,9 @@ import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; * Until we have more specific need the {@link AnalyticsCollection} is just another representation * of a {@link org.elasticsearch.cluster.metadata.DataStream}. * As a consequence, this service is mostly a facade for the data stream API. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsCollectionService { private static final Logger logger = LogManager.getLogger(AnalyticsCollectionService.class); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsConstants.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsConstants.java index 9b45a13d18d0..59c6e445365a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsConstants.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsConstants.java @@ -7,6 +7,10 @@ package org.elasticsearch.xpack.application.analytics; +/** + * @deprecated in 9.0 + */ +@Deprecated public class AnalyticsConstants { private AnalyticsConstants() {} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java index 2166646fc4bf..8df0a1a7f3bc 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java @@ -17,7 +17,9 @@ import java.util.Objects; /** * Event emitter will index Analytics events submitted through a @{PostAnalyticsEventAction.Request} request. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsEventIngestService { private final AnalyticsCollectionResolver collectionResolver; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java index 99a239dd617a..f77236393927 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java @@ -31,6 +31,10 @@ import static org.elasticsearch.xpack.application.analytics.AnalyticsConstants.R import static org.elasticsearch.xpack.application.analytics.AnalyticsConstants.TEMPLATE_VERSION_VARIABLE; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; +/** + * @deprecated in 9.0 + */ +@Deprecated public class AnalyticsTemplateRegistry extends IndexTemplateRegistry { // This number must be incremented when we make changes to built-in templates. diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java index f6fd75888293..aace5620bd64 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionAction.java @@ -24,6 +24,10 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; +/** + * @deprecated in 9.0 + */ +@Deprecated public class DeleteAnalyticsCollectionAction { public static final String NAME = "cluster:admin/xpack/application/analytics/delete"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java index 833fc64f03f8..8e68a99b7418 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionAction.java @@ -25,6 +25,10 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; +/** + * @deprecated in 9.0 + */ +@Deprecated public class GetAnalyticsCollectionAction { public static final String NAME = "cluster:admin/xpack/application/analytics/get"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java index 7d93c33208e5..21056239648f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventAction.java @@ -36,6 +36,10 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +/** + * @deprecated in 9.0 + */ +@Deprecated public class PostAnalyticsEventAction { public static final String NAME = "cluster:admin/xpack/application/analytics/post_event"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java index 301711146890..db77c9cca417 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java @@ -23,6 +23,10 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; +/** + * @deprecated in 9.0 + */ +@Deprecated public class PutAnalyticsCollectionAction { public static final String NAME = "cluster:admin/xpack/application/analytics/put"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestDeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestDeleteAnalyticsCollectionAction.java index 9165b4ee05d8..439905aac848 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestDeleteAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestDeleteAnalyticsCollectionAction.java @@ -23,6 +23,10 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.DELETE; +/** + * @deprecated in 9.0 + */ +@Deprecated @ServerlessScope(Scope.PUBLIC) public class RestDeleteAnalyticsCollectionAction extends EnterpriseSearchBaseRestHandler { public RestDeleteAnalyticsCollectionAction(XPackLicenseState licenseState) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestGetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestGetAnalyticsCollectionAction.java index 07b51d3a29ec..1413a9b6c0a3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestGetAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestGetAnalyticsCollectionAction.java @@ -23,6 +23,10 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; +/** + * @deprecated in 9.0 + */ +@Deprecated @ServerlessScope(Scope.PUBLIC) public class RestGetAnalyticsCollectionAction extends EnterpriseSearchBaseRestHandler { public RestGetAnalyticsCollectionAction(XPackLicenseState licenseState) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPostAnalyticsEventAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPostAnalyticsEventAction.java index 5706e5e38405..bdfd1afb321a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPostAnalyticsEventAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPostAnalyticsEventAction.java @@ -29,6 +29,10 @@ import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; +/** + * @deprecated in 9.0 + */ +@Deprecated @ServerlessScope(Scope.PUBLIC) public class RestPostAnalyticsEventAction extends EnterpriseSearchBaseRestHandler { public RestPostAnalyticsEventAction(XPackLicenseState licenseState) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPutAnalyticsCollectionAction.java index 4c7f50af6f30..05358cdadb00 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPutAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/RestPutAnalyticsCollectionAction.java @@ -24,6 +24,10 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.PUT; +/** + * @deprecated in 9.0 + */ +@Deprecated @ServerlessScope(Scope.PUBLIC) public class RestPutAnalyticsCollectionAction extends EnterpriseSearchBaseRestHandler { public RestPutAnalyticsCollectionAction(XPackLicenseState licenseState) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java index 78fc1e171df1..88dc3ff2889c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java @@ -15,6 +15,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -22,6 +24,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.analytics.AnalyticsCollectionService; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_API_ENDPOINT; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE; + +/** + * @deprecated in 9.0 + */ +@Deprecated public class TransportDeleteAnalyticsCollectionAction extends AcknowledgedTransportMasterNodeAction< DeleteAnalyticsCollectionAction.Request> { @@ -59,6 +68,8 @@ public class TransportDeleteAnalyticsCollectionAction extends AcknowledgedTransp ClusterState state, ActionListener listener ) { + DeprecationLogger.getLogger(TransportDeleteAnalyticsCollectionAction.class) + .warn(DeprecationCategory.API, BEHAVIORAL_ANALYTICS_API_ENDPOINT, BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE); analyticsCollectionService.deleteAnalyticsCollection(state, request, listener.map(v -> AcknowledgedResponse.TRUE)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java index 6d6a19aeb146..62a038de76ae 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java @@ -13,6 +13,8 @@ import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -20,6 +22,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.analytics.AnalyticsCollectionService; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_API_ENDPOINT; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE; + +/** + * @deprecated in 9.0 + */ +@Deprecated public class TransportGetAnalyticsCollectionAction extends TransportMasterNodeReadAction< GetAnalyticsCollectionAction.Request, GetAnalyticsCollectionAction.Response> { @@ -54,6 +63,8 @@ public class TransportGetAnalyticsCollectionAction extends TransportMasterNodeRe ClusterState state, ActionListener listener ) { + DeprecationLogger.getLogger(TransportDeleteAnalyticsCollectionAction.class) + .warn(DeprecationCategory.API, BEHAVIORAL_ANALYTICS_API_ENDPOINT, BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE); analyticsCollectionService.getAnalyticsCollection(state, request, listener); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java index bf81ba75400b..a260ff25cf5f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java @@ -10,17 +10,25 @@ package org.elasticsearch.xpack.application.analytics.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.analytics.AnalyticsEventIngestService; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_API_ENDPOINT; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE; + /** * Transport implementation for the {@link PostAnalyticsEventAction}. * It executes the {@link AnalyticsEventIngestService#addEvent} method if the XPack license is valid, else it calls * the listener's onFailure method with the appropriate exception. + * @deprecated in 9.0 */ +@Deprecated public class TransportPostAnalyticsEventAction extends HandledTransportAction< PostAnalyticsEventAction.Request, PostAnalyticsEventAction.Response> { @@ -31,7 +39,8 @@ public class TransportPostAnalyticsEventAction extends HandledTransportAction< public TransportPostAnalyticsEventAction( TransportService transportService, ActionFilters actionFilters, - AnalyticsEventIngestService eventEmitterService + AnalyticsEventIngestService eventEmitterService, + ClusterService clusterService ) { super( PostAnalyticsEventAction.NAME, @@ -49,6 +58,8 @@ public class TransportPostAnalyticsEventAction extends HandledTransportAction< PostAnalyticsEventAction.Request request, ActionListener listener ) { + DeprecationLogger.getLogger(TransportDeleteAnalyticsCollectionAction.class) + .warn(DeprecationCategory.API, BEHAVIORAL_ANALYTICS_API_ENDPOINT, BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE); this.eventEmitterService.addEvent(request, listener); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java index d34d723c0e76..b02bf3edbad2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java @@ -14,13 +14,23 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.analytics.AnalyticsCollectionService; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_API_ENDPOINT; +import static org.elasticsearch.xpack.application.EnterpriseSearch.BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE; + +/** + * @deprecated in 9.0 + */ +@Deprecated public class TransportPutAnalyticsCollectionAction extends TransportMasterNodeAction< PutAnalyticsCollectionAction.Request, PutAnalyticsCollectionAction.Response> { @@ -33,7 +43,8 @@ public class TransportPutAnalyticsCollectionAction extends TransportMasterNodeAc ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - AnalyticsCollectionService analyticsCollectionService + AnalyticsCollectionService analyticsCollectionService, + FeatureService featureService ) { super( PutAnalyticsCollectionAction.NAME, @@ -60,6 +71,8 @@ public class TransportPutAnalyticsCollectionAction extends TransportMasterNodeAc ClusterState state, ActionListener listener ) { + DeprecationLogger.getLogger(TransportDeleteAnalyticsCollectionAction.class) + .warn(DeprecationCategory.API, BEHAVIORAL_ANALYTICS_API_ENDPOINT, BEHAVIORAL_ANALYTICS_DEPRECATION_MESSAGE); analyticsCollectionService.putAnalyticsCollection(state, request, listener); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEvent.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEvent.java index 3bf8c372cd30..cdf3415d43bd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEvent.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEvent.java @@ -31,7 +31,9 @@ import static org.elasticsearch.xpack.application.analytics.AnalyticsConstants.E /** * This class represents Analytics events object meant to be emitted to the event queue. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsEvent implements Writeable, ToXContentObject { public static final ParseField TIMESTAMP_FIELD = new ParseField("@timestamp"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEventFactory.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEventFactory.java index aa10b143f69e..6cf1c0c094bc 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEventFactory.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/AnalyticsEventFactory.java @@ -29,7 +29,9 @@ import static org.elasticsearch.xpack.application.analytics.event.AnalyticsEvent /** * A utility class for parsing {@link AnalyticsEvent} objects from payloads (such as HTTP POST request bodies) or input streams. + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsEventFactory { public static final AnalyticsEventFactory INSTANCE = new AnalyticsEventFactory(); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/PageViewAnalyticsEvent.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/PageViewAnalyticsEvent.java index 81a4bf5ddfda..ae0191cae019 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/PageViewAnalyticsEvent.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/PageViewAnalyticsEvent.java @@ -22,6 +22,10 @@ import static org.elasticsearch.xpack.application.analytics.event.parser.field.P import static org.elasticsearch.xpack.application.analytics.event.parser.field.SessionAnalyticsEventField.SESSION_FIELD; import static org.elasticsearch.xpack.application.analytics.event.parser.field.UserAnalyticsEventField.USER_FIELD; +/** + * @deprecated in 9.0 + */ +@Deprecated public class PageViewAnalyticsEvent { private static final ObjectParser PARSER = ObjectParser.fromBuilder( "page_view_event", diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchAnalyticsEvent.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchAnalyticsEvent.java index 86453a33a342..16ae078e079b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchAnalyticsEvent.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchAnalyticsEvent.java @@ -20,6 +20,10 @@ import static org.elasticsearch.xpack.application.analytics.event.parser.field.S import static org.elasticsearch.xpack.application.analytics.event.parser.field.SessionAnalyticsEventField.SESSION_FIELD; import static org.elasticsearch.xpack.application.analytics.event.parser.field.UserAnalyticsEventField.USER_FIELD; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SearchAnalyticsEvent { private static final ObjectParser PARSER = ObjectParser.fromBuilder( "search_event", diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchClickAnalyticsEvent.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchClickAnalyticsEvent.java index aeffc818960e..f5ae40b1d06b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchClickAnalyticsEvent.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/event/SearchClickAnalyticsEvent.java @@ -24,6 +24,10 @@ import static org.elasticsearch.xpack.application.analytics.event.parser.field.S import static org.elasticsearch.xpack.application.analytics.event.parser.field.SessionAnalyticsEventField.SESSION_FIELD; import static org.elasticsearch.xpack.application.analytics.event.parser.field.UserAnalyticsEventField.USER_FIELD; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SearchClickAnalyticsEvent { private static final ObjectParser PARSER = ObjectParser.fromBuilder( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java index 1cab0d191197..af55af7b480f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/DocumentAnalyticsEventField.java @@ -18,6 +18,10 @@ import java.util.Map; import static org.elasticsearch.common.Strings.requireNonBlank; +/** + * @deprecated in 9.0 + */ +@Deprecated public class DocumentAnalyticsEventField { public static final ParseField DOCUMENT_FIELD = new ParseField("document"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java index b2436be31919..07f4a6fd0556 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PageAnalyticsEventField.java @@ -16,6 +16,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +/** + * @deprecated in 9.0 + */ +@Deprecated public class PageAnalyticsEventField { public static final ParseField PAGE_FIELD = new ParseField("page"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java index 3dca22d94ad7..68b9d880a3a8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/PaginationAnalyticsEventField.java @@ -17,6 +17,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +/** + * @deprecated in 9.0 + */ +@Deprecated public class PaginationAnalyticsEventField { public static final ParseField PAGINATION_FIELD = new ParseField("page"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java index 438fb20dcf0b..1595dc398e6f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchAnalyticsEventField.java @@ -20,6 +20,10 @@ import static org.elasticsearch.xpack.application.analytics.event.parser.field.P import static org.elasticsearch.xpack.application.analytics.event.parser.field.SearchFiltersAnalyticsEventField.SEARCH_FILTERS_FIELD; import static org.elasticsearch.xpack.application.analytics.event.parser.field.SortOrderAnalyticsEventField.SORT_FIELD; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SearchAnalyticsEventField { public static final ParseField SEARCH_FIELD = new ParseField("search"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java index 261a6c623a7a..b3023920eb2b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchFiltersAnalyticsEventField.java @@ -18,6 +18,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SearchFiltersAnalyticsEventField { public static final ParseField SEARCH_FILTERS_FIELD = new ParseField("filters"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java index e95e777bb88e..602f8b8f5a95 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SearchResultAnalyticsEventField.java @@ -19,6 +19,10 @@ import java.util.Map; import static org.elasticsearch.xpack.application.analytics.event.parser.field.DocumentAnalyticsEventField.DOCUMENT_FIELD; import static org.elasticsearch.xpack.application.analytics.event.parser.field.PageAnalyticsEventField.PAGE_FIELD; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SearchResultAnalyticsEventField { public static final ParseField SEARCH_RESULTS_TOTAL_FIELD = new ParseField("total_results"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java index 866984c62b75..96c4fcf83c85 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SessionAnalyticsEventField.java @@ -20,6 +20,10 @@ import java.util.Map; import static org.elasticsearch.common.Strings.requireNonBlank; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SessionAnalyticsEventField { public static final ParseField SESSION_FIELD = new ParseField("session"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java index df591572a767..5ff7ebd843fc 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/SortOrderAnalyticsEventField.java @@ -18,6 +18,10 @@ import java.util.Map; import static org.elasticsearch.common.Strings.requireNonBlank; +/** + * @deprecated in 9.0 + */ +@Deprecated public class SortOrderAnalyticsEventField { public static final ParseField SORT_FIELD = new ParseField("sort"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java index 186f581d4b6f..76ff15d900bf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/event/parser/field/UserAnalyticsEventField.java @@ -19,6 +19,10 @@ import java.util.Map; import static org.elasticsearch.common.Strings.requireNonBlank; +/** + * @deprecated in 9.0 + */ +@Deprecated public class UserAnalyticsEventField { public static final ParseField USER_FIELD = new ParseField("user"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java index a49e29654b0d..cb2df7f16db6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java @@ -34,6 +34,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; +/** + * @deprecated in 9.0 + */ +@Deprecated public class AnalyticsEventEmitter extends AbstractLifecycleComponent { private static final Logger logger = LogManager.getLogger(AnalyticsEventEmitter.class); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java index ac78a981c2d0..fbf34e5e3fa4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java @@ -20,7 +20,9 @@ import org.elasticsearch.injection.guice.Inject; * - flush_delay: the maximum delay between two flushes (default: 10s.) * - max_events_per_bulk: the maximum number of events that can be added to the bulk before flushing the bulk (default: 1000) * - max_number_of_retries: the maximum number of retries when bulk execution fails (default: 3) + * @deprecated in 9.0 */ +@Deprecated public class AnalyticsEventIngestConfig { private static final String SETTING_ROOT_PATH = "xpack.applications.behavioral_analytics.ingest"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java index 343d050ffbfd..595d4147e9f1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java @@ -26,7 +26,9 @@ import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; /** * Event ingest is done through a {@link BulkProcessor2}. This class is responsible for instantiating the bulk processor. + * @deprecated in 9.0 */ +@Deprecated public class BulkProcessorFactory { private static final Logger logger = LogManager.getLogger(AnalyticsEventEmitter.class); From 28f4d87ba51922f46d154d5ff57d82cd02251a62 Mon Sep 17 00:00:00 2001 From: Jill Guyonnet Date: Tue, 25 Feb 2025 16:38:47 +0100 Subject: [PATCH 16/31] [Fleet] Add `upgrade_attempts` to `.fleet-agents` index (#123256) * [Fleet] Add upgrade_attemps to .fleet-agents index * Change type to date --------- Co-authored-by: Elastic Machine --- .../template-resources/src/main/resources/fleet-agents.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 8b1c13f3152e..fb7f96cd223d 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -310,6 +310,9 @@ "upgrade_status": { "type": "keyword" }, + "upgrade_attempts": { + "type": "date" + }, "user_provided_metadata": { "type": "object", "enabled": false From c02b484eab3f865b0b3862926d3855506a96b03d Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 25 Feb 2025 07:59:15 -0800 Subject: [PATCH 17/31] Use mock block factory in breaker tests (#122965) We have two leaks reported in breaker tests, but we do not have their traces. This PR integrates MockBlockFactory for breaker tests to help identify these potential leaks. Relates #122810 --- muted-tests.yml | 2 -- .../compute/data/BlockFactoryProvider.java | 12 ++++++++++++ .../esql/action/AbstractEsqlIntegTestCase.java | 18 ++++++++++++++++++ .../xpack/esql/action/EsqlActionBreakerIT.java | 18 ++++++++++++++++++ .../xpack/esql/plugin/EsqlPlugin.java | 16 +++++++++++++--- .../TransportEsqlAsyncGetResultsAction.java | 5 +++-- .../plugin/TransportEsqlAsyncStopAction.java | 5 +++-- .../esql/plugin/TransportEsqlQueryAction.java | 10 +++++----- 8 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactoryProvider.java diff --git a/muted-tests.yml b/muted-tests.yml index c5a962741eb0..759a13acfa54 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -295,8 +295,6 @@ tests: - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT method: testSearchWithRandomDisconnects issue: https://github.com/elastic/elasticsearch/issues/122707 -- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT - issue: https://github.com/elastic/elasticsearch/issues/122810 - class: org.elasticsearch.snapshots.DedicatedClusterSnapshotRestoreIT method: testRestoreShrinkIndex issue: https://github.com/elastic/elasticsearch/issues/121717 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactoryProvider.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactoryProvider.java new file mode 100644 index 000000000000..4c851cc226d3 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactoryProvider.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +public record BlockFactoryProvider(BlockFactory blockFactory) { + +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java index 90bf34b49939..971a2a7705c1 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java @@ -17,7 +17,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockFactoryProvider; import org.elasticsearch.compute.operator.exchange.ExchangeService; +import org.elasticsearch.compute.test.MockBlockFactory; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.health.node.selection.HealthNode; @@ -60,6 +62,22 @@ public abstract class AbstractEsqlIntegTestCase extends ESIntegTestCase { } public void ensureBlocksReleased() { + for (String node : internalCluster().getNodeNames()) { + BlockFactoryProvider blockFactoryProvider = internalCluster().getInstance(BlockFactoryProvider.class, node); + try { + if (blockFactoryProvider.blockFactory() instanceof MockBlockFactory mockBlockFactory) { + assertBusy(() -> { + try { + mockBlockFactory.ensureAllBlocksAreReleased(); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + } + } catch (Exception e) { + throw new RuntimeException("failed to check mock factory", e); + } + } for (String node : internalCluster().getNodeNames()) { CircuitBreakerService breakerService = internalCluster().getInstance(CircuitBreakerService.class, node); CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.REQUEST); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index 57f6b55d3184..ef4a2c1e0036 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -12,18 +12,23 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockFactoryProvider; import org.elasticsearch.compute.operator.exchange.ExchangeService; +import org.elasticsearch.compute.test.MockBlockFactory; import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import java.util.ArrayList; import java.util.Collection; @@ -50,6 +55,8 @@ public class EsqlActionBreakerIT extends EsqlActionIT { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(InternalExchangePlugin.class); plugins.add(InternalTransportSettingPlugin.class); + assertTrue(plugins.removeIf(p -> p.isAssignableFrom(EsqlPlugin.class))); + plugins.add(EsqlTestPluginWithMockBlockFactory.class); return plugins; } @@ -79,6 +86,17 @@ public class EsqlActionBreakerIT extends EsqlActionIT { .build(); } + public static class EsqlTestPluginWithMockBlockFactory extends EsqlPlugin { + @Override + protected BlockFactoryProvider blockFactoryProvider( + CircuitBreaker breaker, + BigArrays bigArrays, + ByteSizeValue maxPrimitiveArraySize + ) { + return new BlockFactoryProvider(new MockBlockFactory(breaker, bigArrays, maxPrimitiveArraySize)); + } + } + private EsqlQueryResponse runWithBreaking(EsqlQueryRequest request) throws CircuitBreakingException { setRequestCircuitBreakerLimit(ByteSizeValue.ofBytes(between(256, 2048))); try { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 4379e2e8041a..7b7b056741e2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockFactoryProvider; import org.elasticsearch.compute.data.BlockWritables; import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; @@ -111,15 +112,24 @@ public class EsqlPlugin extends Plugin implements ActionPlugin { BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE ); BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking(); - BlockFactory blockFactory = new BlockFactory(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize); + var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize); setupSharedSecrets(); return List.of( new PlanExecutor(new IndexResolver(services.client()), services.telemetryProvider().getMeterRegistry(), getLicenseState()), - new ExchangeService(services.clusterService().getSettings(), services.threadPool(), ThreadPool.Names.SEARCH, blockFactory), - blockFactory + new ExchangeService( + services.clusterService().getSettings(), + services.threadPool(), + ThreadPool.Names.SEARCH, + blockFactoryProvider.blockFactory() + ), + blockFactoryProvider ); } + protected BlockFactoryProvider blockFactoryProvider(CircuitBreaker breaker, BigArrays bigArrays, ByteSizeValue maxPrimitiveArraySize) { + return new BlockFactoryProvider(new BlockFactory(breaker, bigArrays, maxPrimitiveArraySize)); + } + private void setupSharedSecrets() { try { // EsqlQueryRequestBuilder. initializes the shared secret access diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java index 5658db059918..cc917dfa7a30 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockFactoryProvider; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -44,7 +45,7 @@ public class TransportEsqlAsyncGetResultsAction extends AbstractTransportQlAsync Client client, ThreadPool threadPool, BigArrays bigArrays, - BlockFactory blockFactory + BlockFactoryProvider blockFactoryProvider ) { super( EsqlAsyncGetResultAction.NAME, @@ -57,7 +58,7 @@ public class TransportEsqlAsyncGetResultsAction extends AbstractTransportQlAsync bigArrays, EsqlQueryTask.class ); - this.blockFactory = blockFactory; + this.blockFactory = blockFactoryProvider.blockFactory(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java index a1a3072c69b8..d5670d9876f1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncStopAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockFactoryProvider; import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -64,13 +65,13 @@ public class TransportEsqlAsyncStopAction extends HandledTransportAction( XPackPlugin.ASYNC_RESULTS_INDEX, From 48d20839fb21df68fea18acec7fe49d03216b1c6 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 03:44:53 +1100 Subject: [PATCH 18/31] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to c66fdaf (#123385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | docker.elastic.co/wolfi/chainguard-base | digest | `d74b1fd` -> `c66fdaf` | --- ### Configuration 📅 **Schedule**: Branch creation - "after 1pm on tuesday" (UTC), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://redirect.github.com/renovatebot/renovate). --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index b0129d26a818..ec0c2521ac40 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -22,7 +22,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c66fdafe581a6ab1668a962015de4ce4666a60ed601d24f019f03bb4aaab8eeb", "-wolfi", "apk" ), From 88cf2487e7deb38686870e40bf08b2b729b7d848 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 25 Feb 2025 10:46:22 -0600 Subject: [PATCH 19/31] Fixing serialization of ScriptStats cache_evictions_history (#123384) --- docs/changelog/123384.yaml | 5 +++ .../org/elasticsearch/script/ScriptStats.java | 3 +- .../script/ScriptStatsTests.java | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/123384.yaml diff --git a/docs/changelog/123384.yaml b/docs/changelog/123384.yaml new file mode 100644 index 000000000000..33d42b79c41e --- /dev/null +++ b/docs/changelog/123384.yaml @@ -0,0 +1,5 @@ +pr: 123384 +summary: Fixing serialization of `ScriptStats` `cache_evictions_history` +area: Stats +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java index a25d9587bcb0..08ce12232a95 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.collect.Iterators.single; +import static org.elasticsearch.script.ScriptContextStats.Fields.CACHE_EVICTIONS_HISTORY; import static org.elasticsearch.script.ScriptContextStats.Fields.COMPILATIONS_HISTORY; import static org.elasticsearch.script.ScriptStats.Fields.CACHE_EVICTIONS; import static org.elasticsearch.script.ScriptStats.Fields.COMPILATIONS; @@ -205,7 +206,7 @@ public record ScriptStats( builder.endObject(); } if (cacheEvictionsHistory != null && cacheEvictionsHistory.areTimingsEmpty() == false) { - builder.startObject(COMPILATIONS_HISTORY); + builder.startObject(CACHE_EVICTIONS_HISTORY); cacheEvictionsHistory.toXContent(builder, params); builder.endObject(); } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java index df81e8ebcbb1..b60afca0939a 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -78,6 +78,37 @@ public class ScriptStatsTests extends ESTestCase { assertThat(Strings.toString(builder), equalTo(expected)); } + public void testXContentChunkedHistory() throws Exception { + ScriptStats stats = new ScriptStats(5, 6, 7, new TimeSeries(10, 20, 30, 40), new TimeSeries(100, 200, 300, 400)); + final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + + builder.startObject(); + for (var it = stats.toXContentChunked(ToXContent.EMPTY_PARAMS); it.hasNext();) { + it.next().toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endObject(); + String expected = """ + { + "script" : { + "compilations" : 5, + "cache_evictions" : 6, + "compilation_limit_triggered" : 7, + "compilations_history" : { + "5m" : 10, + "15m" : 20, + "24h" : 30 + }, + "cache_evictions_history" : { + "5m" : 100, + "15m" : 200, + "24h" : 300 + }, + "contexts" : [ ] + } + }"""; + assertThat(Strings.toString(builder), equalTo(expected)); + } + public void testSerializeEmptyTimeSeries() throws IOException { ScriptContextStats stats = new ScriptContextStats("c", 3333, new TimeSeries(1111), new TimeSeries(2222)); From b5e482bc9182f2e3feaa9623593199cc9191e131 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Feb 2025 09:58:39 -0800 Subject: [PATCH 20/31] Enable entitlements by default (#122907) Entitlements are almost complete. This commit enables them by default, in preparation for 8.18/9.0. --- .../java/org/elasticsearch/server/cli/SystemJvmOptions.java | 2 +- .../entitlement/initialization/EntitlementInitialization.java | 2 ++ .../main/java/org/elasticsearch/bootstrap/Elasticsearch.java | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 97c70837c218..d6e9ac8d9c9e 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -28,7 +28,7 @@ final class SystemJvmOptions { static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { String distroType = sysprops.get("es.distribution.type"); boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot"); - boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false")); + boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "true")); // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled; return Stream.of( diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 055660d54079..64d84989b25c 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -178,6 +178,7 @@ public class EntitlementInitialization { if (bootstrapArgs.pidFile() != null) { serverModuleFileDatas.add(FileData.ofPath(bootstrapArgs.pidFile(), READ_WRITE)); } + Collections.addAll( serverScopes, new Scope( @@ -186,6 +187,7 @@ public class EntitlementInitialization { new CreateClassLoaderEntitlement(), new FilesEntitlement( List.of( + // TODO: what in es.base is accessing shared repo? FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE), FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE) ) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 5ccb4af45ade..79f15bb20817 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -120,9 +120,9 @@ class Elasticsearch { final PrintStream out = getStdout(); final PrintStream err = getStderr(); final ServerArgs args; - final boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(System.getProperty("es.entitlements.enabled", "false")); + final boolean entitlementsEnabled = Booleans.parseBoolean(System.getProperty("es.entitlements.enabled", "true")); // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly - final boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled; + final boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsEnabled; try { initSecurityProperties(); From 3075b0776e29005bfa4eb1e173634f9b22d873e1 Mon Sep 17 00:00:00 2001 From: Matt Culbreth Date: Tue, 25 Feb 2025 15:05:49 -0500 Subject: [PATCH 21/31] Remove @UpdateForV9 annotations from Data Management code (#123171) * Remove `@UpdateForV9` as necessary backport has been done * Change UpdateForV9 to UpdateForV10 for ESRestTestCase due to N-2 support --- .../cluster/metadata/ComposableIndexTemplate.java | 3 --- .../org/elasticsearch/test/rest/ESRestTestCase.java | 11 ++++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 45433c4cba10..923ce3df1e51 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -373,8 +372,6 @@ public class ComposableIndexTemplate implements SimpleDiffable