From 405b88b882f6279fbd46dcec4413e403eec77a8c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 13 Mar 2024 09:45:12 -0700 Subject: [PATCH] Add zstd to native access (#105715) This commit makes zstd compression available to Elasticsearch. The library is pulled in through maven in jar files for each platform, then bundled in a new platform directory under lib. Access to the zstd compression/decompression is through NativeAccess. --- build-tools-internal/build.gradle | 4 + .../internal/JdkDownloadPluginFuncTest.groovy | 6 +- .../src/main/groovy/elasticsearch.ide.gradle | 9 +- .../internal/ElasticsearchJavaBasePlugin.java | 28 ++++ .../gradle/internal/MrjarPlugin.java | 2 +- .../gradle/internal/test/TestUtil.java | 25 ++++ .../fixtures/AbstractGradleFuncTest.groovy | 7 + distribution/archives/build.gradle | 2 +- distribution/build.gradle | 12 +- distribution/packages/build.gradle | 3 +- .../server/cli/SystemJvmOptions.java | 42 ++++++ .../server/cli/JvmOptionsParserTests.java | 57 ++++++- libs/native/build.gradle | 5 - .../jna/JnaCloseableByteBuffer.java | 35 +++++ .../nativeaccess/jna/JnaJavaLibrary.java | 19 +++ .../jna/JnaNativeLibraryProvider.java | 17 ++- .../nativeaccess/jna/JnaZstdLibrary.java | 62 ++++++++ libs/native/libraries/build.gradle | 64 ++++++++ libs/native/src/main/java/module-info.java | 2 +- .../nativeaccess/AbstractNativeAccess.java | 20 ++- .../nativeaccess/CloseableByteBuffer.java | 18 +++ .../nativeaccess/NativeAccess.java | 8 + .../nativeaccess/NativeAccessHolder.java | 4 +- .../nativeaccess/NoopNativeAccess.java | 23 ++- .../nativeaccess/PosixNativeAccess.java | 2 +- .../nativeaccess/WindowsNativeAccess.java | 2 +- .../org/elasticsearch/nativeaccess/Zstd.java | 81 ++++++++++ .../nativeaccess/lib/JavaLibrary.java | 15 ++ .../nativeaccess/lib/NativeLibrary.java | 2 +- .../nativeaccess/lib/ZstdLibrary.java | 24 +++ .../jdk/JdkCloseableByteBuffer.java | 34 +++++ .../nativeaccess/jdk/JdkJavaLibrary.java | 19 +++ .../jdk/JdkNativeLibraryProvider.java | 16 +- .../nativeaccess/jdk/JdkSystemdLibrary.java | 2 + .../nativeaccess/jdk/JdkZstdLibrary.java | 91 +++++++++++ .../elasticsearch/nativeaccess/ZstdTests.java | 141 ++++++++++++++++++ settings.gradle | 1 + 37 files changed, 870 insertions(+), 34 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java create mode 100644 libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java create mode 100644 libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java create mode 100644 libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java create mode 100644 libs/native/libraries/build.gradle create mode 100644 libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java create mode 100644 libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java create mode 100644 libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java create mode 100644 libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java create mode 100644 libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java create mode 100644 libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java create mode 100644 libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java create mode 100644 libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 758cdf687e6b..24647c366c45 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -119,6 +119,10 @@ gradlePlugin { id = 'elasticsearch.java-doc' implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavadocPlugin' } + javaBase { + id = 'elasticsearch.java-base' + implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin' + } java { id = 'elasticsearch.java' implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy index 67a04ebc5b7a..24131c633e9d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy @@ -8,7 +8,7 @@ package org.elasticsearch.gradle.internal -import spock.lang.TempDir + import spock.lang.Unroll import com.github.tomakehurst.wiremock.WireMockServer @@ -103,10 +103,6 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.jdk-download' apply false } - - subprojects { - - } """ 3.times { subProject(':sub-' + it) << """ diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index ce068d4ca649..ccbe9cd2f4a2 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -9,6 +9,7 @@ import org.elasticsearch.gradle.util.Pair import org.elasticsearch.gradle.util.GradleUtils import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.TestUtil import org.jetbrains.gradle.ext.JUnit import java.nio.file.Files @@ -128,9 +129,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { ':x-pack:plugin:esql:compute:gen:jar', ':server:generateModulesList', ':server:generatePluginsList', - ':generateProviderImpls'].collect { elasticsearchProject.right()?.task(it) ?: it }) + ':generateProviderImpls', + ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs'].collect { elasticsearchProject.right()?.task(it) ?: it }) } + // this path is produced by the extractLibs task above + String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform") + idea { project { vcs = 'Git' @@ -162,6 +167,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { '-ea', '-Djava.security.manager=allow', '-Djava.locale.providers=SPI,COMPAT', + '-Djava.library.path=' + testLibraryPath, + '-Djna.library.path=' + testLibraryPath, // TODO: only open these for mockito when it is modularized '--add-opens=java.base/java.security.cert=ALL-UNNAMED', '--add-opens=java.base/java.nio.channels=ALL-UNNAMED', diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index e224b16bf588..dbdb065858f4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -12,11 +12,15 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin; import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.test.TestUtil; +import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolutionStrategy; +import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.Provider; @@ -26,10 +30,13 @@ import org.gradle.api.tasks.compile.AbstractCompile; import org.gradle.api.tasks.compile.CompileOptions; import org.gradle.api.tasks.compile.GroovyCompile; import org.gradle.api.tasks.compile.JavaCompile; +import org.gradle.api.tasks.testing.Test; import org.gradle.jvm.toolchain.JavaLanguageVersion; import org.gradle.jvm.toolchain.JavaToolchainService; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import javax.inject.Inject; @@ -59,6 +66,7 @@ public class ElasticsearchJavaBasePlugin implements Plugin { configureConfigurations(project); configureCompile(project); configureInputNormalization(project); + configureNativeLibraryPath(project); // convenience access to common versions used in dependencies project.getExtensions().getExtraProperties().set("versions", VersionProperties.getVersions()); @@ -165,6 +173,26 @@ public class ElasticsearchJavaBasePlugin implements Plugin { project.getNormalization().getRuntimeClasspath().ignore("IMPL-JARS/**/META-INF/MANIFEST.MF"); } + private static void configureNativeLibraryPath(Project project) { + String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries"; + Configuration nativeConfig = project.getConfigurations().create("nativeLibs"); + nativeConfig.defaultDependencies(deps -> { + deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default"))); + }); + // This input to the following lambda needs to be serializable. Configuration is not serializable, but FileCollection is. + FileCollection nativeConfigFiles = nativeConfig; + + project.getTasks().withType(Test.class).configureEach(test -> { + var systemProperties = test.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class); + var libraryPath = (Supplier) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath()); + + test.dependsOn(nativeConfigFiles); + // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops + systemProperties.systemProperty("java.library.path", libraryPath); + systemProperties.systemProperty("jna.library.path", libraryPath); + }); + } + private static Provider releaseVersionProviderFromCompileTask(Project project, AbstractCompile compileTask) { return project.provider(() -> { JavaVersion javaVersion = JavaVersion.toVersion(compileTask.getTargetCompatibility()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java index 46fa38a44f56..a9b332c3cfd3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java @@ -105,7 +105,7 @@ public class MrjarPlugin implements Plugin { testTask.dependsOn(jarTask); SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project); - FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath(); + FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput(); FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath(); testTask.setClasspath(testRuntime.minus(mainRuntime).plus(project.files(jarTask))); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java new file mode 100644 index 000000000000..53742b78accb --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test; + +import org.elasticsearch.gradle.Architecture; +import org.elasticsearch.gradle.ElasticsearchDistribution; + +import java.util.Locale; + +public class TestUtil { + + public static String getTestLibraryPath(String nativeLibsDir) { + String arch = Architecture.current().toString().toLowerCase(Locale.ROOT); + String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch); + String existingLibraryPath = System.getProperty("java.library.path"); + + return String.format(Locale.ROOT, "%s/%s:%s", nativeLibsDir, platform, existingLibraryPath); + } +} diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 8218829fe017..49e942746219 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -50,6 +50,13 @@ abstract class AbstractGradleFuncTest extends Specification { propertiesFile = testProjectDir.newFile('gradle.properties') propertiesFile << "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" + + def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries") + nativeLibsProject << """ + plugins { + id 'base' + } + """ } def cleanup() { diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 0508f29ef595..4d7850477dbf 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -15,7 +15,7 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b return copySpec { into("elasticsearch-${version}") { into('lib') { - with libFiles + with libFiles(os, architecture) } into('config') { dirMode 0750 diff --git a/distribution/build.gradle b/distribution/build.gradle index c8cc60b6facf..c3f9192ecee0 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -261,7 +261,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole'].each { + ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each { create(it) { canBeConsumed = false canBeResolved = true @@ -292,6 +292,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libsKeystoreCli project(path: ':distribution:tools:keystore-cli') libsSecurityCli project(':x-pack:plugin:security:cli') libsGeoIpCli project(':distribution:tools:geoip-cli') + libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries') } project.ext { @@ -299,7 +300,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { /***************************************************************************** * Common files in all distributions * *****************************************************************************/ - libFiles = + libFiles = { os, architecture -> copySpec { // Delay by using closures, since they have not yet been configured, so no jar task exists yet. from(configurations.libs) @@ -330,7 +331,14 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { into('tools/ansi-console') { from(configurations.libsAnsiConsole) } + into('platform') { + from(configurations.libsNative) + if (os != null) { + include (os + '-' + architecture + '/*') + } + } } + } modulesFiles = { os, architecture -> copySpec { diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 1983736e4ee9..6b57f32310c9 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -131,6 +131,7 @@ def commonPackageConfig(String type, String architecture) { // top level "into" directive is not inherited from ospackage for some reason, so we must // specify it again explicitly for copying common files + String platform = 'linux-' + ((architecture == 'x64') ? 'x86_64' : architecture) into('/usr/share/elasticsearch') { into('bin') { with binFiles(type, false) @@ -140,7 +141,7 @@ def commonPackageConfig(String type, String architecture) { fileMode 0644 } into('lib') { - with libFiles + with libFiles('linux', architecture) } into('modules') { with modulesFiles('linux', architecture) diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 850ee3fc71a2..0e95021a3af7 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -10,7 +10,11 @@ package org.elasticsearch.server.cli; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.SuppressForbidden; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -21,6 +25,8 @@ final class SystemJvmOptions { static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { String distroType = sysprops.get("es.distribution.type"); boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot"); + String libraryPath = findLibraryPath(sysprops); + return Stream.of( /* * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl; @@ -71,6 +77,8 @@ final class SystemJvmOptions { maybeOverrideDockerCgroup(distroType), maybeSetActiveProcessorCount(nodeSettings), setReplayFile(distroType, isHotspot), + "-Djava.library.path=" + libraryPath, + "-Djna.library.path=" + libraryPath, // Pass through distribution type "-Des.distribution.type=" + distroType ).filter(e -> e.isEmpty() == false).collect(Collectors.toList()); @@ -127,4 +135,38 @@ final class SystemJvmOptions { } return ""; } + + private static String findLibraryPath(Map sysprops) { + // working dir is ES installation, so we use relative path here + Path platformDir = Paths.get("lib", "platform"); + String existingPath = sysprops.get("java.library.path"); + assert existingPath != null; + + String osname = sysprops.get("os.name"); + String os; + if (osname.startsWith("Windows")) { + os = "windows"; + } else if (osname.startsWith("Linux")) { + os = "linux"; + } else if (osname.startsWith("Mac OS")) { + os = "darwin"; + } else { + os = "unsupported_os[" + osname + "]"; + } + String archname = sysprops.get("os.arch"); + String arch; + if (archname.equals("amd64")) { + arch = "x64"; + } else if (archname.equals("aarch64")) { + arch = archname; + } else { + arch = "unsupported_arch[" + archname + "]"; + } + return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath; + } + + @SuppressForbidden(reason = "no way to get path separator with nio") + private static String getPathSeparator() { + return File.pathSeparator; + } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java index 101be4301b52..c24623c75b5c 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.ESTestCase.WithoutSecurityManager; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -29,10 +30,12 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; @@ -41,6 +44,15 @@ import static org.hamcrest.Matchers.not; @WithoutSecurityManager public class JvmOptionsParserTests extends ESTestCase { + private static final Map TEST_SYSPROPS = Map.of( + "os.name", + "Linux", + "os.arch", + "aarch64", + "java.library.path", + "/usr/lib" + ); + public void testSubstitution() { final List jvmOptions = JvmOptionsParser.substitutePlaceholders( List.of("-Djava.io.tmpdir=${ES_TMPDIR}"), @@ -350,30 +362,65 @@ public class JvmOptionsParserTests extends ESTestCase { public void testNodeProcessorsActiveCount() { { - final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of()); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, TEST_SYSPROPS); assertThat(jvmOptions, not(hasItem(containsString("-XX:ActiveProcessorCount=")))); } { Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check rounding Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 0.2).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check validation Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 10000).build(); - var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of())); + var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS)); assertThat(e.getMessage(), containsString("setting [node.processors] must be <=")); } } public void testCommandLineDistributionType() { - final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of("es.distribution.type", "testdistro")); + var sysprops = new HashMap<>(TEST_SYSPROPS); + sysprops.put("es.distribution.type", "testdistro"); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops); assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro")); } + + public void testLibraryPath() { + assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64"); + assertLibraryPath("Mac OS", "amd64", "darwin-x64"); + assertLibraryPath("Linux", "aarch64", "linux-aarch64"); + assertLibraryPath("Linux", "amd64", "linux-x64"); + assertLibraryPath("Windows", "amd64", "windows-x64"); + assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64"); + assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]"); + } + + private void assertLibraryPath(String os, String arch, String expected) { + String existingPath = "/usr/lib"; + var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops); + Map options = new HashMap<>(); + for (var jvmOption : jvmOptions) { + if (jvmOption.startsWith("-D")) { + String[] parts = jvmOption.substring(2).split("="); + assert parts.length == 2; + options.put(parts[0], parts[1]); + } + } + String separator = FileSystems.getDefault().getSeparator(); + assertThat( + options, + hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath))) + ); + assertThat( + options, + hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath))) + ); + } } diff --git a/libs/native/build.gradle b/libs/native/build.gradle index 83a169ce7c2d..dbe546619c7a 100644 --- a/libs/native/build.gradle +++ b/libs/native/build.gradle @@ -6,12 +6,7 @@ * Side Public License, v 1. */ -import org.elasticsearch.gradle.transform.UnzipTransform -import org.elasticsearch.gradle.internal.GenerateProviderManifest import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask -import org.gradle.api.internal.artifacts.ArtifactAttributes - -import java.util.stream.Collectors apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.build' diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java new file mode 100644 index 000000000000..e47b17e23470 --- /dev/null +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jna; + +import com.sun.jna.Memory; + +import org.elasticsearch.nativeaccess.CloseableByteBuffer; + +import java.nio.ByteBuffer; + +class JnaCloseableByteBuffer implements CloseableByteBuffer { + private final Memory memory; + private final ByteBuffer bufferView; + + JnaCloseableByteBuffer(int len) { + this.memory = new Memory(len); + this.bufferView = memory.getByteBuffer(0, len); + } + + @Override + public ByteBuffer buffer() { + return bufferView; + } + + @Override + public void close() { + memory.close(); + } +} diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java new file mode 100644 index 000000000000..852696886368 --- /dev/null +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jna; + +import org.elasticsearch.nativeaccess.CloseableByteBuffer; +import org.elasticsearch.nativeaccess.lib.JavaLibrary; + +class JnaJavaLibrary implements JavaLibrary { + @Override + public CloseableByteBuffer newBuffer(int len) { + return new JnaCloseableByteBuffer(len); + } +} diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java index 7d43cb2e3d4b..8ffa3121f3e5 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java @@ -8,14 +8,29 @@ package org.elasticsearch.nativeaccess.jna; +import org.elasticsearch.nativeaccess.lib.JavaLibrary; import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; import org.elasticsearch.nativeaccess.lib.PosixCLibrary; import org.elasticsearch.nativeaccess.lib.SystemdLibrary; +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; import java.util.Map; public class JnaNativeLibraryProvider extends NativeLibraryProvider { + public JnaNativeLibraryProvider() { - super("jna", Map.of(PosixCLibrary.class, JnaPosixCLibrary::new, SystemdLibrary.class, JnaSystemdLibrary::new)); + super( + "jna", + Map.of( + JavaLibrary.class, + JnaJavaLibrary::new, + PosixCLibrary.class, + JnaPosixCLibrary::new, + SystemdLibrary.class, + JnaSystemdLibrary::new, + ZstdLibrary.class, + JnaZstdLibrary::new + ) + ); } } diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java new file mode 100644 index 000000000000..f0581633ea96 --- /dev/null +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jna; + +import com.sun.jna.Library; +import com.sun.jna.Native; + +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; + +import java.nio.ByteBuffer; + +class JnaZstdLibrary implements ZstdLibrary { + + private interface NativeFunctions extends Library { + long ZSTD_compressBound(int scrLen); + + long ZSTD_compress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen, int compressionLevel); + + boolean ZSTD_isError(long code); + + String ZSTD_getErrorName(long code); + + long ZSTD_decompress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen); + } + + private final NativeFunctions functions; + + JnaZstdLibrary() { + this.functions = Native.load("zstd", NativeFunctions.class); + } + + @Override + public long compressBound(int scrLen) { + return functions.ZSTD_compressBound(scrLen); + } + + @Override + public long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel) { + return functions.ZSTD_compress(dst, dst.remaining(), src, src.remaining(), compressionLevel); + } + + @Override + public boolean isError(long code) { + return functions.ZSTD_isError(code); + } + + @Override + public String getErrorName(long code) { + return functions.ZSTD_getErrorName(code); + } + + @Override + public long decompress(ByteBuffer dst, ByteBuffer src) { + return functions.ZSTD_decompress(dst, dst.remaining(), src, src.remaining()); + } +} diff --git a/libs/native/libraries/build.gradle b/libs/native/libraries/build.gradle new file mode 100644 index 000000000000..23d2b6e2219d --- /dev/null +++ b/libs/native/libraries/build.gradle @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.transform.UnzipTransform + +apply plugin: 'base' + +configurations { + libs { + attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE) + canBeConsumed = false + } +} + +var zstdVersion = "1.5.5" + +repositories { + exclusiveContent { + forRepository { + maven { + url "https://artifactory.elastic.dev/artifactory/elasticsearch-zstd" + metadataSources { + artifact() + } + } + } + filter { + includeModule("org.elasticsearch", "zstd") + } + } +} + +dependencies { + registerTransform(UnzipTransform, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE); + transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + libs "org.elasticsearch:zstd:${zstdVersion}:darwin-aarch64" + libs "org.elasticsearch:zstd:${zstdVersion}:darwin-x86-64" + libs "org.elasticsearch:zstd:${zstdVersion}:linux-aarch64" + libs "org.elasticsearch:zstd:${zstdVersion}:linux-x86-64" + libs "org.elasticsearch:zstd:${zstdVersion}:windows-x86-64" +} + +def extractLibs = tasks.register('extractLibs', Copy) { + from configurations.libs + into layout.buildDirectory.dir('platform') + // TODO: fix architecture in uploaded libs + filesMatching("*-x86-64/*") { + it.path = it.path.replace("x86-64", "x64") + } + filesMatching("win32*/*") { + it.path = it.path.replace("win32", "windows") + } +} + +artifacts { + 'default' extractLibs +} diff --git a/libs/native/src/main/java/module-info.java b/libs/native/src/main/java/module-info.java index ea049ff888cb..46f6d8244359 100644 --- a/libs/native/src/main/java/module-info.java +++ b/libs/native/src/main/java/module-info.java @@ -14,7 +14,7 @@ module org.elasticsearch.nativeaccess { requires org.elasticsearch.base; requires org.elasticsearch.logging; - exports org.elasticsearch.nativeaccess to org.elasticsearch.server, org.elasticsearch.systemd; + exports org.elasticsearch.nativeaccess to org.elasticsearch.nativeaccess.jna, org.elasticsearch.server, org.elasticsearch.systemd; // allows jna to implement a library provider, and ProviderLocator to load it exports org.elasticsearch.nativeaccess.lib to org.elasticsearch.nativeaccess.jna, org.elasticsearch.base; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java index fa23966dbeb7..764dc7c67c9e 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java @@ -10,15 +10,22 @@ package org.elasticsearch.nativeaccess; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; +import org.elasticsearch.nativeaccess.lib.JavaLibrary; +import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; abstract class AbstractNativeAccess implements NativeAccess { protected static final Logger logger = LogManager.getLogger(NativeAccess.class); private final String name; + private final JavaLibrary javaLib; + private final Zstd zstd; - protected AbstractNativeAccess(String name) { + protected AbstractNativeAccess(String name, NativeLibraryProvider libraryProvider) { this.name = name; + this.javaLib = libraryProvider.getLibrary(JavaLibrary.class); + this.zstd = new Zstd(libraryProvider.getLibrary(ZstdLibrary.class)); } String getName() { @@ -29,4 +36,15 @@ abstract class AbstractNativeAccess implements NativeAccess { public Systemd systemd() { return null; } + + @Override + public Zstd getZstd() { + return zstd; + } + + @Override + public CloseableByteBuffer newBuffer(int len) { + assert len > 0; + return javaLib.newBuffer(len); + } } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java new file mode 100644 index 000000000000..aa5d94080afa --- /dev/null +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess; + +import java.nio.ByteBuffer; + +public interface CloseableByteBuffer extends AutoCloseable { + ByteBuffer buffer(); + + @Override + void close(); +} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java index 77b638690d1b..5b2be93dadc1 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java @@ -28,4 +28,12 @@ public interface NativeAccess { boolean definitelyRunningAsRoot(); Systemd systemd(); + + /** + * Returns an accessor to zstd compression functions. + * @return an object used to compress and decompress bytes using zstd + */ + Zstd getZstd(); + + CloseableByteBuffer newBuffer(int len); } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java index 6abbe02c4786..562e7163cd09 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java @@ -37,10 +37,10 @@ class NativeAccessHolder { logger.warn("Unable to load native provider. Native methods will be disabled.", e); } if (inst == null) { - inst = new NoopNativeAccess(); + INSTANCE = new NoopNativeAccess(); } else { logger.info("Using [" + libProvider.getName() + "] native provider and native methods for [" + inst.getName() + "]"); + INSTANCE = inst; } - INSTANCE = inst; } } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java index 6eb6145699fe..c13fc97324ea 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java @@ -8,11 +8,14 @@ package org.elasticsearch.nativeaccess; -class NoopNativeAccess extends AbstractNativeAccess { +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; - NoopNativeAccess() { - super("noop"); - } +class NoopNativeAccess implements NativeAccess { + + private static final Logger logger = LogManager.getLogger(NativeAccess.class); + + NoopNativeAccess() {} @Override public boolean definitelyRunningAsRoot() { @@ -25,4 +28,16 @@ class NoopNativeAccess extends AbstractNativeAccess { logger.warn("Cannot get systemd access because native access is not available"); return null; } + + @Override + public Zstd getZstd() { + logger.warn("cannot compress with zstd because native access is not available"); + return null; + } + + @Override + public CloseableByteBuffer newBuffer(int len) { + logger.warn("cannot allocate buffer because native access is not available"); + return null; + } } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java index 050f9e89a067..99dde99c67af 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java @@ -16,7 +16,7 @@ abstract class PosixNativeAccess extends AbstractNativeAccess { protected final PosixCLibrary libc; PosixNativeAccess(String name, NativeLibraryProvider libraryProvider) { - super(name); + super(name, libraryProvider); this.libc = libraryProvider.getLibrary(PosixCLibrary.class); } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java index 86d3952e1504..7ea3bb65130b 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java @@ -13,7 +13,7 @@ import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; class WindowsNativeAccess extends AbstractNativeAccess { WindowsNativeAccess(NativeLibraryProvider libraryProvider) { - super("Windows"); + super("Windows", libraryProvider); } @Override diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java new file mode 100644 index 000000000000..6a0d348d5251 --- /dev/null +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess; + +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; + +import java.nio.ByteBuffer; +import java.util.Objects; + +public final class Zstd { + + private final ZstdLibrary zstdLib; + + Zstd(ZstdLibrary zstdLib) { + this.zstdLib = zstdLib; + } + + /** + * Compress the content of {@code src} into {@code dst} at compression level {@code level}, and return the number of compressed bytes. + * {@link ByteBuffer#position()} and {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. + */ + public int compress(ByteBuffer dst, ByteBuffer src, int level) { + Objects.requireNonNull(dst, "Null destination buffer"); + Objects.requireNonNull(src, "Null source buffer"); + assert dst.isDirect(); + assert dst.isReadOnly() == false; + assert src.isDirect(); + assert src.isReadOnly() == false; + long ret = zstdLib.compress(dst, src, level); + if (zstdLib.isError(ret)) { + throw new IllegalArgumentException(zstdLib.getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalStateException("Integer overflow? ret=" + ret); + } + return (int) ret; + } + + /** + * Compress the content of {@code src} into {@code dst}, and return the number of decompressed bytes. {@link ByteBuffer#position()} and + * {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. + */ + public int decompress(ByteBuffer dst, ByteBuffer src) { + Objects.requireNonNull(dst, "Null destination buffer"); + Objects.requireNonNull(src, "Null source buffer"); + assert dst.isDirect(); + assert dst.isReadOnly() == false; + assert src.isDirect(); + assert src.isReadOnly() == false; + long ret = zstdLib.decompress(dst, src); + if (zstdLib.isError(ret)) { + throw new IllegalArgumentException(zstdLib.getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalStateException("Integer overflow? ret=" + ret); + } + return (int) ret; + } + + /** + * Return the maximum number of compressed bytes given an input length. + */ + public int compressBound(int srcLen) { + long ret = zstdLib.compressBound(srcLen); + if (zstdLib.isError(ret)) { + throw new IllegalArgumentException(zstdLib.getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + srcLen + + " bytes may require up to " + + Long.toUnsignedString(ret) + + " bytes, which overflows the maximum capacity of a ByteBuffer" + ); + } + return (int) ret; + } +} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java new file mode 100644 index 000000000000..50a3022fa77c --- /dev/null +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.lib; + +import org.elasticsearch.nativeaccess.CloseableByteBuffer; + +public non-sealed interface JavaLibrary extends NativeLibrary { + CloseableByteBuffer newBuffer(int len); +} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java index cf2116440a8b..1fb868e1c389 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java @@ -9,4 +9,4 @@ package org.elasticsearch.nativeaccess.lib; /** A marker interface for libraries that can be loaded by {@link org.elasticsearch.nativeaccess.lib.NativeLibraryProvider} */ -public sealed interface NativeLibrary permits PosixCLibrary, SystemdLibrary {} +public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, SystemdLibrary, ZstdLibrary {} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java new file mode 100644 index 000000000000..feb1dbe8e3d6 --- /dev/null +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.lib; + +import java.nio.ByteBuffer; + +public non-sealed interface ZstdLibrary extends NativeLibrary { + + long compressBound(int scrLen); + + long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel); + + boolean isError(long code); + + String getErrorName(long code); + + long decompress(ByteBuffer dst, ByteBuffer src); +} diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java new file mode 100644 index 000000000000..d802fd8be7a6 --- /dev/null +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jdk; + +import org.elasticsearch.nativeaccess.CloseableByteBuffer; + +import java.lang.foreign.Arena; +import java.nio.ByteBuffer; + +class JdkCloseableByteBuffer implements CloseableByteBuffer { + private final Arena arena; + private final ByteBuffer bufferView; + + JdkCloseableByteBuffer(int len) { + this.arena = Arena.ofShared(); + this.bufferView = this.arena.allocate(len).asByteBuffer(); + } + + @Override + public ByteBuffer buffer() { + return bufferView; + } + + @Override + public void close() { + arena.close(); + } +} diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java new file mode 100644 index 000000000000..60a3966463a7 --- /dev/null +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jdk; + +import org.elasticsearch.nativeaccess.CloseableByteBuffer; +import org.elasticsearch.nativeaccess.lib.JavaLibrary; + +class JdkJavaLibrary implements JavaLibrary { + @Override + public CloseableByteBuffer newBuffer(int len) { + return new JdkCloseableByteBuffer(len); + } +} diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java index b808dc315105..35cc16653de3 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java @@ -8,15 +8,29 @@ package org.elasticsearch.nativeaccess.jdk; +import org.elasticsearch.nativeaccess.lib.JavaLibrary; import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; import org.elasticsearch.nativeaccess.lib.PosixCLibrary; import org.elasticsearch.nativeaccess.lib.SystemdLibrary; +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; import java.util.Map; public class JdkNativeLibraryProvider extends NativeLibraryProvider { public JdkNativeLibraryProvider() { - super("jdk", Map.of(PosixCLibrary.class, JdkPosixCLibrary::new, SystemdLibrary.class, JdkSystemdLibrary::new)); + super( + "jdk", + Map.of( + JavaLibrary.class, + JdkJavaLibrary::new, + PosixCLibrary.class, + JdkPosixCLibrary::new, + SystemdLibrary.class, + JdkSystemdLibrary::new, + ZstdLibrary.class, + JdkZstdLibrary::new + ) + ); } } diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java index 682b94b6f4f7..745b93ac918d 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java @@ -24,6 +24,7 @@ import static java.lang.foreign.ValueLayout.JAVA_INT; import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle; class JdkSystemdLibrary implements SystemdLibrary { + static { System.load(findLibSystemd()); } @@ -39,6 +40,7 @@ class JdkSystemdLibrary implements SystemdLibrary { continue; } try (var stream = Files.walk(basepath)) { + var foundpath = stream.filter(Files::isDirectory).map(p -> p.resolve(libsystemd)).filter(Files::exists).findAny(); if (foundpath.isPresent()) { return foundpath.get().toAbsolutePath().toString(); diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java new file mode 100644 index 000000000000..632240a84425 --- /dev/null +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.jdk; + +import org.elasticsearch.nativeaccess.lib.ZstdLibrary; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.MemorySegment; +import java.lang.invoke.MethodHandle; +import java.nio.ByteBuffer; + +import static java.lang.foreign.ValueLayout.ADDRESS; +import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN; +import static java.lang.foreign.ValueLayout.JAVA_INT; +import static java.lang.foreign.ValueLayout.JAVA_LONG; +import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle; + +class JdkZstdLibrary implements ZstdLibrary { + + static { + System.loadLibrary("zstd"); + } + + private static final MethodHandle compressBound$mh = downcallHandle("ZSTD_compressBound", FunctionDescriptor.of(JAVA_LONG, JAVA_INT)); + private static final MethodHandle compress$mh = downcallHandle( + "ZSTD_compress", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_INT, ADDRESS, JAVA_INT, JAVA_INT) + ); + private static final MethodHandle isError$mh = downcallHandle("ZSTD_isError", FunctionDescriptor.of(JAVA_BOOLEAN, JAVA_LONG)); + private static final MethodHandle getErrorName$mh = downcallHandle("ZSTD_getErrorName", FunctionDescriptor.of(ADDRESS, JAVA_LONG)); + private static final MethodHandle decompress$mh = downcallHandle( + "ZSTD_decompress", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_INT, ADDRESS, JAVA_INT) + ); + + @Override + public long compressBound(int srcLen) { + try { + return (long) compressBound$mh.invokeExact(srcLen); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel) { + var nativeDst = MemorySegment.ofBuffer(dst); + var nativeSrc = MemorySegment.ofBuffer(src); + try { + return (long) compress$mh.invokeExact(nativeDst, dst.remaining(), nativeSrc, src.remaining(), compressionLevel); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public boolean isError(long code) { + try { + return (boolean) isError$mh.invokeExact(code); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public String getErrorName(long code) { + try { + MemorySegment str = (MemorySegment) getErrorName$mh.invokeExact(code); + return str.reinterpret(Long.MAX_VALUE).getUtf8String(0); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public long decompress(ByteBuffer dst, ByteBuffer src) { + var nativeDst = MemorySegment.ofBuffer(dst); + var nativeSrc = MemorySegment.ofBuffer(src); + try { + return (long) decompress$mh.invokeExact(nativeDst, dst.remaining(), nativeSrc, src.remaining()); + } catch (Throwable t) { + throw new AssertionError(t); + } + } +} diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java new file mode 100644 index 000000000000..d051961b06c5 --- /dev/null +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.BeforeClass; + +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class ZstdTests extends ESTestCase { + + static NativeAccess nativeAccess; + static Zstd zstd; + + @BeforeClass + public static void getZstd() { + nativeAccess = NativeAccess.instance(); + zstd = nativeAccess.getZstd(); + } + + public void testCompressBound() { + assertThat(zstd.compressBound(0), Matchers.greaterThanOrEqualTo(1)); + assertThat(zstd.compressBound(100), Matchers.greaterThanOrEqualTo(100)); + expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(Integer.MAX_VALUE)); + expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(-1)); + expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(-100)); + expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(Integer.MIN_VALUE)); + } + + public void testCompressValidation() { + try (var src = nativeAccess.newBuffer(1000); var dst = nativeAccess.newBuffer(500)) { + var srcBuf = src.buffer(); + var dstBuf = dst.buffer(); + + var npe1 = expectThrows(NullPointerException.class, () -> zstd.compress(null, srcBuf, 0)); + assertThat(npe1.getMessage(), equalTo("Null destination buffer")); + var npe2 = expectThrows(NullPointerException.class, () -> zstd.compress(dstBuf, null, 0)); + assertThat(npe2.getMessage(), equalTo("Null source buffer")); + + // dst capacity too low + for (int i = 0; i < srcBuf.remaining(); ++i) { + srcBuf.put(i, randomByte()); + } + var e = expectThrows(IllegalArgumentException.class, () -> zstd.compress(dstBuf, srcBuf, 0)); + assertThat(e.getMessage(), equalTo("Destination buffer is too small")); + } + } + + public void testDecompressValidation() { + try ( + var original = nativeAccess.newBuffer(1000); + var compressed = nativeAccess.newBuffer(500); + var restored = nativeAccess.newBuffer(500) + ) { + var originalBuf = original.buffer(); + var compressedBuf = compressed.buffer(); + + var npe1 = expectThrows(NullPointerException.class, () -> zstd.decompress(null, originalBuf)); + assertThat(npe1.getMessage(), equalTo("Null destination buffer")); + var npe2 = expectThrows(NullPointerException.class, () -> zstd.decompress(compressedBuf, null)); + assertThat(npe2.getMessage(), equalTo("Null source buffer")); + + // Invalid compressed format + for (int i = 0; i < originalBuf.remaining(); ++i) { + originalBuf.put(i, (byte) i); + } + var e = expectThrows(IllegalArgumentException.class, () -> zstd.decompress(compressedBuf, originalBuf)); + assertThat(e.getMessage(), equalTo("Unknown frame descriptor")); + + int compressedLength = zstd.compress(compressedBuf, originalBuf, 0); + compressedBuf.limit(compressedLength); + e = expectThrows(IllegalArgumentException.class, () -> zstd.decompress(restored.buffer(), compressedBuf)); + assertThat(e.getMessage(), equalTo("Destination buffer is too small")); + + } + } + + public void testOneByte() { + doTestRoundtrip(new byte[] { 'z' }); + } + + public void testConstant() { + byte[] b = new byte[randomIntBetween(100, 1000)]; + Arrays.fill(b, randomByte()); + doTestRoundtrip(b); + } + + public void testCycle() { + byte[] b = new byte[randomIntBetween(100, 1000)]; + for (int i = 0; i < b.length; ++i) { + b[i] = (byte) (i & 0x0F); + } + doTestRoundtrip(b); + } + + private void doTestRoundtrip(byte[] data) { + try ( + var original = nativeAccess.newBuffer(data.length); + var compressed = nativeAccess.newBuffer(zstd.compressBound(data.length)); + var restored = nativeAccess.newBuffer(data.length) + ) { + original.buffer().put(0, data); + int compressedLength = zstd.compress(compressed.buffer(), original.buffer(), randomIntBetween(-3, 9)); + compressed.buffer().limit(compressedLength); + int decompressedLength = zstd.decompress(restored.buffer(), compressed.buffer()); + assertThat(restored.buffer(), equalTo(original.buffer())); + assertThat(decompressedLength, equalTo(data.length)); + } + + // Now with non-zero offsets + final int compressedOffset = randomIntBetween(1, 1000); + final int decompressedOffset = randomIntBetween(1, 1000); + try ( + var original = nativeAccess.newBuffer(decompressedOffset + data.length); + var compressed = nativeAccess.newBuffer(compressedOffset + zstd.compressBound(data.length)); + var restored = nativeAccess.newBuffer(decompressedOffset + data.length) + ) { + original.buffer().put(decompressedOffset, data); + original.buffer().position(decompressedOffset); + compressed.buffer().position(compressedOffset); + int compressedLength = zstd.compress(compressed.buffer(), original.buffer(), randomIntBetween(-3, 9)); + compressed.buffer().limit(compressedOffset + compressedLength); + restored.buffer().position(decompressedOffset); + int decompressedLength = zstd.decompress(restored.buffer(), compressed.buffer()); + assertThat( + restored.buffer().slice(decompressedOffset, data.length), + equalTo(original.buffer().slice(decompressedOffset, data.length)) + ); + assertThat(decompressedLength, equalTo(data.length)); + } + } +} diff --git a/settings.gradle b/settings.gradle index c183971bc12c..97cce0a476d9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -154,6 +154,7 @@ project(":libs").children.each { libsProject -> lp.name = lp.name // for :libs:elasticsearch-x-content:impl } } +project(":libs:elasticsearch-native:libraries").name = "elasticsearch-native-libraries" project(":qa:stable-api").children.each { libsProject -> libsProject.name = "elasticsearch-${libsProject.name}"