Add zstd to native access (#105715)

This commit makes zstd compression available to Elasticsearch. The
library is pulled in through maven in jar files for each platform, then
bundled in a new platform directory under lib. Access to the zstd
compression/decompression is through NativeAccess.

build-tools-internal/build.gradle

@@ -119,6 +119,10 @@ gradlePlugin {
       id = 'elasticsearch.java-doc'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavadocPlugin'
     }
+    javaBase {
+      id = 'elasticsearch.java-base'
+      implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin'
+    }
     java {
       id = 'elasticsearch.java'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal
 
-import spock.lang.TempDir
+
 import spock.lang.Unroll
 import com.github.tomakehurst.wiremock.WireMockServer
 
@@ -103,10 +103,6 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
             plugins {
              id 'elasticsearch.jdk-download' apply false
             }
-
-            subprojects {
-
-            }
         """
         3.times {
             subProject(':sub-' + it) << """

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -9,6 +9,7 @@
 import org.elasticsearch.gradle.util.Pair
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.TestUtil
 import org.jetbrains.gradle.ext.JUnit
 
 import java.nio.file.Files
@@ -128,9 +129,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',
       ':server:generatePluginsList',
-      ':generateProviderImpls'].collect { elasticsearchProject.right()?.task(it) ?: it })
+      ':generateProviderImpls',
+      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 
+  // this path is produced by the extractLibs task above
+  String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
+
   idea {
     project {
       vcs = 'Git'
@@ -162,6 +167,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.library.path=' + testLibraryPath,
+              '-Djna.library.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -12,11 +12,15 @@ import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.test.TestUtil;
+import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolutionStrategy;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
@@ -26,10 +30,13 @@ import org.gradle.api.tasks.compile.AbstractCompile;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.GroovyCompile;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
 
 import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
 
 import javax.inject.Inject;
 
@@ -59,6 +66,7 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
         configureConfigurations(project);
         configureCompile(project);
         configureInputNormalization(project);
+        configureNativeLibraryPath(project);
 
         // convenience access to common versions used in dependencies
         project.getExtensions().getExtraProperties().set("versions", VersionProperties.getVersions());
@@ -165,6 +173,26 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
         project.getNormalization().getRuntimeClasspath().ignore("IMPL-JARS/**/META-INF/MANIFEST.MF");
     }
 
+    private static void configureNativeLibraryPath(Project project) {
+        String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries";
+        Configuration nativeConfig = project.getConfigurations().create("nativeLibs");
+        nativeConfig.defaultDependencies(deps -> {
+            deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default")));
+        });
+        // This input to the following lambda needs to be serializable. Configuration is not serializable, but FileCollection is.
+        FileCollection nativeConfigFiles = nativeConfig;
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            var systemProperties = test.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class);
+            var libraryPath = (Supplier<String>) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath());
+
+            test.dependsOn(nativeConfigFiles);
+            // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops
+            systemProperties.systemProperty("java.library.path", libraryPath);
+            systemProperties.systemProperty("jna.library.path", libraryPath);
+        });
+    }
+
     private static Provider<Integer> releaseVersionProviderFromCompileTask(Project project, AbstractCompile compileTask) {
         return project.provider(() -> {
             JavaVersion javaVersion = JavaVersion.toVersion(compileTask.getTargetCompatibility());

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -105,7 +105,7 @@ public class MrjarPlugin implements Plugin<Project> {
             testTask.dependsOn(jarTask);
 
             SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
-            FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
+            FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput();
             FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
             testTask.setClasspath(testRuntime.minus(mainRuntime).plus(project.files(jarTask)));
         });

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.ElasticsearchDistribution;
+
+import java.util.Locale;
+
+public class TestUtil {
+
+    public static String getTestLibraryPath(String nativeLibsDir) {
+        String arch = Architecture.current().toString().toLowerCase(Locale.ROOT);
+        String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch);
+        String existingLibraryPath = System.getProperty("java.library.path");
+
+        return String.format(Locale.ROOT, "%s/%s:%s", nativeLibsDir, platform, existingLibraryPath);
+    }
+}

build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -50,6 +50,13 @@ abstract class AbstractGradleFuncTest extends Specification {
         propertiesFile = testProjectDir.newFile('gradle.properties')
         propertiesFile <<
             "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
+
+        def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries")
+        nativeLibsProject << """
+            plugins {
+                id 'base'
+            }
+        """
     }
 
     def cleanup() {

distribution/archives/build.gradle

@@ -15,7 +15,7 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles(os, architecture)
       }
       into('config') {
         dirMode 0750

distribution/build.gradle

@@ -261,7 +261,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,6 +292,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
+    libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries')
   }
 
   project.ext {
@@ -299,7 +300,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles =
+    libFiles = { os, architecture ->
       copySpec {
         // Delay by using closures, since they have not yet been configured, so no jar task exists yet.
         from(configurations.libs)
@@ -330,7 +331,14 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         into('tools/ansi-console') {
           from(configurations.libsAnsiConsole)
         }
+        into('platform') {
+          from(configurations.libsNative)
+          if (os != null) {
+            include (os + '-' + architecture + '/*')
+          }
+        }
       }
+    }
 
     modulesFiles = { os, architecture ->
       copySpec {

distribution/packages/build.gradle

@@ -131,6 +131,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // top level "into" directive is not inherited from ospackage for some reason, so we must
     // specify it again explicitly for copying common files
+    String platform = 'linux-' + ((architecture == 'x64') ? 'x86_64' : architecture)
     into('/usr/share/elasticsearch') {
       into('bin') {
         with binFiles(type, false)
@@ -140,7 +141,7 @@ def commonPackageConfig(String type, String architecture) {
         fileMode 0644
       }
       into('lib') {
-        with libFiles
+        with libFiles('linux', architecture)
       }
       into('modules') {
         with modulesFiles('linux', architecture)

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -10,7 +10,11 @@ package org.elasticsearch.server.cli;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.SuppressForbidden;
 
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -21,6 +25,8 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
+        String libraryPath = findLibraryPath(sysprops);
+
         return Stream.of(
             /*
              * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl;
@@ -71,6 +77,8 @@ final class SystemJvmOptions {
             maybeOverrideDockerCgroup(distroType),
             maybeSetActiveProcessorCount(nodeSettings),
             setReplayFile(distroType, isHotspot),
+            "-Djava.library.path=" + libraryPath,
+            "-Djna.library.path=" + libraryPath,
             // Pass through distribution type
             "-Des.distribution.type=" + distroType
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
@@ -127,4 +135,38 @@ final class SystemJvmOptions {
         }
         return "";
     }
+
+    private static String findLibraryPath(Map<String, String> sysprops) {
+        // working dir is ES installation, so we use relative path here
+        Path platformDir = Paths.get("lib", "platform");
+        String existingPath = sysprops.get("java.library.path");
+        assert existingPath != null;
+
+        String osname = sysprops.get("os.name");
+        String os;
+        if (osname.startsWith("Windows")) {
+            os = "windows";
+        } else if (osname.startsWith("Linux")) {
+            os = "linux";
+        } else if (osname.startsWith("Mac OS")) {
+            os = "darwin";
+        } else {
+            os = "unsupported_os[" + osname + "]";
+        }
+        String archname = sysprops.get("os.arch");
+        String arch;
+        if (archname.equals("amd64")) {
+            arch = "x64";
+        } else if (archname.equals("aarch64")) {
+            arch = archname;
+        } else {
+            arch = "unsupported_arch[" + archname + "]";
+        }
+        return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath;
+    }
+
+    @SuppressForbidden(reason = "no way to get path separator with nio")
+    private static String getPathSeparator() {
+        return File.pathSeparator;
+    }
 }

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
+import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -29,10 +30,12 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
@@ -41,6 +44,15 @@ import static org.hamcrest.Matchers.not;
 @WithoutSecurityManager
 public class JvmOptionsParserTests extends ESTestCase {
 
+    private static final Map<String, String> TEST_SYSPROPS = Map.of(
+        "os.name",
+        "Linux",
+        "os.arch",
+        "aarch64",
+        "java.library.path",
+        "/usr/lib"
+    );
+
     public void testSubstitution() {
         final List<String> jvmOptions = JvmOptionsParser.substitutePlaceholders(
             List.of("-Djava.io.tmpdir=${ES_TMPDIR}"),
@@ -350,30 +362,65 @@ public class JvmOptionsParserTests extends ESTestCase {
 
     public void testNodeProcessorsActiveCount() {
         {
-            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of());
+            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, TEST_SYSPROPS);
             assertThat(jvmOptions, not(hasItem(containsString("-XX:ActiveProcessorCount="))));
         }
         {
             Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1).build();
-            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of());
+            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS);
             assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1"));
         }
         {
             // check rounding
             Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 0.2).build();
-            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of());
+            final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS);
             assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1"));
         }
         {
             // check validation
             Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 10000).build();
-            var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()));
+            var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, TEST_SYSPROPS));
             assertThat(e.getMessage(), containsString("setting [node.processors] must be <="));
         }
     }
 
     public void testCommandLineDistributionType() {
-        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of("es.distribution.type", "testdistro"));
+        var sysprops = new HashMap<>(TEST_SYSPROPS);
+        sysprops.put("es.distribution.type", "testdistro");
+        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
         assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro"));
     }
+
+    public void testLibraryPath() {
+        assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64");
+        assertLibraryPath("Mac OS", "amd64", "darwin-x64");
+        assertLibraryPath("Linux", "aarch64", "linux-aarch64");
+        assertLibraryPath("Linux", "amd64", "linux-x64");
+        assertLibraryPath("Windows", "amd64", "windows-x64");
+        assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64");
+        assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]");
+    }
+
+    private void assertLibraryPath(String os, String arch, String expected) {
+        String existingPath = "/usr/lib";
+        var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath);
+        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
+        Map<String, String> options = new HashMap<>();
+        for (var jvmOption : jvmOptions) {
+            if (jvmOption.startsWith("-D")) {
+                String[] parts = jvmOption.substring(2).split("=");
+                assert parts.length == 2;
+                options.put(parts[0], parts[1]);
+            }
+        }
+        String separator = FileSystems.getDefault().getSeparator();
+        assertThat(
+            options,
+            hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
+        );
+        assertThat(
+            options,
+            hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
+        );
+    }
 }

libs/native/build.gradle

@@ -6,12 +6,7 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.transform.UnzipTransform
-import org.elasticsearch.gradle.internal.GenerateProviderManifest
 import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask
-import org.gradle.api.internal.artifacts.ArtifactAttributes
-
-import java.util.stream.Collectors
 
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.build'

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jna;
+
+import com.sun.jna.Memory;
+
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+
+import java.nio.ByteBuffer;
+
+class JnaCloseableByteBuffer implements CloseableByteBuffer {
+    private final Memory memory;
+    private final ByteBuffer bufferView;
+
+    JnaCloseableByteBuffer(int len) {
+        this.memory = new Memory(len);
+        this.bufferView = memory.getByteBuffer(0, len);
+    }
+
+    @Override
+    public ByteBuffer buffer() {
+        return bufferView;
+    }
+
+    @Override
+    public void close() {
+        memory.close();
+    }
+}

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jna;
+
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+import org.elasticsearch.nativeaccess.lib.JavaLibrary;
+
+class JnaJavaLibrary implements JavaLibrary {
+    @Override
+    public CloseableByteBuffer newBuffer(int len) {
+        return new JnaCloseableByteBuffer(len);
+    }
+}

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java

@@ -8,14 +8,29 @@
 
 package org.elasticsearch.nativeaccess.jna;
 
+import org.elasticsearch.nativeaccess.lib.JavaLibrary;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
 import java.util.Map;
 
 public class JnaNativeLibraryProvider extends NativeLibraryProvider {
+
     public JnaNativeLibraryProvider() {
-        super("jna", Map.of(PosixCLibrary.class, JnaPosixCLibrary::new, SystemdLibrary.class, JnaSystemdLibrary::new));
+        super(
+            "jna",
+            Map.of(
+                JavaLibrary.class,
+                JnaJavaLibrary::new,
+                PosixCLibrary.class,
+                JnaPosixCLibrary::new,
+                SystemdLibrary.class,
+                JnaSystemdLibrary::new,
+                ZstdLibrary.class,
+                JnaZstdLibrary::new
+            )
+        );
     }
 }

libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jna;
+
+import com.sun.jna.Library;
+import com.sun.jna.Native;
+
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
+
+import java.nio.ByteBuffer;
+
+class JnaZstdLibrary implements ZstdLibrary {
+
+    private interface NativeFunctions extends Library {
+        long ZSTD_compressBound(int scrLen);
+
+        long ZSTD_compress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen, int compressionLevel);
+
+        boolean ZSTD_isError(long code);
+
+        String ZSTD_getErrorName(long code);
+
+        long ZSTD_decompress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen);
+    }
+
+    private final NativeFunctions functions;
+
+    JnaZstdLibrary() {
+        this.functions = Native.load("zstd", NativeFunctions.class);
+    }
+
+    @Override
+    public long compressBound(int scrLen) {
+        return functions.ZSTD_compressBound(scrLen);
+    }
+
+    @Override
+    public long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel) {
+        return functions.ZSTD_compress(dst, dst.remaining(), src, src.remaining(), compressionLevel);
+    }
+
+    @Override
+    public boolean isError(long code) {
+        return functions.ZSTD_isError(code);
+    }
+
+    @Override
+    public String getErrorName(long code) {
+        return functions.ZSTD_getErrorName(code);
+    }
+
+    @Override
+    public long decompress(ByteBuffer dst, ByteBuffer src) {
+        return functions.ZSTD_decompress(dst, dst.remaining(), src, src.remaining());
+    }
+}

libs/native/libraries/build.gradle

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.transform.UnzipTransform
+
+apply plugin: 'base'
+
+configurations {
+  libs {
+    attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE)
+    canBeConsumed = false
+  }
+}
+
+var zstdVersion = "1.5.5"
+
+repositories {
+  exclusiveContent {
+    forRepository {
+      maven {
+        url "https://artifactory.elastic.dev/artifactory/elasticsearch-zstd"
+        metadataSources {
+          artifact()
+        }
+      }
+    }
+    filter {
+      includeModule("org.elasticsearch", "zstd")
+    }
+  }
+}
+
+dependencies {
+  registerTransform(UnzipTransform, transformSpec -> {
+    transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE);
+    transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+  });
+  libs "org.elasticsearch:zstd:${zstdVersion}:darwin-aarch64"
+  libs "org.elasticsearch:zstd:${zstdVersion}:darwin-x86-64"
+  libs "org.elasticsearch:zstd:${zstdVersion}:linux-aarch64"
+  libs "org.elasticsearch:zstd:${zstdVersion}:linux-x86-64"
+  libs "org.elasticsearch:zstd:${zstdVersion}:windows-x86-64"
+}
+
+def extractLibs = tasks.register('extractLibs', Copy) {
+  from configurations.libs
+  into layout.buildDirectory.dir('platform')
+  // TODO: fix architecture in uploaded libs
+  filesMatching("*-x86-64/*") {
+    it.path = it.path.replace("x86-64", "x64")
+  }
+  filesMatching("win32*/*") {
+    it.path = it.path.replace("win32", "windows")
+  }
+}
+
+artifacts {
+  'default' extractLibs
+}

libs/native/src/main/java/module-info.java

@@ -14,7 +14,7 @@ module org.elasticsearch.nativeaccess {
     requires org.elasticsearch.base;
     requires org.elasticsearch.logging;
 
-    exports org.elasticsearch.nativeaccess to org.elasticsearch.server, org.elasticsearch.systemd;
+    exports org.elasticsearch.nativeaccess to org.elasticsearch.nativeaccess.jna, org.elasticsearch.server, org.elasticsearch.systemd;
     // allows jna to implement a library provider, and ProviderLocator to load it
     exports org.elasticsearch.nativeaccess.lib to org.elasticsearch.nativeaccess.jna, org.elasticsearch.base;
 

libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java

@@ -10,15 +10,22 @@ package org.elasticsearch.nativeaccess;
 
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.nativeaccess.lib.JavaLibrary;
+import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
 abstract class AbstractNativeAccess implements NativeAccess {
 
     protected static final Logger logger = LogManager.getLogger(NativeAccess.class);
 
     private final String name;
+    private final JavaLibrary javaLib;
+    private final Zstd zstd;
 
-    protected AbstractNativeAccess(String name) {
+    protected AbstractNativeAccess(String name, NativeLibraryProvider libraryProvider) {
         this.name = name;
+        this.javaLib = libraryProvider.getLibrary(JavaLibrary.class);
+        this.zstd = new Zstd(libraryProvider.getLibrary(ZstdLibrary.class));
     }
 
     String getName() {
@@ -29,4 +36,15 @@ abstract class AbstractNativeAccess implements NativeAccess {
     public Systemd systemd() {
         return null;
     }
+
+    @Override
+    public Zstd getZstd() {
+        return zstd;
+    }
+
+    @Override
+    public CloseableByteBuffer newBuffer(int len) {
+        assert len > 0;
+        return javaLib.newBuffer(len);
+    }
 }

libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess;
+
+import java.nio.ByteBuffer;
+
+public interface CloseableByteBuffer extends AutoCloseable {
+    ByteBuffer buffer();
+
+    @Override
+    void close();
+}

libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java

@@ -28,4 +28,12 @@ public interface NativeAccess {
     boolean definitelyRunningAsRoot();
 
     Systemd systemd();
+
+    /**
+     * Returns an accessor to zstd compression functions.
+     * @return an object used to compress and decompress bytes using zstd
+     */
+    Zstd getZstd();
+
+    CloseableByteBuffer newBuffer(int len);
 }

libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java

@@ -37,10 +37,10 @@ class NativeAccessHolder {
             logger.warn("Unable to load native provider. Native methods will be disabled.", e);
         }
         if (inst == null) {
-            inst = new NoopNativeAccess();
+            INSTANCE = new NoopNativeAccess();
         } else {
             logger.info("Using [" + libProvider.getName() + "] native provider and native methods for [" + inst.getName() + "]");
+            INSTANCE = inst;
         }
-        INSTANCE = inst;
     }
 }

libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java

@@ -8,11 +8,14 @@
 
 package org.elasticsearch.nativeaccess;
 
-class NoopNativeAccess extends AbstractNativeAccess {
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 
-    NoopNativeAccess() {
+class NoopNativeAccess implements NativeAccess {
-        super("noop");
+
-    }
+    private static final Logger logger = LogManager.getLogger(NativeAccess.class);
+
+    NoopNativeAccess() {}
 
     @Override
     public boolean definitelyRunningAsRoot() {
@@ -25,4 +28,16 @@ class NoopNativeAccess extends AbstractNativeAccess {
         logger.warn("Cannot get systemd access because native access is not available");
         return null;
     }
+
+    @Override
+    public Zstd getZstd() {
+        logger.warn("cannot compress with zstd because native access is not available");
+        return null;
+    }
+
+    @Override
+    public CloseableByteBuffer newBuffer(int len) {
+        logger.warn("cannot allocate buffer because native access is not available");
+        return null;
+    }
 }

libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java

@@ -16,7 +16,7 @@ abstract class PosixNativeAccess extends AbstractNativeAccess {
     protected final PosixCLibrary libc;
 
     PosixNativeAccess(String name, NativeLibraryProvider libraryProvider) {
-        super(name);
+        super(name, libraryProvider);
         this.libc = libraryProvider.getLibrary(PosixCLibrary.class);
     }
 

libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java

@@ -13,7 +13,7 @@ import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 class WindowsNativeAccess extends AbstractNativeAccess {
 
     WindowsNativeAccess(NativeLibraryProvider libraryProvider) {
-        super("Windows");
+        super("Windows", libraryProvider);
     }
 
     @Override

libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess;
+
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public final class Zstd {
+
+    private final ZstdLibrary zstdLib;
+
+    Zstd(ZstdLibrary zstdLib) {
+        this.zstdLib = zstdLib;
+    }
+
+    /**
+     * Compress the content of {@code src} into {@code dst} at compression level {@code level}, and return the number of compressed bytes.
+     * {@link ByteBuffer#position()} and {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified.
+     */
+    public int compress(ByteBuffer dst, ByteBuffer src, int level) {
+        Objects.requireNonNull(dst, "Null destination buffer");
+        Objects.requireNonNull(src, "Null source buffer");
+        assert dst.isDirect();
+        assert dst.isReadOnly() == false;
+        assert src.isDirect();
+        assert src.isReadOnly() == false;
+        long ret = zstdLib.compress(dst, src, level);
+        if (zstdLib.isError(ret)) {
+            throw new IllegalArgumentException(zstdLib.getErrorName(ret));
+        } else if (ret < 0 || ret > Integer.MAX_VALUE) {
+            throw new IllegalStateException("Integer overflow? ret=" + ret);
+        }
+        return (int) ret;
+    }
+
+    /**
+     * Compress the content of {@code src} into {@code dst}, and return the number of decompressed bytes. {@link ByteBuffer#position()} and
+     * {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified.
+     */
+    public int decompress(ByteBuffer dst, ByteBuffer src) {
+        Objects.requireNonNull(dst, "Null destination buffer");
+        Objects.requireNonNull(src, "Null source buffer");
+        assert dst.isDirect();
+        assert dst.isReadOnly() == false;
+        assert src.isDirect();
+        assert src.isReadOnly() == false;
+        long ret = zstdLib.decompress(dst, src);
+        if (zstdLib.isError(ret)) {
+            throw new IllegalArgumentException(zstdLib.getErrorName(ret));
+        } else if (ret < 0 || ret > Integer.MAX_VALUE) {
+            throw new IllegalStateException("Integer overflow? ret=" + ret);
+        }
+        return (int) ret;
+    }
+
+    /**
+     * Return the maximum number of compressed bytes given an input length.
+     */
+    public int compressBound(int srcLen) {
+        long ret = zstdLib.compressBound(srcLen);
+        if (zstdLib.isError(ret)) {
+            throw new IllegalArgumentException(zstdLib.getErrorName(ret));
+        } else if (ret < 0 || ret > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException(
+                srcLen
+                    + " bytes may require up to "
+                    + Long.toUnsignedString(ret)
+                    + " bytes, which overflows the maximum capacity of a ByteBuffer"
+            );
+        }
+        return (int) ret;
+    }
+}

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.lib;
+
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+
+public non-sealed interface JavaLibrary extends NativeLibrary {
+    CloseableByteBuffer newBuffer(int len);
+}

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java

@@ -9,4 +9,4 @@
 package org.elasticsearch.nativeaccess.lib;
 
 /** A marker interface for libraries that can be loaded by {@link org.elasticsearch.nativeaccess.lib.NativeLibraryProvider} */
-public sealed interface NativeLibrary permits PosixCLibrary, SystemdLibrary {}
+public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, SystemdLibrary, ZstdLibrary {}

libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.lib;
+
+import java.nio.ByteBuffer;
+
+public non-sealed interface ZstdLibrary extends NativeLibrary {
+
+    long compressBound(int scrLen);
+
+    long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel);
+
+    boolean isError(long code);
+
+    String getErrorName(long code);
+
+    long decompress(ByteBuffer dst, ByteBuffer src);
+}

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jdk;
+
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+
+import java.lang.foreign.Arena;
+import java.nio.ByteBuffer;
+
+class JdkCloseableByteBuffer implements CloseableByteBuffer {
+    private final Arena arena;
+    private final ByteBuffer bufferView;
+
+    JdkCloseableByteBuffer(int len) {
+        this.arena = Arena.ofShared();
+        this.bufferView = this.arena.allocate(len).asByteBuffer();
+    }
+
+    @Override
+    public ByteBuffer buffer() {
+        return bufferView;
+    }
+
+    @Override
+    public void close() {
+        arena.close();
+    }
+}

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jdk;
+
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+import org.elasticsearch.nativeaccess.lib.JavaLibrary;
+
+class JdkJavaLibrary implements JavaLibrary {
+    @Override
+    public CloseableByteBuffer newBuffer(int len) {
+        return new JdkCloseableByteBuffer(len);
+    }
+}

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java

@@ -8,15 +8,29 @@
 
 package org.elasticsearch.nativeaccess.jdk;
 
+import org.elasticsearch.nativeaccess.lib.JavaLibrary;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
 import java.util.Map;
 
 public class JdkNativeLibraryProvider extends NativeLibraryProvider {
 
     public JdkNativeLibraryProvider() {
-        super("jdk", Map.of(PosixCLibrary.class, JdkPosixCLibrary::new, SystemdLibrary.class, JdkSystemdLibrary::new));
+        super(
+            "jdk",
+            Map.of(
+                JavaLibrary.class,
+                JdkJavaLibrary::new,
+                PosixCLibrary.class,
+                JdkPosixCLibrary::new,
+                SystemdLibrary.class,
+                JdkSystemdLibrary::new,
+                ZstdLibrary.class,
+                JdkZstdLibrary::new
+            )
+        );
     }
 }

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java

@@ -24,6 +24,7 @@ import static java.lang.foreign.ValueLayout.JAVA_INT;
 import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle;
 
 class JdkSystemdLibrary implements SystemdLibrary {
+
     static {
         System.load(findLibSystemd());
     }
@@ -39,6 +40,7 @@ class JdkSystemdLibrary implements SystemdLibrary {
                 continue;
             }
             try (var stream = Files.walk(basepath)) {
+
                 var foundpath = stream.filter(Files::isDirectory).map(p -> p.resolve(libsystemd)).filter(Files::exists).findAny();
                 if (foundpath.isPresent()) {
                     return foundpath.get().toAbsolutePath().toString();

libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.jdk;
+
+import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
+
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.MemorySegment;
+import java.lang.invoke.MethodHandle;
+import java.nio.ByteBuffer;
+
+import static java.lang.foreign.ValueLayout.ADDRESS;
+import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN;
+import static java.lang.foreign.ValueLayout.JAVA_INT;
+import static java.lang.foreign.ValueLayout.JAVA_LONG;
+import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle;
+
+class JdkZstdLibrary implements ZstdLibrary {
+
+    static {
+        System.loadLibrary("zstd");
+    }
+
+    private static final MethodHandle compressBound$mh = downcallHandle("ZSTD_compressBound", FunctionDescriptor.of(JAVA_LONG, JAVA_INT));
+    private static final MethodHandle compress$mh = downcallHandle(
+        "ZSTD_compress",
+        FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_INT, ADDRESS, JAVA_INT, JAVA_INT)
+    );
+    private static final MethodHandle isError$mh = downcallHandle("ZSTD_isError", FunctionDescriptor.of(JAVA_BOOLEAN, JAVA_LONG));
+    private static final MethodHandle getErrorName$mh = downcallHandle("ZSTD_getErrorName", FunctionDescriptor.of(ADDRESS, JAVA_LONG));
+    private static final MethodHandle decompress$mh = downcallHandle(
+        "ZSTD_decompress",
+        FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_INT, ADDRESS, JAVA_INT)
+    );
+
+    @Override
+    public long compressBound(int srcLen) {
+        try {
+            return (long) compressBound$mh.invokeExact(srcLen);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public long compress(ByteBuffer dst, ByteBuffer src, int compressionLevel) {
+        var nativeDst = MemorySegment.ofBuffer(dst);
+        var nativeSrc = MemorySegment.ofBuffer(src);
+        try {
+            return (long) compress$mh.invokeExact(nativeDst, dst.remaining(), nativeSrc, src.remaining(), compressionLevel);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public boolean isError(long code) {
+        try {
+            return (boolean) isError$mh.invokeExact(code);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public String getErrorName(long code) {
+        try {
+            MemorySegment str = (MemorySegment) getErrorName$mh.invokeExact(code);
+            return str.reinterpret(Long.MAX_VALUE).getUtf8String(0);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public long decompress(ByteBuffer dst, ByteBuffer src) {
+        var nativeDst = MemorySegment.ofBuffer(dst);
+        var nativeSrc = MemorySegment.ofBuffer(src);
+        try {
+            return (long) decompress$mh.invokeExact(nativeDst, dst.remaining(), nativeSrc, src.remaining());
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+}

libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess;
+
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+import org.junit.BeforeClass;
+
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ZstdTests extends ESTestCase {
+
+    static NativeAccess nativeAccess;
+    static Zstd zstd;
+
+    @BeforeClass
+    public static void getZstd() {
+        nativeAccess = NativeAccess.instance();
+        zstd = nativeAccess.getZstd();
+    }
+
+    public void testCompressBound() {
+        assertThat(zstd.compressBound(0), Matchers.greaterThanOrEqualTo(1));
+        assertThat(zstd.compressBound(100), Matchers.greaterThanOrEqualTo(100));
+        expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(Integer.MAX_VALUE));
+        expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(-1));
+        expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(-100));
+        expectThrows(IllegalArgumentException.class, () -> zstd.compressBound(Integer.MIN_VALUE));
+    }
+
+    public void testCompressValidation() {
+        try (var src = nativeAccess.newBuffer(1000); var dst = nativeAccess.newBuffer(500)) {
+            var srcBuf = src.buffer();
+            var dstBuf = dst.buffer();
+
+            var npe1 = expectThrows(NullPointerException.class, () -> zstd.compress(null, srcBuf, 0));
+            assertThat(npe1.getMessage(), equalTo("Null destination buffer"));
+            var npe2 = expectThrows(NullPointerException.class, () -> zstd.compress(dstBuf, null, 0));
+            assertThat(npe2.getMessage(), equalTo("Null source buffer"));
+
+            // dst capacity too low
+            for (int i = 0; i < srcBuf.remaining(); ++i) {
+                srcBuf.put(i, randomByte());
+            }
+            var e = expectThrows(IllegalArgumentException.class, () -> zstd.compress(dstBuf, srcBuf, 0));
+            assertThat(e.getMessage(), equalTo("Destination buffer is too small"));
+        }
+    }
+
+    public void testDecompressValidation() {
+        try (
+            var original = nativeAccess.newBuffer(1000);
+            var compressed = nativeAccess.newBuffer(500);
+            var restored = nativeAccess.newBuffer(500)
+        ) {
+            var originalBuf = original.buffer();
+            var compressedBuf = compressed.buffer();
+
+            var npe1 = expectThrows(NullPointerException.class, () -> zstd.decompress(null, originalBuf));
+            assertThat(npe1.getMessage(), equalTo("Null destination buffer"));
+            var npe2 = expectThrows(NullPointerException.class, () -> zstd.decompress(compressedBuf, null));
+            assertThat(npe2.getMessage(), equalTo("Null source buffer"));
+
+            // Invalid compressed format
+            for (int i = 0; i < originalBuf.remaining(); ++i) {
+                originalBuf.put(i, (byte) i);
+            }
+            var e = expectThrows(IllegalArgumentException.class, () -> zstd.decompress(compressedBuf, originalBuf));
+            assertThat(e.getMessage(), equalTo("Unknown frame descriptor"));
+
+            int compressedLength = zstd.compress(compressedBuf, originalBuf, 0);
+            compressedBuf.limit(compressedLength);
+            e = expectThrows(IllegalArgumentException.class, () -> zstd.decompress(restored.buffer(), compressedBuf));
+            assertThat(e.getMessage(), equalTo("Destination buffer is too small"));
+
+        }
+    }
+
+    public void testOneByte() {
+        doTestRoundtrip(new byte[] { 'z' });
+    }
+
+    public void testConstant() {
+        byte[] b = new byte[randomIntBetween(100, 1000)];
+        Arrays.fill(b, randomByte());
+        doTestRoundtrip(b);
+    }
+
+    public void testCycle() {
+        byte[] b = new byte[randomIntBetween(100, 1000)];
+        for (int i = 0; i < b.length; ++i) {
+            b[i] = (byte) (i & 0x0F);
+        }
+        doTestRoundtrip(b);
+    }
+
+    private void doTestRoundtrip(byte[] data) {
+        try (
+            var original = nativeAccess.newBuffer(data.length);
+            var compressed = nativeAccess.newBuffer(zstd.compressBound(data.length));
+            var restored = nativeAccess.newBuffer(data.length)
+        ) {
+            original.buffer().put(0, data);
+            int compressedLength = zstd.compress(compressed.buffer(), original.buffer(), randomIntBetween(-3, 9));
+            compressed.buffer().limit(compressedLength);
+            int decompressedLength = zstd.decompress(restored.buffer(), compressed.buffer());
+            assertThat(restored.buffer(), equalTo(original.buffer()));
+            assertThat(decompressedLength, equalTo(data.length));
+        }
+
+        // Now with non-zero offsets
+        final int compressedOffset = randomIntBetween(1, 1000);
+        final int decompressedOffset = randomIntBetween(1, 1000);
+        try (
+            var original = nativeAccess.newBuffer(decompressedOffset + data.length);
+            var compressed = nativeAccess.newBuffer(compressedOffset + zstd.compressBound(data.length));
+            var restored = nativeAccess.newBuffer(decompressedOffset + data.length)
+        ) {
+            original.buffer().put(decompressedOffset, data);
+            original.buffer().position(decompressedOffset);
+            compressed.buffer().position(compressedOffset);
+            int compressedLength = zstd.compress(compressed.buffer(), original.buffer(), randomIntBetween(-3, 9));
+            compressed.buffer().limit(compressedOffset + compressedLength);
+            restored.buffer().position(decompressedOffset);
+            int decompressedLength = zstd.decompress(restored.buffer(), compressed.buffer());
+            assertThat(
+                restored.buffer().slice(decompressedOffset, data.length),
+                equalTo(original.buffer().slice(decompressedOffset, data.length))
+            );
+            assertThat(decompressedLength, equalTo(data.length));
+        }
+    }
+}

settings.gradle

@@ -154,6 +154,7 @@ project(":libs").children.each { libsProject ->
     lp.name = lp.name  // for :libs:elasticsearch-x-content:impl
   }
 }
+project(":libs:elasticsearch-native:libraries").name = "elasticsearch-native-libraries"
 
 project(":qa:stable-api").children.each { libsProject ->
   libsProject.name = "elasticsearch-${libsProject.name}"