First attempt at running on darwin aarch64 (#67103)

This PR is a first attempt to get the build to run on an Apple M1 (ARM 64 / aarch64) machine.
I think the changes are mostly reasonable, apart from some hard-coding to use the Azul JVM,
which at the time of writing seems to be the only available JVM. I'll follow up when our preferred
JVM is available.

buildSrc/build.gradle

@@ -174,7 +174,9 @@ if (project != rootProject) {
     distribution project(':distribution:archives:windows-zip')
     distribution project(':distribution:archives:oss-windows-zip')
     distribution project(':distribution:archives:darwin-tar')
+    distribution project(':distribution:archives:darwin-aarch64-tar')
     distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:oss-darwin-aarch64-tar')
     distribution project(':distribution:archives:linux-aarch64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -177,11 +177,14 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
 
         Version distroVersion = Version.fromString(distribution.getVersion());
         String extension = distribution.getType().toString();
-        String classifier = ":x86_64";
+        String classifier = ":" + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
             if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":" + distribution.getPlatform() + "-x86_64";
+                classifier = ":"
+                    + distribution.getPlatform()
+                    + "-"
+                    + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
             } else {
                 classifier = "";
             }

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -34,7 +34,7 @@ import java.util.regex.Pattern;
 public class Jdk implements Buildable, Iterable<File> {
 
     private static final List<String> ALLOWED_ARCHITECTURES = List.of("aarch64", "x64");
-    private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
+    private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk", "azul");
     private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -36,6 +36,7 @@ public class JdkDownloadPlugin implements Plugin<Project> {
 
     public static final String VENDOR_ADOPTOPENJDK = "adoptopenjdk";
     public static final String VENDOR_OPENJDK = "openjdk";
+    public static final String VENDOR_AZUL = "azul";
 
     private static final String REPO_NAME_PREFIX = "jdk_repo_";
     private static final String EXTENSION_NAME = "jdks";
@@ -131,6 +132,22 @@ public class JdkDownloadPlugin implements Plugin<Project> {
                     + jdk.getBuild()
                     + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
             }
+        } else if (jdk.getVendor().equals(VENDOR_AZUL)) {
+            repoUrl = "https://cdn.azul.com";
+
+            // The following is an absolute hack until AdoptOpenJdk provides Apple aarch64 builds
+            switch (jdk.getMajor()) {
+                case "15":
+                    artifactPattern = "zulu/bin/zulu" + jdk.getMajor() + ".28.1013-ca-jdk15.0.1-macosx_[classifier].[ext]";
+                    break;
+
+                case "11":
+                    artifactPattern = "zulu/bin/zulu" + jdk.getMajor() + ".43.1021-ca-jdk11.0.9.1-macosx_[classifier].[ext]";
+                    break;
+
+                default:
+                    throw new GradleException("Unknown Azul JDK major version  [" + jdk.getMajor() + "]");
+            }
         } else {
             throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
         }

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -143,7 +143,8 @@ public class InternalDistributionBwcSetupPlugin implements InternalPlugin {
         String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch";
 
         String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName.substring(artifactFileName.length() - 3);
-        int archIndex = artifactFileName.indexOf("x86_64");
+        int x86ArchIndex = artifactFileName.indexOf("x86_64");
+        int aarch64ArchIndex = artifactFileName.indexOf("aarch64");
 
         bwcProject.getConfigurations().create(distributionProject.name);
         bwcProject.getArtifacts().add(distributionProject.name, distFile, artifact -> {
@@ -152,9 +153,12 @@ public class InternalDistributionBwcSetupPlugin implements InternalPlugin {
             artifact.setType(suffix);
 
             String classifier = "";
-            if (archIndex != -1) {
-                int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2);
-                classifier = "-" + artifactFileName.substring(osIndex + 1, archIndex - 1) + "-x86_64";
+            if (x86ArchIndex != -1) {
+                int osIndex = artifactFileName.lastIndexOf('-', x86ArchIndex - 2);
+                classifier = "-" + artifactFileName.substring(osIndex + 1, x86ArchIndex - 1) + "-x86_64";
+            } else if (aarch64ArchIndex != -1) {
+                int osIndex = artifactFileName.lastIndexOf('-', aarch64ArchIndex - 2);
+                classifier = "-" + artifactFileName.substring(osIndex + 1, aarch64ArchIndex - 1) + "-aarch64";
             }
             artifact.setClassifier(classifier);
         });
@@ -166,7 +170,18 @@ public class InternalDistributionBwcSetupPlugin implements InternalPlugin {
         projects.addAll(asList("deb", "rpm", "oss-deb", "oss-rpm"));
 
         if (bwcVersion.onOrAfter("7.0.0")) { // starting with 7.0 we bundle a jdk which means we have platform-specific archives
-            projects.addAll(asList("oss-windows-zip", "windows-zip", "oss-darwin-tar", "darwin-tar", "oss-linux-tar", "linux-tar"));
+            projects.addAll(
+                asList(
+                    "oss-windows-zip",
+                    "windows-zip",
+                    "oss-darwin-tar",
+                    "oss-darwin-aarch64-tar",
+                    "darwin-tar",
+                    "darwin-aarch64-tar",
+                    "oss-linux-tar",
+                    "linux-tar"
+                )
+            );
         } else { // prior to 7.0 we published only a single zip and tar archives for oss and default distributions
             projects.addAll(asList("oss-zip", "zip", "tar", "oss-tar"));
         }
@@ -179,7 +194,7 @@ public class InternalDistributionBwcSetupPlugin implements InternalPlugin {
                 if (name.contains("zip") || name.contains("tar")) {
                     int index = name.lastIndexOf('-');
                     String baseName = name.startsWith("oss-") ? name.substring(4, index) : name.substring(0, index);
-                    classifier = "-" + baseName + "-x86_64";
+                    classifier = "-" + baseName + (name.contains("aarch64") ? "-aarch64" : "-x86_64");
                     extension = name.substring(index + 1);
                     if (extension.equals("tar")) {
                         extension += ".gz";

buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java

@@ -47,7 +47,7 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
             "11.0.2+33",
             "linux",
             "x64",
-            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
+            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk, azul]"
         );
     }
 

distribution/archives/build.gradle

@@ -115,6 +115,13 @@ distribution_archives {
     }
   }
 
+  darwinAarch64Tar {
+    archiveClassifier = 'darwin-aarch64'
+    content {
+      archiveFiles(modulesFiles(false, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, true)
+    }
+  }
+
   ossDarwinTar {
     archiveClassifier = 'darwin-x86_64'
     content {
@@ -122,6 +129,13 @@ distribution_archives {
     }
   }
 
+  ossDarwinAarch64Tar {
+    archiveClassifier = 'darwin-aarch64'
+    content {
+      archiveFiles(modulesFiles(true, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', true, true)
+    }
+  }
+
   noJdkDarwinTar {
     archiveClassifier = 'no-jdk-darwin-x86_64'
     content {
@@ -136,6 +150,20 @@ distribution_archives {
     }
   }
 
+  noJdkDarwinAarch64Tar {
+    archiveClassifier = 'no-jdk-darwin-aarch64'
+    content {
+      archiveFiles(modulesFiles(false, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, false)
+    }
+  }
+
+  ossNoJdkDarwinAarch64Tar {
+    archiveClassifier = 'no-jdk-darwin-aarch64'
+    content {
+      archiveFiles(modulesFiles(true, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', true, false)
+    }
+  }
+
   linuxAarch64Tar {
     archiveClassifier = 'linux-aarch64'
     content {
@@ -184,4 +212,4 @@ subprojects {
   apply plugin: 'elasticsearch.internal-distribution-archive-check'
 
   group = "org.elasticsearch.distribution.${name.startsWith("oss-") ? "oss" : "default"}"
-}
+}

distribution/archives/darwin-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/no-jdk-darwin-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/oss-darwin-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/oss-no-jdk-darwin-aarch64-tar/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/build.gradle

@@ -304,11 +304,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
   // Setup all required JDKs
   project.jdks {
     ['darwin', 'windows', 'linux'].each { platform ->
-      (platform == 'linux' ? ['x64', 'aarch64'] : ['x64']).each { architecture ->
+      (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64'] : ['x64']).each { architecture ->
         "bundled_${platform}_${architecture}" {
           it.platform = platform
           it.version = VersionProperties.getBundledJdk(platform)
-          it.vendor = VersionProperties.bundledJdkVendor
+          it.vendor = (platform == 'darwin' && architecture == 'aarch64') ? 'azul' : VersionProperties.bundledJdkVendor
           it.architecture = architecture
         }
       }
@@ -369,7 +369,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     modulesFiles = { oss, platform ->
       copySpec {
         eachFile {
-          if (it.relativePath.segments[-2] == 'bin' || (platform == 'darwin-x86_64' && it.relativePath.segments[-2] == 'MacOS')) {
+          if (it.relativePath.segments[-2] == 'bin' || ((platform == 'darwin-x86_64' || platform == 'darwin-aarch64') && it.relativePath.segments[-2] == 'MacOS')) {
             // bin files, wherever they are within modules (eg platform specific) should be executable
             // and MacOS is an alternative to bin on macOS
             it.mode = 0755
@@ -383,7 +383,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         } else {
           buildModules = buildDefaultModulesTaskProvider
         }
-        List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64']
+        List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64']
         if (platform != null) {
           excludePlatforms.remove(excludePlatforms.indexOf(platform))
         } else {
@@ -675,6 +675,7 @@ subprojects {
 
 ['archives:windows-zip', 'archives:oss-windows-zip',
  'archives:darwin-tar', 'archives:oss-darwin-tar',
+ 'archives:darwin-aarch64-tar', 'archives:oss-darwin-aarch64-tar',
  'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
  'archives:linux-tar', 'archives:oss-linux-tar',
  'archives:integ-test-zip',

settings.gradle

@@ -24,8 +24,12 @@ List projects = [
   'distribution:archives:no-jdk-windows-zip',
   'distribution:archives:oss-darwin-tar',
   'distribution:archives:darwin-tar',
+  'distribution:archives:darwin-aarch64-tar',
+  'distribution:archives:oss-darwin-aarch64-tar',
   'distribution:archives:oss-no-jdk-darwin-tar',
   'distribution:archives:no-jdk-darwin-tar',
+  'distribution:archives:oss-no-jdk-darwin-aarch64-tar',
+  'distribution:archives:no-jdk-darwin-aarch64-tar',
   'distribution:archives:oss-linux-aarch64-tar',
   'distribution:archives:oss-linux-tar',
   'distribution:archives:linux-aarch64-tar',