Bundle JDK (AdoptOpenJDK 11) in Logstash artifacts (ARM64)

Changed Linux creation artifacts (tar.gz/deb/rpm) to include the ARM JDK.
Extracted common parts of artifact.rake into functions to be shared between ARM and Intel bundling tasks

build.gradle

@@ -541,9 +541,9 @@ class JDKDetails {
     private final String extension
     final String localPackageName
     final String unpackedJdkName
-    private String arch = "x64"
+    private String arch
 
-    JDKDetails(versionYml, osName) {
+    JDKDetails(versionYml, osName, jdkArch) {
         revision = versionYml.bundled_jdk.revision
         build = versionYml.bundled_jdk.build
         vendor = versionYml.bundled_jdk.vendor
@@ -557,8 +557,9 @@ class JDKDetails {
             default:
                 extension = "tar.gz"
         }
+        arch = parseJdkArchitecture(jdkArch)
         unpackedJdkName = "jdk-${revision}-${osName}"
-        localPackageName = "${unpackedJdkName}.${extension}"
+        localPackageName = "${unpackedJdkName}-${arch}.${extension}"
     }
 
     String createDownloadUrl() {
@@ -579,16 +580,30 @@ class JDKDetails {
             return "mac"
         return osName
     }
+
+    private String parseJdkArchitecture(String jdkArch) {
+        switch (jdkArch) {
+            case "x86_64":
+                return "x64"
+                break
+            case "arm64":
+                return "aarch64"
+                break
+            default:
+                throw RuntimeException("Can't handle CPU architechture: ${jdkArch}")
+        }
+    }
 }
 
 tasks.register("downloadJdk", Download) {
-    // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin]
+    // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] -Pjdk_arch=[arm64|x86_64]
 
     project.ext.set("versionFound", true)
     String osName = selectOsType()
 
     def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text)
-    def jdkDetails = new JDKDetails(versionYml, osName)
+    String jdkArch = project.ext.jdk_arch
+    def jdkDetails = new JDKDetails(versionYml, osName, jdkArch)
 
     description "Download JDK ${jdkDetails.major}, OS: ${osName}"
 
@@ -663,7 +678,7 @@ tasks.register("decompressJdk") {
 tasks.register("copyJdk", Copy) {
     dependsOn = [decompressJdk, bootstrap]
     description = "Download, unpack and copy the JDK"
-    // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin]
+    // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] -Pjdk_arch=[arm64|x86_64]
     doLast {
         System.out.println "Download location is ${project.ext.jdkDownloadLocation}, Decompressing ${project.ext.jdkDirectory} to \"${project.ext.jdkBundlingDirectory}\""
     }

qa/rspec/commands/debian.rb

@@ -37,7 +37,11 @@ module ServiceTester
     end
 
     def architecture_extension
+      if java.lang.System.getProperty("os.arch") == "amd64"
         "amd64"
+      else
+        "arm64"
+      end
     end
 
     def install(package, host=nil)

qa/rspec/commands/redhat.rb

@@ -37,7 +37,11 @@ module ServiceTester
     end
 
     def architecture_extension
+      if java.lang.System.getProperty("os.arch") == "amd64"
         "x86_64"
+      else
+        "aarch64"
+      end
     end
 
     def install(package, host=nil)

rakelib/artifacts.rake

@@ -135,24 +135,37 @@ namespace "artifact" do
   desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies"
   task "archives" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
-    ["linux", "windows", "darwin"].each do |os_name|
+    license_details = ['ELASTIC-LICENSE']
-      puts("[artifact:archives] Building tar.gz/zip of default plugins for OS: #{os_name}")
+    create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
-      system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}")
+    create_archive_pack(license_details, "arm64", "linux")
-      case os_name
-      when "linux"
-        build_tar('ELASTIC-LICENSE', platform: '-linux-x86_64')
-      when "windows"
-        build_zip('ELASTIC-LICENSE', platform: '-windows-x86_64')
-      when "darwin"
-        build_tar('ELASTIC-LICENSE', platform: '-darwin-x86_64')
-      end
-      system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}")
-    end
 
     #without JDK
     system("./gradlew bootstrap") #force the build of Logstash jars
-    build_tar('ELASTIC-LICENSE', platform: '-no-jdk')
+    build_tar(*license_details, platform: '-no-jdk')
-    build_zip('ELASTIC-LICENSE', platform: '-no-jdk')
+    build_zip(*license_details, platform: '-no-jdk')
+  end
+
+  def create_archive_pack(license_details, arch, *oses)
+    oses.each do |os_name|
+      puts("[artifact:archives] Building tar.gz/zip of default plugins for OS: #{os_name}, arch: #{arch}")
+      create_single_archive_pack(os_name, arch, license_details)
+    end
+  end
+
+  def create_single_archive_pack(os_name, arch, license_details)
+    system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name} -Pjdk_arch=#{arch}")
+    if arch == 'arm64'
+      arch = 'aarch64'
+    end
+    case os_name
+    when "linux"
+      build_tar(*license_details, platform: "-linux-#{arch}")
+    when "windows"
+      build_zip(*license_details, platform: "-windows-#{arch}")
+    when "darwin"
+      build_tar(*license_details, platform: "-darwin-#{arch}")
+    end
+    system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}")
   end
 
   desc "Build a not JDK bundled tar.gz of default logstash plugins with all dependencies"
@@ -163,74 +176,71 @@ namespace "artifact" do
   desc "Build all (jdk bundled and not) OSS tar.gz and zip of default logstash plugins with all dependencies"
   task "archives_oss" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
-    ["linux", "windows", "darwin"].each do |os_name|
+    license_details = ['APACHE-LICENSE-2.0',"-oss", oss_excluder]
-      puts("[artifact:archives_oss] Building OSS tar.gz/zip of default plugins for OS: #{os_name}")
+    create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
-      system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}")
+    create_archive_pack(license_details, "arm64", "linux")
-      case os_name
-      when "linux"
-        build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-linux-x86_64')
-      when "windows"
-        build_zip('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-windows-x86_64')
-      when "darwin"
-        build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-darwin-x86_64')
-      end
-      system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}")
-    end
 
     #without JDK
     system("./gradlew bootstrap") #force the build of Logstash jars
-    build_tar('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk')
+    build_tar(*license_details, platform: '-no-jdk')
-    build_zip('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk')
+    build_zip(*license_details, platform: '-no-jdk')
   end
 
   desc "Build an RPM of logstash with all dependencies"
   task "rpm" => ["prepare", "generate_build_metadata"] do
-    puts("[artifact:rpm] building rpm package")
+    #with bundled JDKs
-    system("./gradlew copyJdk -Pjdk_bundle_os=linux")
+    puts("[artifact:rpm] building rpm package x86_64")
-    package_with_jdk("centos", "5")
+    package_with_jdk("centos", "x86_64")
-    system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux')
+
+    puts("[artifact:rpm] building rpm package arm64")
+    package_with_jdk("centos", "arm64")
 
     #without JDKs
     system("./gradlew bootstrap") #force the build of Logstash jars
-    package("centos", "5")
+    package("centos")
   end
 
   desc "Build an RPM of logstash with all dependencies"
   task "rpm_oss" => ["prepare", "generate_build_metadata"] do
-    puts("[artifact:rpm] building rpm package")
+    #with bundled JDKs
-    system("./gradlew copyJdk -Pjdk_bundle_os=linux")
+    puts("[artifact:rpm] building rpm OSS package x86_64")
-    package_with_jdk("centos", "5", :oss)
+    package_with_jdk("centos", "x86_64", :oss)
-    system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux')
+
+    puts("[artifact:rpm] building rpm OSS package arm64")
+    package_with_jdk("centos", "arm64", :oss)
 
     #without JDKs
     system("./gradlew bootstrap") #force the build of Logstash jars
-    package("centos", "5", :oss)
+    package("centos", :oss)
   end
 
 
   desc "Build a DEB of logstash with all dependencies"
   task "deb" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
-    puts("[artifact:deb] building deb package for OS: linux")
+    puts("[artifact:deb] building deb package for x86_64")
-    system("./gradlew copyJdk -Pjdk_bundle_os=linux")
+    package_with_jdk("ubuntu", "x86_64")
-    package_with_jdk("ubuntu", "12.04")
+
-    system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux')
+    puts("[artifact:deb] building deb package for OS: linux arm64")
+    package_with_jdk("ubuntu", "arm64")
 
     #without JDKs
     system("./gradlew bootstrap") #force the build of Logstash jars
-    package("ubuntu", "12.04")
+    package("ubuntu")
   end
 
   desc "Build a DEB of logstash with all dependencies"
   task "deb_oss" => ["prepare", "generate_build_metadata"] do
-    puts("[artifact:deb_oss] building deb package")
+    #with bundled JDKs
-    system("./gradlew copyJdk -Pjdk_bundle_os=linux")
+    puts("[artifact:deb_oss] building deb OSS package x86_64")
-    package_with_jdk("ubuntu", "12.04", :oss)
+    package_with_jdk("ubuntu", "x86_64", :oss)
-    system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux')
+
+    puts("[artifact:deb_oss] building deb OSS package arm64")
+    package_with_jdk("ubuntu", "arm64", :oss)
 
     #without JDKs
     system("./gradlew bootstrap") #force the build of Logstash jars
-    package("ubuntu", "12.04", :oss)
+    package("ubuntu", :oss)
   end
 
   desc "Generate logstash core gems"
@@ -477,11 +487,13 @@ namespace "artifact" do
     puts "Complete: #{zippath}"
   end
 
-  def package_with_jdk(platform, version, variant=:standard)
+  def package_with_jdk(platform, jdk_arch, variant=:standard)
-    package(platform, version, variant, true)
+    system("./gradlew copyJdk -Pjdk_bundle_os=linux -Pjdk_arch=#{jdk_arch}")
+    package(platform, variant, true, jdk_arch)
+    system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux')
   end
 
-  def package(platform, version, variant=:standard, bundle_jdk=false)
+  def package(platform, variant=:standard, bundle_jdk=false, jdk_arch='x86_64')
     oss = variant == :oss
 
     require "stud/temporary"
@@ -541,16 +553,7 @@ namespace "artifact" do
       dir.input("#{path}=/etc/logstash")
     end
 
-    if bundle_jdk
+    arch_suffix = bundle_jdk ? map_architecture_for_package_type(platform, jdk_arch) : "no-jdk"
-      case platform
-        when "debian", "ubuntu"
-          arch_suffix = "amd64"
-        else
-          arch_suffix = "x86_64"
-      end
-    else
-      arch_suffix = "no-jdk"
-    end
 
     ensure_logstash_version_constant_defined
     package_filename = "logstash#{suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}-#{arch_suffix}.TYPE"
@@ -623,17 +626,7 @@ namespace "artifact" do
     # TODO(sissel): Invoke Pleaserun to generate the init scripts/whatever
 
     out.name = oss ? "logstash-oss" : "logstash"
-    out.architecture = "all"
+    out.architecture = bundle_jdk ? map_architecture_for_package_type(platform, jdk_arch) : "all"
-    if bundle_jdk
-      case platform
-        when "redhat", "centos"
-          out.architecture = "x86_64"
-        when "debian", "ubuntu"
-          out.architecture = "amd64"
-      end
-    else
-      out.architecture = "all"
-    end
     out.version = "#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}".gsub(/[.-]([[:alpha:]])/, '~\1')
     # TODO(sissel): Include the git commit hash?
     out.iteration = "1" # what revision?
@@ -673,6 +666,26 @@ namespace "artifact" do
     end
   end # def package
 
+  def map_architecture_for_package_type(platform, jdk_arch)
+    if jdk_arch == 'x86_64'
+      case platform
+        when "debian", "ubuntu"
+          return "amd64"
+        else
+          return "x86_64"
+      end
+    elsif jdk_arch == 'arm64'
+      case platform
+        when "debian", "ubuntu"
+          return "arm64"
+        else
+          return "aarch64"
+      end
+    else
+      raise "CPU architecture not recognized: #{jdk_arch}"
+    end
+  end
+
   def build_docker(flavor)
     env = {
       "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),