diff --git a/build.gradle b/build.gradle index 572fa7735..083d6233c 100644 --- a/build.gradle +++ b/build.gradle @@ -541,9 +541,9 @@ class JDKDetails { private final String extension final String localPackageName final String unpackedJdkName - private String arch = "x64" + private String arch - JDKDetails(versionYml, osName) { + JDKDetails(versionYml, osName, jdkArch) { revision = versionYml.bundled_jdk.revision build = versionYml.bundled_jdk.build vendor = versionYml.bundled_jdk.vendor @@ -557,8 +557,9 @@ class JDKDetails { default: extension = "tar.gz" } + arch = parseJdkArchitecture(jdkArch) unpackedJdkName = "jdk-${revision}-${osName}" - localPackageName = "${unpackedJdkName}.${extension}" + localPackageName = "${unpackedJdkName}-${arch}.${extension}" } String createDownloadUrl() { @@ -579,16 +580,30 @@ class JDKDetails { return "mac" return osName } + + private String parseJdkArchitecture(String jdkArch) { + switch (jdkArch) { + case "x86_64": + return "x64" + break + case "arm64": + return "aarch64" + break + default: + throw RuntimeException("Can't handle CPU architechture: ${jdkArch}") + } + } } tasks.register("downloadJdk", Download) { - // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] + // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] -Pjdk_arch=[arm64|x86_64] project.ext.set("versionFound", true) String osName = selectOsType() def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text) - def jdkDetails = new JDKDetails(versionYml, osName) + String jdkArch = project.ext.jdk_arch + def jdkDetails = new JDKDetails(versionYml, osName, jdkArch) description "Download JDK ${jdkDetails.major}, OS: ${osName}" @@ -663,7 +678,7 @@ tasks.register("decompressJdk") { tasks.register("copyJdk", Copy) { dependsOn = [decompressJdk, bootstrap] description = "Download, unpack and copy the JDK" - // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] + // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] -Pjdk_arch=[arm64|x86_64] doLast { System.out.println "Download location is ${project.ext.jdkDownloadLocation}, Decompressing ${project.ext.jdkDirectory} to \"${project.ext.jdkBundlingDirectory}\"" } diff --git a/qa/rspec/commands/debian.rb b/qa/rspec/commands/debian.rb index 160239664..dcbe28084 100644 --- a/qa/rspec/commands/debian.rb +++ b/qa/rspec/commands/debian.rb @@ -37,7 +37,11 @@ module ServiceTester end def architecture_extension - "amd64" + if java.lang.System.getProperty("os.arch") == "amd64" + "amd64" + else + "arm64" + end end def install(package, host=nil) diff --git a/qa/rspec/commands/redhat.rb b/qa/rspec/commands/redhat.rb index ee9ca909a..5929f982b 100644 --- a/qa/rspec/commands/redhat.rb +++ b/qa/rspec/commands/redhat.rb @@ -37,7 +37,11 @@ module ServiceTester end def architecture_extension - "x86_64" + if java.lang.System.getProperty("os.arch") == "amd64" + "x86_64" + else + "aarch64" + end end def install(package, host=nil) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index baf16f67c..575dc9128 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -135,24 +135,37 @@ namespace "artifact" do desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do #with bundled JDKs - ["linux", "windows", "darwin"].each do |os_name| - puts("[artifact:archives] Building tar.gz/zip of default plugins for OS: #{os_name}") - system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}") - case os_name - when "linux" - build_tar('ELASTIC-LICENSE', platform: '-linux-x86_64') - when "windows" - build_zip('ELASTIC-LICENSE', platform: '-windows-x86_64') - when "darwin" - build_tar('ELASTIC-LICENSE', platform: '-darwin-x86_64') - end - system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}") - end + license_details = ['ELASTIC-LICENSE'] + create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin") + create_archive_pack(license_details, "arm64", "linux") #without JDK system("./gradlew bootstrap") #force the build of Logstash jars - build_tar('ELASTIC-LICENSE', platform: '-no-jdk') - build_zip('ELASTIC-LICENSE', platform: '-no-jdk') + build_tar(*license_details, platform: '-no-jdk') + build_zip(*license_details, platform: '-no-jdk') + end + + def create_archive_pack(license_details, arch, *oses) + oses.each do |os_name| + puts("[artifact:archives] Building tar.gz/zip of default plugins for OS: #{os_name}, arch: #{arch}") + create_single_archive_pack(os_name, arch, license_details) + end + end + + def create_single_archive_pack(os_name, arch, license_details) + system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name} -Pjdk_arch=#{arch}") + if arch == 'arm64' + arch = 'aarch64' + end + case os_name + when "linux" + build_tar(*license_details, platform: "-linux-#{arch}") + when "windows" + build_zip(*license_details, platform: "-windows-#{arch}") + when "darwin" + build_tar(*license_details, platform: "-darwin-#{arch}") + end + system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}") end desc "Build a not JDK bundled tar.gz of default logstash plugins with all dependencies" @@ -163,74 +176,71 @@ namespace "artifact" do desc "Build all (jdk bundled and not) OSS tar.gz and zip of default logstash plugins with all dependencies" task "archives_oss" => ["prepare", "generate_build_metadata"] do #with bundled JDKs - ["linux", "windows", "darwin"].each do |os_name| - puts("[artifact:archives_oss] Building OSS tar.gz/zip of default plugins for OS: #{os_name}") - system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}") - case os_name - when "linux" - build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-linux-x86_64') - when "windows" - build_zip('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-windows-x86_64') - when "darwin" - build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-darwin-x86_64') - end - system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}") - end + license_details = ['APACHE-LICENSE-2.0',"-oss", oss_excluder] + create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin") + create_archive_pack(license_details, "arm64", "linux") #without JDK system("./gradlew bootstrap") #force the build of Logstash jars - build_tar('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk') - build_zip('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk') + build_tar(*license_details, platform: '-no-jdk') + build_zip(*license_details, platform: '-no-jdk') end desc "Build an RPM of logstash with all dependencies" task "rpm" => ["prepare", "generate_build_metadata"] do - puts("[artifact:rpm] building rpm package") - system("./gradlew copyJdk -Pjdk_bundle_os=linux") - package_with_jdk("centos", "5") - system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + #with bundled JDKs + puts("[artifact:rpm] building rpm package x86_64") + package_with_jdk("centos", "x86_64") + + puts("[artifact:rpm] building rpm package arm64") + package_with_jdk("centos", "arm64") #without JDKs system("./gradlew bootstrap") #force the build of Logstash jars - package("centos", "5") + package("centos") end desc "Build an RPM of logstash with all dependencies" task "rpm_oss" => ["prepare", "generate_build_metadata"] do - puts("[artifact:rpm] building rpm package") - system("./gradlew copyJdk -Pjdk_bundle_os=linux") - package_with_jdk("centos", "5", :oss) - system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + #with bundled JDKs + puts("[artifact:rpm] building rpm OSS package x86_64") + package_with_jdk("centos", "x86_64", :oss) + + puts("[artifact:rpm] building rpm OSS package arm64") + package_with_jdk("centos", "arm64", :oss) #without JDKs system("./gradlew bootstrap") #force the build of Logstash jars - package("centos", "5", :oss) + package("centos", :oss) end desc "Build a DEB of logstash with all dependencies" task "deb" => ["prepare", "generate_build_metadata"] do #with bundled JDKs - puts("[artifact:deb] building deb package for OS: linux") - system("./gradlew copyJdk -Pjdk_bundle_os=linux") - package_with_jdk("ubuntu", "12.04") - system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + puts("[artifact:deb] building deb package for x86_64") + package_with_jdk("ubuntu", "x86_64") + + puts("[artifact:deb] building deb package for OS: linux arm64") + package_with_jdk("ubuntu", "arm64") #without JDKs system("./gradlew bootstrap") #force the build of Logstash jars - package("ubuntu", "12.04") + package("ubuntu") end desc "Build a DEB of logstash with all dependencies" task "deb_oss" => ["prepare", "generate_build_metadata"] do - puts("[artifact:deb_oss] building deb package") - system("./gradlew copyJdk -Pjdk_bundle_os=linux") - package_with_jdk("ubuntu", "12.04", :oss) - system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + #with bundled JDKs + puts("[artifact:deb_oss] building deb OSS package x86_64") + package_with_jdk("ubuntu", "x86_64", :oss) + + puts("[artifact:deb_oss] building deb OSS package arm64") + package_with_jdk("ubuntu", "arm64", :oss) #without JDKs system("./gradlew bootstrap") #force the build of Logstash jars - package("ubuntu", "12.04", :oss) + package("ubuntu", :oss) end desc "Generate logstash core gems" @@ -477,11 +487,13 @@ namespace "artifact" do puts "Complete: #{zippath}" end - def package_with_jdk(platform, version, variant=:standard) - package(platform, version, variant, true) + def package_with_jdk(platform, jdk_arch, variant=:standard) + system("./gradlew copyJdk -Pjdk_bundle_os=linux -Pjdk_arch=#{jdk_arch}") + package(platform, variant, true, jdk_arch) + system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') end - def package(platform, version, variant=:standard, bundle_jdk=false) + def package(platform, variant=:standard, bundle_jdk=false, jdk_arch='x86_64') oss = variant == :oss require "stud/temporary" @@ -541,16 +553,7 @@ namespace "artifact" do dir.input("#{path}=/etc/logstash") end - if bundle_jdk - case platform - when "debian", "ubuntu" - arch_suffix = "amd64" - else - arch_suffix = "x86_64" - end - else - arch_suffix = "no-jdk" - end + arch_suffix = bundle_jdk ? map_architecture_for_package_type(platform, jdk_arch) : "no-jdk" ensure_logstash_version_constant_defined package_filename = "logstash#{suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}-#{arch_suffix}.TYPE" @@ -623,17 +626,7 @@ namespace "artifact" do # TODO(sissel): Invoke Pleaserun to generate the init scripts/whatever out.name = oss ? "logstash-oss" : "logstash" - out.architecture = "all" - if bundle_jdk - case platform - when "redhat", "centos" - out.architecture = "x86_64" - when "debian", "ubuntu" - out.architecture = "amd64" - end - else - out.architecture = "all" - end + out.architecture = bundle_jdk ? map_architecture_for_package_type(platform, jdk_arch) : "all" out.version = "#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}".gsub(/[.-]([[:alpha:]])/, '~\1') # TODO(sissel): Include the git commit hash? out.iteration = "1" # what revision? @@ -673,6 +666,26 @@ namespace "artifact" do end end # def package + def map_architecture_for_package_type(platform, jdk_arch) + if jdk_arch == 'x86_64' + case platform + when "debian", "ubuntu" + return "amd64" + else + return "x86_64" + end + elsif jdk_arch == 'arm64' + case platform + when "debian", "ubuntu" + return "arm64" + else + return "aarch64" + end + else + raise "CPU architecture not recognized: #{jdk_arch}" + end + end + def build_docker(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),