diff --git a/bin/logstash b/bin/logstash index ae23c5163..696361964 100755 --- a/bin/logstash +++ b/bin/logstash @@ -1,7 +1,5 @@ #!/bin/bash -# Run logstash from source -# -# This is most useful when done from a git checkout. +# Run logstash # # Usage: # bin/logstash [arguments] diff --git a/bin/logstash.lib.sh b/bin/logstash.lib.sh index 33c05fbc4..3abe5fcfc 100755 --- a/bin/logstash.lib.sh +++ b/bin/logstash.lib.sh @@ -72,15 +72,38 @@ parse_jvm_options() { fi } +setup_bundled_jdk_part() { + OS_NAME="$(uname -s)" + if [ $OS_NAME = "Darwin" ]; then + BUNDLED_JDK_PART="jdk.app/Contents/Home" + else + BUNDLED_JDK_PART="jdk" + fi +} + setup_java() { # set the path to java into JAVACMD which will be picked up by JRuby to launch itself if [ -z "$JAVACMD" ]; then - if [ -x "$JAVA_HOME/bin/java" ]; then - JAVACMD="$JAVA_HOME/bin/java" - else + setup_bundled_jdk_part + JAVACMD_TEST=`command -v java` + if [ -n "$JAVA_HOME" ]; then + echo "Using JAVA_HOME defined java: ${JAVA_HOME}" + if [ -x "$JAVA_HOME/bin/java" ]; then + JAVACMD="$JAVA_HOME/bin/java" + if [ -d "${LOGSTASH_HOME}/${BUNDLED_JDK_PART}" -a -x "${LOGSTASH_HOME}/${BUNDLED_JDK_PART}/bin/java" ]; then + echo "WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK" + fi + else + echo "Invalid JAVA_HOME, doesn't contain bin/java executable" + fi + elif [ -d "${LOGSTASH_HOME}/${BUNDLED_JDK_PART}" -a -x "${LOGSTASH_HOME}/${BUNDLED_JDK_PART}/bin/java" ]; then + echo "Using bundled JDK: ${LOGSTASH_HOME}/${BUNDLED_JDK_PART}" + JAVACMD="${LOGSTASH_HOME}/${BUNDLED_JDK_PART}/bin/java" + elif [ -n "$JAVACMD_TEST" ]; then set +e JAVACMD=`command -v java` set -e + echo "Using system java: $JAVACMD" fi fi diff --git a/bin/setup.bat b/bin/setup.bat index b846894a9..a1138e401 100644 --- a/bin/setup.bat +++ b/bin/setup.bat @@ -22,8 +22,18 @@ rem ### 2: set java if defined JAVA_HOME ( set JAVA="%JAVA_HOME%\bin\java.exe" + echo Using JAVA_HOME defined java: %JAVA_HOME% + if exist "%LS_HOME%\jdk" ( + echo WARNING, using JAVA_HOME while Logstash distribution comes with a bundled JDK + ) ) else ( - for %%I in (java.exe) do set JAVA="%%~$PATH:I" + if exist "%LS_HOME%\jdk" ( + set JAVA="%LS_HOME%\jdk\bin\java.exe" + echo "Using bundled JDK: %JAVA%"" + ) else ( + for %%I in (java.exe) do set JAVA="%%~$PATH:I" + echo "Using system java: %JAVA%" + ) ) if not exist %JAVA% ( diff --git a/bin/system-install b/bin/system-install index d81c5e87b..7a2d37300 100755 --- a/bin/system-install +++ b/bin/system-install @@ -61,7 +61,7 @@ done # bin/logstash-plugin is a short lived ruby script thus we can use aggressive "faster starting JRuby options" # see https://github.com/jruby/jruby/wiki/Improving-startup-time -export JRUBY_OPTS="$JRUBY_OPTS -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -X-C -Xcompile.invokedynamic=false" +export JRUBY_OPTS="$JRUBY_OPTS $OPEN_JAVA_MODULES -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -X-C -Xcompile.invokedynamic=false" tempfile=$(mktemp) if [ "x${PRESTART}" == "x" ]; then diff --git a/build.gradle b/build.gradle index 6f3fdd25e..40fcab52a 100644 --- a/build.gradle +++ b/build.gradle @@ -25,18 +25,20 @@ buildscript { } } dependencies { + classpath 'org.yaml:snakeyaml:1.17' classpath "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1" } } plugins { id "de.undercouch.download" version "4.0.4" + id "com.dorongold.task-tree" version "1.5" } apply plugin: 'de.undercouch.download' apply from: "rubyUtils.gradle" - +import org.yaml.snakeyaml.Yaml import de.undercouch.gradle.tasks.download.Download import groovy.json.JsonSlurper @@ -242,7 +244,7 @@ tasks.register("assembleTarDistribution") { inputs.files fileTree("${projectDir}/x-pack") outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz") doLast { - rake(projectDir, buildDir, 'artifact:tar') + rake(projectDir, buildDir, 'artifact:no_bundle_jdk_tar') } } @@ -257,7 +259,7 @@ tasks.register("assembleOssTarDistribution") { inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") doLast { - rake(projectDir, buildDir, 'artifact:tar_oss') + rake(projectDir, buildDir, 'artifact:archives_oss') } } @@ -274,7 +276,7 @@ tasks.register("assembleZipDistribution") { inputs.files fileTree("${projectDir}/x-pack") outputs.files file("${buildDir}/logstash-${project.version}.zip") doLast { - rake(projectDir, buildDir, 'artifact:zip') + rake(projectDir, buildDir, 'artifact:archives') } } @@ -290,7 +292,7 @@ tasks.register("assembleOssZipDistribution") { inputs.files fileTree("${projectDir}/logstash-core/src") outputs.files file("${buildDir}/logstash-${project.version}.zip") doLast { - rake(projectDir, buildDir, 'artifact:zip_oss') + rake(projectDir, buildDir, 'artifact:archives_oss') } } @@ -422,7 +424,7 @@ tasks.register("deleteLocalEs", Delete) { delete ('./build/elasticsearch') } -tasks.register("copyEs", Copy){ +tasks.register("copyEs", Copy) { dependsOn = [downloadEs, deleteLocalEs] from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation)) into "./build/" @@ -509,11 +511,165 @@ bootstrap.dependsOn assemblyDeps // Elasticsearch doesn't yet have a build we can fetch // So for now we'll remove this to unblock builds, but finding a way // to compartimentalize failures is needed going forward -//check.dependsOn runIntegrationTests +//check.dependsOn runIntegrationTest -Boolean oss = System.getenv('OSS').equals('true') +runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test") -if (!oss) { +def selectOsType() { + if (project.ext.has("jdk_bundle_os")) { + return project.ext.jdk_bundle_os + } + String osName = System.properties['os.name'] + switch (osName.toLowerCase()) { + case ~/mac os x/: + return "darwin" + case ~/windows.*/: + return "windows" + case ~/linux/: + return "linux" + default: + throw new IllegalArgumentException("Can't determine OS type from name: $osName") + } +} + +class JDKDetails { + final String revision + final String build + final String vendor + final int major + private final String osName + private final String extension + final String localPackageName + final String unpackedJdkName + private String arch = "x64" + + JDKDetails(versionYml, osName) { + revision = versionYml.bundled_jdk.revision + build = versionYml.bundled_jdk.build + vendor = versionYml.bundled_jdk.vendor + major = revision.split('\\.').first() as int + this.osName = osName + + switch (osName) { + case "windows": + extension = "zip" + break + default: + extension = "tar.gz" + } + unpackedJdkName = "jdk-${revision}-${osName}" + localPackageName = "${unpackedJdkName}.${extension}" + } + + String createDownloadUrl() { + switch (vendor) { + case "adoptopenjdk": + String releaseName = major > 8 ? + "jdk-${revision}+${build}": + "jdk${revision}u${build}" + String adoptOsName = adaptOsName(osName) + return "https://api.adoptopenjdk.net/v3/binary/version/${releaseName}/${adoptOsName}/${arch}/jdk/hotspot/normal/${vendor}" + default: + throw RuntimeException("Can't handle vendor: ${vendor}") + } + } + + private String adaptOsName(String osName) { + if (osName == "darwin") + return "mac" + return osName + } +} + +tasks.register("downloadJdk", Download) { + // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] + + project.ext.set("versionFound", true) + String osName = selectOsType() + + def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text) + def jdkDetails = new JDKDetails(versionYml, osName) + + description "Download JDK ${jdkDetails.major}, OS: ${osName}" + + // find url of build artifact + String artifactApiUrl = jdkDetails.createDownloadUrl() + + project.ext.set("jdkURL", System.getenv("JDK_URL") ?: artifactApiUrl) + project.ext.set("jdkDownloadLocation", "${projectDir}/build/${jdkDetails.localPackageName}") + project.ext.set("jdkDirectory", "${projectDir}/build/${jdkDetails.unpackedJdkName}") + + String jdkFolderName = osName == "darwin" ? "jdk.app" : "jdk" + project.ext.set("jdkBundlingDirectory", "${projectDir}/${jdkFolderName}") + + src project.ext.jdkURL + onlyIfNewer true + overwrite false + inputs.file("${projectDir}/versions.yml") + outputs.file(project.ext.jdkDownloadLocation) + dest new File(project.ext.jdkDownloadLocation) + + doLast { + mkdir project.ext.jdkBundlingDirectory + println "Downloaded to ${project.ext.jdkDownloadLocation}" + } +} + +tasks.register("deleteLocalJdk", Delete) { + // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] + String osName = selectOsType() + String jdkFolderName = osName == "darwin" ? "jdk.app" : "jdk" + String jdkBundlingDirectory = "${projectDir}/${jdkFolderName}" + delete jdkBundlingDirectory +} + +// Cannot use tarTree as it does not handle symlinks +tasks.register("untarJdk", Exec) { + dependsOn downloadJdk + description = "unpack the downloaded JDK's tar.gz" + commandLine 'tar', 'xf', project.ext.jdkDownloadLocation, '-C', project.ext.jdkBundlingDirectory, '--strip-components', '1' + inputs.file(project.ext.jdkDownloadLocation) + outputs.dir(project.ext.jdkBundlingDirectory) +} + +tasks.register("unzipJdk", Copy) { + dependsOn downloadJdk + description = "unpack the downloaded JDK's zip" + String rootName = null + from(zipTree("$project.ext.jdkDownloadLocation")) { + eachFile { fcd -> + rootName = rootName ?: fcd.relativePath.segments[0] + fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1)) + } + } + into project.ext.jdkBundlingDirectory + doLast { + delete "${project.ext.jdkBundlingDirectory}/$rootName" + } +} + +tasks.register("decompressJdk") { + description = "unpack the downloaded JDK's (wrapper task for unzipJdk, untarJdk)" + String osName = selectOsType() + switch (osName) { + case "windows": + dependsOn ":unzipJdk" + break + default: + dependsOn ":untarJdk" + } +} + +tasks.register("copyJdk", Copy) { + dependsOn = [decompressJdk, bootstrap] + description = "Download, unpack and copy the JDK" + // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] + doLast { + System.out.println "Download location is ${project.ext.jdkDownloadLocation}, Decompressing ${project.ext.jdkDirectory} to \"${project.ext.jdkBundlingDirectory}\"" + } +} + +if (System.getenv('OSS') != 'true') { project(":logstash-xpack") { ["rubyTests", "rubyIntegrationTests", "test"].each { tsk -> tasks.getByPath(":logstash-xpack:" + tsk).configure { diff --git a/docker/templates/Dockerfile.j2 b/docker/templates/Dockerfile.j2 index d6987d63c..4931df84a 100644 --- a/docker/templates/Dockerfile.j2 +++ b/docker/templates/Dockerfile.j2 @@ -6,10 +6,10 @@ {% endif -%} {% if image_flavor == 'oss' -%} - {% set tarball = 'logstash-oss-%s.tar.gz' % elastic_version -%} + {% set tarball = 'logstash-oss-%s-linux-x86_64.tar.gz' % elastic_version -%} {% set license = 'Apache 2.0' -%} {% else -%} - {% set tarball = 'logstash-%s.tar.gz' % elastic_version -%} + {% set tarball = 'logstash-%s-linux-x86_64.tar.gz' % elastic_version -%} {% set license = 'Elastic License' -%} {% endif -%} @@ -29,7 +29,7 @@ FROM {{ base_image }} # Install Java and the "which" command, which is needed by Logstash's shell # scripts. # Minimal distributions also require findutils tar gzip (procps for integration tests) -RUN {{ package_manager }} update -y && {{ package_manager }} install -y procps findutils tar gzip java-11-openjdk-devel which && \ +RUN {{ package_manager }} update -y && {{ package_manager }} install -y procps findutils tar gzip which shadow-utils && \ {{ package_manager }} clean all # Provide a non-root user to run the process. diff --git a/qa/acceptance/spec/lib/artifact_operation_spec.rb b/qa/acceptance/spec/lib/artifact_operation_spec.rb index 82930e38a..982bdd134 100644 --- a/qa/acceptance/spec/lib/artifact_operation_spec.rb +++ b/qa/acceptance/spec/lib/artifact_operation_spec.rb @@ -17,6 +17,7 @@ require_relative '../spec_helper' require_relative '../shared_examples/installed' +require_relative '../shared_examples/installed_with_jdk' require_relative '../shared_examples/running' require_relative '../shared_examples/updated' @@ -26,6 +27,7 @@ describe "artifacts operation" do config.servers.each do |address| logstash = ServiceTester::Artifact.new(address, config.lookup[address]) it_behaves_like "installable", logstash + it_behaves_like "installable_with_jdk", logstash it_behaves_like "updated", logstash end end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb index 5598e9ab5..da1ede672 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb @@ -49,6 +49,7 @@ shared_examples "logstash list" do |logstash| stdout = StringIO.new(result.stdout) stdout.set_encoding(Encoding::UTF_8) while line = stdout.gets + next if line.match(/^Using system java:.*$/) match = line.match(/^#{plugin_name_with_version}$/) expect(match).to_not be_nil diff --git a/qa/acceptance/spec/shared_examples/installed_with_jdk.rb b/qa/acceptance/spec/shared_examples/installed_with_jdk.rb new file mode 100644 index 000000000..df74f4914 --- /dev/null +++ b/qa/acceptance/spec/shared_examples/installed_with_jdk.rb @@ -0,0 +1,52 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative '../spec_helper' +require 'logstash/version' + +# This test checks if a package is possible to be installed without errors. +RSpec.shared_examples "installable_with_jdk" do |logstash| + + before(:all) do + #unset to force it using bundled JDK to run LS + logstash.run_command("unset JAVA_HOME") + end + + before(:each) do + logstash.uninstall + logstash.install({:bundled_jdk => true, :version => LOGSTASH_VERSION}) + end + + after(:each) do + logstash.uninstall + end + + it "is installed on #{logstash.hostname}" do + expect(logstash).to be_installed + end + + it "is running on #{logstash.hostname}" do + logstash.start_service + expect(logstash).to be_running_with("/usr/share/logstash/jdk/bin/java") + logstash.stop_service + end + + it "is removable on #{logstash.hostname}" do + logstash.uninstall + expect(logstash).to be_removed + end +end diff --git a/qa/acceptance/spec/shared_examples/updated.rb b/qa/acceptance/spec/shared_examples/updated.rb index cae08700c..a0762497f 100644 --- a/qa/acceptance/spec/shared_examples/updated.rb +++ b/qa/acceptance/spec/shared_examples/updated.rb @@ -28,7 +28,7 @@ RSpec.shared_examples "updated" do |logstash| end before(:each) do - options={:version => LOGSTASH_LATEST_VERSION, :snapshot => false, :base => "./" } + options={:version => LOGSTASH_LATEST_VERSION, :snapshot => false, :base => "./", :skip_jdk_infix => true } logstash.install(options) # make sure latest version is installed end diff --git a/qa/docker/shared_examples/container.rb b/qa/docker/shared_examples/container.rb index 0d8f339b0..5d76d79fa 100644 --- a/qa/docker/shared_examples/container.rb +++ b/qa/docker/shared_examples/container.rb @@ -11,7 +11,17 @@ shared_examples_for 'the container is configured correctly' do |flavor| context 'logstash' do it 'should run with the correct version' do - expect(exec_in_container(@container, 'logstash --version')).to match /#{version}/ + console_out = exec_in_container(@container, 'logstash --version') + console_filtered = console_out.split("\n") + .delete_if do |line| + line =~ /Using JAVA_HOME defined java|Using system java: / + end.join + expect(console_filtered).to match /#{version}/ + end + + it 'should run with the bundled JDK' do + first_console_line = exec_in_container(@container, 'logstash --version').split("\n")[0] + expect(first_console_line).to match /Using bundled JDK: \/usr\/share\/logstash\/jdk/ end it 'should be running an API server on port 9600' do @@ -42,8 +52,8 @@ shared_examples_for 'the container is configured correctly' do |flavor| end it 'should have all files owned by the logstash user' do - expect(exec_in_container(@container, 'find /usr/share/logstash ! -user logstash')).to be_nil - expect(exec_in_container(@container, 'find /usr/share/logstash -user logstash')).not_to be_nil + expect(exec_in_container(@container, 'find /usr/share/logstash ! -user logstash')).to be_empty + expect(exec_in_container(@container, 'find /usr/share/logstash -user logstash')).not_to be_empty end it 'should have a logstash user with uid 1000' do diff --git a/qa/docker/spec/spec_helper.rb b/qa/docker/spec/spec_helper.rb index c98961d87..2cc712df0 100644 --- a/qa/docker/spec/spec_helper.rb +++ b/qa/docker/spec/spec_helper.rb @@ -77,7 +77,7 @@ def java_process(container, column) end def exec_in_container(container, command) - container.exec(command.split)[0][0] + container.exec(command.split)[0].join end def architecture_for_flavor(flavor) diff --git a/qa/integration/specs/cli/prepare_offline_pack_spec.rb b/qa/integration/specs/cli/prepare_offline_pack_spec.rb index 695529d8d..c3273d5a8 100644 --- a/qa/integration/specs/cli/prepare_offline_pack_spec.rb +++ b/qa/integration/specs/cli/prepare_offline_pack_spec.rb @@ -86,7 +86,7 @@ describe "CLI > logstash-plugin prepare-offline-pack" do filters = @logstash_plugin.list(plugins_to_pack.first) .stderr_and_stdout.split("\n") .delete_if do |line| - line =~ /cext|├──|└──|logstash-integration|JAVA_OPT|fatal|^WARNING|^warning: ignoring JAVA_TOOL_OPTIONS|^OpenJDK 64-Bit Server VM warning|Option \w+ was deprecated/ + line =~ /cext|├──|└──|logstash-integration|JAVA_OPT|fatal|^WARNING|^warning: ignoring JAVA_TOOL_OPTIONS|^OpenJDK 64-Bit Server VM warning|Option \w+ was deprecated|Using JAVA_HOME defined java|Using system java: |\[\[: not found/ end expect(unpacked.plugins.collect(&:name)).to include(*filters) diff --git a/qa/rspec/commands.rb b/qa/rspec/commands.rb index 4b48fcd1c..d2c97c132 100644 --- a/qa/rspec/commands.rb +++ b/qa/rspec/commands.rb @@ -42,6 +42,8 @@ module ServiceTester @host = host @options = options @client = CommandsFactory.fetch(options["type"], options["host"]) + @bundled_jdk = false + @skip_jdk_infix = false end def hostname @@ -74,7 +76,10 @@ module ServiceTester def install(options={}) base = options.fetch(:base, ServiceTester::Base::LOCATION) - package = client.package_for(filename(options), base) + @bundled_jdk = options.fetch(:bundled_jdk, false) + @skip_jdk_infix = options.fetch(:skip_jdk_infix, false) + filename = filename(options) + package = client.package_for(filename, @skip_jdk_infix, @bundled_jdk, base) client.install(package, host) end diff --git a/qa/rspec/commands/base.rb b/qa/rspec/commands/base.rb index 52804c862..1229f66be 100644 --- a/qa/rspec/commands/base.rb +++ b/qa/rspec/commands/base.rb @@ -82,5 +82,23 @@ module ServiceTester def delete_file(path, host) run_command("rm -rf #{path}", host) end + + def package_for(filename, skip_jdk_infix, bundled_jdk, base=ServiceTester::Base::LOCATION) + jdk_arch_ext = jdk_architecture_extension(skip_jdk_infix, bundled_jdk) + File.join(base, "#{filename}#{jdk_arch_ext}.#{package_extension}") + end + + private + def jdk_architecture_extension(skip_jdk_infix, bundled_jdk) + if skip_jdk_infix + "" + else + if bundled_jdk + "-" + architecture_extension + else + "-no-jdk" + end + end + end end end diff --git a/qa/rspec/commands/debian.rb b/qa/rspec/commands/debian.rb index f4a6d490a..160239664 100644 --- a/qa/rspec/commands/debian.rb +++ b/qa/rspec/commands/debian.rb @@ -32,8 +32,12 @@ module ServiceTester stdout.match(/^Status: install ok installed$/) end - def package_for(filename, base=ServiceTester::Base::LOCATION) - File.join(base, "#{filename}.deb") + def package_extension + "deb" + end + + def architecture_extension + "amd64" end def install(package, host=nil) diff --git a/qa/rspec/commands/redhat.rb b/qa/rspec/commands/redhat.rb index eaa6a1964..ee9ca909a 100644 --- a/qa/rspec/commands/redhat.rb +++ b/qa/rspec/commands/redhat.rb @@ -32,8 +32,12 @@ module ServiceTester stdout.match(/^logstash.noarch/) end - def package_for(filename, base=ServiceTester::Base::LOCATION) - File.join(base, "#{filename}.rpm") + def package_extension + "rpm" + end + + def architecture_extension + "x86_64" end def install(package, host=nil) diff --git a/qa/rspec/commands/suse.rb b/qa/rspec/commands/suse.rb index 733c0685a..c66ba048c 100644 --- a/qa/rspec/commands/suse.rb +++ b/qa/rspec/commands/suse.rb @@ -29,8 +29,12 @@ module ServiceTester stdout.match(/^i | logstash | An extensible logging pipeline | package$/) end - def package_for(filename, base=ServiceTester::Base::LOCATION) - File.join(base, "#{filename}.rpm") + def package_extension() + "rpm" + end + + def architecture_extension() + "x86_64" end def install(package, host=nil) diff --git a/qa/rspec/commands/system_helpers.rb b/qa/rspec/commands/system_helpers.rb index a720f7c84..58354687a 100644 --- a/qa/rspec/commands/system_helpers.rb +++ b/qa/rspec/commands/system_helpers.rb @@ -19,14 +19,16 @@ require_relative "base" module ServiceTester module SystemD - def running?(hosts, package) + def running?(hosts, package, jdk_path='/usr/bin/java') stdout = "" at(hosts, {in: :serial}) do |host| cmd = sudo_exec!("service #{package} status") stdout = cmd.stdout end + stdout.force_encoding(Encoding::UTF_8) ( stdout.match(/Active: active \(running\)/) && + stdout.match(/^\s*└─\d*\s.*#{jdk_path}/) && stdout.match(/#{package}.service - #{package}/) ) end @@ -40,13 +42,19 @@ module ServiceTester end module InitD - def running?(hosts, package) + def running?(hosts, package, jdk_path='/usr/bin/java') stdout = "" at(hosts, {in: :serial}) do |host| cmd = sudo_exec!("initctl status #{package}") stdout = cmd.stdout end - stdout.match(/#{package} start\/running/) + running = stdout.match(/#{package} start\/running/) + pid = stdout.match(/#{package} start\/running, process (\d*)/).captures[0] + at(hosts, {in: :serial}) do |host| + cmd = sudo_exec!("ps ax | grep #{pid}") + stdout = cmd.stdout + end + (running && stdout.match(/#{jdk_path}/)) end def service_manager(service, action, host=nil) diff --git a/qa/rspec/matchers/be_running.rb b/qa/rspec/matchers/be_running.rb index c02777693..f4463249e 100644 --- a/qa/rspec/matchers/be_running.rb +++ b/qa/rspec/matchers/be_running.rb @@ -23,3 +23,9 @@ RSpec::Matchers.define :be_running do subject.running?(subject.hosts, subject.name) end end + +RSpec::Matchers.define :be_running_with do |expected_jdk_path| + match do |subject| + subject.running?(subject.hosts, subject.name, expected_jdk_path) + end +end diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index fa085e68a..baf16f67c 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -59,9 +59,17 @@ namespace "artifact" do # plugins which use jar-dependencies. # See more in https://github.com/elastic/logstash/issues/4818 "vendor/??*/**/.mvn/**/*", + + # Without this when JRuby runs 'pleaserun' gem using the AdoptOpenJDK, during the post install script + # it claims that modules are not open for private introspection and suggest it's missing --add-opens + # so including these files JRuby run with modules opened to private introspection. + "vendor/jruby/bin/.jruby.java_opts", + "vendor/jruby/bin/.jruby.module_opts", "Gemfile", "Gemfile.lock", "x-pack/**/*", + "jdk/**/*", + "jdk.app/**/*", ] end @@ -124,53 +132,104 @@ namespace "artifact" do task "all" => ["prepare", "build"] task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_ubi8"] - desc "Build a tar.gz of default logstash plugins with all dependencies" - task "tar" => ["prepare", "generate_build_metadata"] do - puts("[artifact:tar] Building tar.gz of default plugins") + desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" + task "archives" => ["prepare", "generate_build_metadata"] do + #with bundled JDKs + ["linux", "windows", "darwin"].each do |os_name| + puts("[artifact:archives] Building tar.gz/zip of default plugins for OS: #{os_name}") + system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}") + case os_name + when "linux" + build_tar('ELASTIC-LICENSE', platform: '-linux-x86_64') + when "windows" + build_zip('ELASTIC-LICENSE', platform: '-windows-x86_64') + when "darwin" + build_tar('ELASTIC-LICENSE', platform: '-darwin-x86_64') + end + system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}") + end + + #without JDK + system("./gradlew bootstrap") #force the build of Logstash jars + build_tar('ELASTIC-LICENSE', platform: '-no-jdk') + build_zip('ELASTIC-LICENSE', platform: '-no-jdk') + end + + desc "Build a not JDK bundled tar.gz of default logstash plugins with all dependencies" + task "no_bundle_jdk_tar" => ["prepare", "generate_build_metadata"] do build_tar('ELASTIC-LICENSE') end - desc "Build an OSS tar.gz of default logstash plugins with all dependencies" - task "tar_oss" => ["prepare", "generate_build_metadata"] do - puts("[artifact:tar] Building tar.gz of default plugins") - build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder) - end + desc "Build all (jdk bundled and not) OSS tar.gz and zip of default logstash plugins with all dependencies" + task "archives_oss" => ["prepare", "generate_build_metadata"] do + #with bundled JDKs + ["linux", "windows", "darwin"].each do |os_name| + puts("[artifact:archives_oss] Building OSS tar.gz/zip of default plugins for OS: #{os_name}") + system("./gradlew copyJdk -Pjdk_bundle_os=#{os_name}") + case os_name + when "linux" + build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-linux-x86_64') + when "windows" + build_zip('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-windows-x86_64') + when "darwin" + build_tar('APACHE-LICENSE-2.0', "-oss", oss_excluder, platform: '-darwin-x86_64') + end + system("./gradlew deleteLocalJdk -Pjdk_bundle_os=#{os_name}") + end - desc "Build a zip of default logstash plugins with all dependencies" - task "zip" => ["prepare", "generate_build_metadata"] do - puts("[artifact:zip] Building zip of default plugins") - build_zip('ELASTIC-LICENSE') + #without JDK + system("./gradlew bootstrap") #force the build of Logstash jars + build_tar('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk') + build_zip('APACHE-LICENSE-2.0',"-oss", oss_excluder, platform: '-no-jdk') end - desc "Build a zip of default logstash plugins with all dependencies" - task "zip_oss" => ["prepare", "generate_build_metadata"] do - puts("[artifact:zip] Building zip of default plugins") - build_zip('APACHE-LICENSE-2.0',"-oss", oss_excluder) - end - - desc "Build an RPM of logstash with all dependencies" task "rpm" => ["prepare", "generate_build_metadata"] do puts("[artifact:rpm] building rpm package") + system("./gradlew copyJdk -Pjdk_bundle_os=linux") + package_with_jdk("centos", "5") + system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + + #without JDKs + system("./gradlew bootstrap") #force the build of Logstash jars package("centos", "5") end desc "Build an RPM of logstash with all dependencies" task "rpm_oss" => ["prepare", "generate_build_metadata"] do puts("[artifact:rpm] building rpm package") + system("./gradlew copyJdk -Pjdk_bundle_os=linux") + package_with_jdk("centos", "5", :oss) + system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + + #without JDKs + system("./gradlew bootstrap") #force the build of Logstash jars package("centos", "5", :oss) end desc "Build a DEB of logstash with all dependencies" task "deb" => ["prepare", "generate_build_metadata"] do - puts("[artifact:deb] building deb package") + #with bundled JDKs + puts("[artifact:deb] building deb package for OS: linux") + system("./gradlew copyJdk -Pjdk_bundle_os=linux") + package_with_jdk("ubuntu", "12.04") + system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + + #without JDKs + system("./gradlew bootstrap") #force the build of Logstash jars package("ubuntu", "12.04") end desc "Build a DEB of logstash with all dependencies" task "deb_oss" => ["prepare", "generate_build_metadata"] do - puts("[artifact:deb] building deb package") + puts("[artifact:deb_oss] building deb package") + system("./gradlew copyJdk -Pjdk_bundle_os=linux") + package_with_jdk("ubuntu", "12.04", :oss) + system('./gradlew deleteLocalJdk -Pjdk_bundle_os=linux') + + #without JDKs + system("./gradlew bootstrap") #force the build of Logstash jars package("ubuntu", "12.04", :oss) end @@ -197,19 +256,19 @@ namespace "artifact" do end desc "Build docker image" - task "docker" => ["prepare", "generate_build_metadata", "tar"] do + task "docker" => ["prepare", "generate_build_metadata", "archives"] do puts("[docker] Building docker image") build_docker('full') end desc "Build OSS docker image" - task "docker_oss" => ["prepare", "generate_build_metadata", "tar_oss"] do + task "docker_oss" => ["prepare", "generate_build_metadata", "archives_oss"] do puts("[docker_oss] Building OSS docker image") build_docker('oss') end desc "Build UBI8 docker image" - task "docker_ubi8" => %w(prepare generate_build_metadata tar) do + task "docker_ubi8" => %w(prepare generate_build_metadata archives) do puts("[docker_ubi8] Building UBI docker image") build_docker('ubi8') end @@ -247,10 +306,8 @@ namespace "artifact" do Rake::Task["artifact:deb_oss"].invoke Rake::Task["artifact:rpm"].invoke Rake::Task["artifact:rpm_oss"].invoke - Rake::Task["artifact:zip"].invoke - Rake::Task["artifact:zip_oss"].invoke - Rake::Task["artifact:tar"].invoke - Rake::Task["artifact:tar_oss"].invoke + Rake::Task["artifact:archives"].invoke + Rake::Task["artifact:archives_oss"].invoke unless ENV['SKIP_DOCKER'] == "1" Rake::Task["artifact:docker"].invoke Rake::Task["artifact:docker_oss"].invoke @@ -343,47 +400,45 @@ namespace "artifact" do end end - def build_tar(license, tar_suffix = nil, excluder=nil) + + def build_tar(license, tar_suffix = nil, excluder=nil, platform: '') require "zlib" - require "archive/tar/minitar" + require 'rubygems' + require 'rubygems/package' ensure_logstash_version_constant_defined - tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.tar.gz" + tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}#{platform}.tar.gz" if File.exist?(tarpath) && ENV['SKIP_PREPARE'] == "1" && !source_modified_since?(File.mtime(tarpath)) puts("[artifact:tar] Source code not modified. Skipping build of #{tarpath}") return end puts("[artifact:tar] building #{tarpath}") gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION) - tar = Archive::Tar::Minitar::Output.new(gz) - files(excluder).each do |path| - write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") + Gem::Package::TarWriter.new(gz) do |tar| + files(excluder).each do |path| + write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") + end + + source_license_path = "licenses/#{license}.txt" + fail("Missing source license: #{source_license_path}") unless File.exists?(source_license_path) + write_to_tar(tar, source_license_path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/LICENSE.txt") + + # add build.rb to tar + metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb") + path_in_tar = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_tar) + write_to_tar(tar, BUILD_METADATA_FILE.path, path_in_tar) end - - source_license_path = "licenses/#{license}.txt" - fail("Missing source license: #{source_license_path}") unless File.exists?(source_license_path) - write_to_tar(tar, source_license_path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/LICENSE.txt") - - # add build.rb to tar - metadata_file_path_in_tar = File.join("logstash-core", "lib", "logstash", "build.rb") - path_in_tar = File.join("logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}", metadata_file_path_in_tar) - write_to_tar(tar, BUILD_METADATA_FILE.path, path_in_tar) - - tar.close gz.close - puts "Complete: #{tarpath}" end + def write_to_tar(tar, path, path_in_tar) stat = File.lstat(path) - opts = { - :size => stat.size, - :mode => stat.mode, - :mtime => stat.mtime - } if stat.directory? - tar.tar.mkdir(path_in_tar, opts) + tar.mkdir(path_in_tar, stat.mode) + elsif stat.symlink? + tar.add_symlink(path_in_tar, File.readlink(path), stat.mode) else - tar.tar.add_file_simple(path_in_tar, opts) do |io| + tar.add_file_simple(path_in_tar, stat.mode, stat.size) do |io| File.open(path,'rb') do |fd| chunk = nil size = 0 @@ -396,10 +451,10 @@ namespace "artifact" do end end - def build_zip(license, zip_suffix = "", excluder=nil) + def build_zip(license, zip_suffix = "", excluder=nil, platform: '') require 'zip' ensure_logstash_version_constant_defined - zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.zip" + zippath = "build/logstash#{zip_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}#{platform}.zip" puts("[artifact:zip] building #{zippath}") File.unlink(zippath) if File.exists?(zippath) Zip::File.open(zippath, Zip::File::CREATE) do |zipfile| @@ -422,7 +477,11 @@ namespace "artifact" do puts "Complete: #{zippath}" end - def package(platform, version, variant=:standard) + def package_with_jdk(platform, version, variant=:standard) + package(platform, version, variant, true) + end + + def package(platform, version, variant=:standard, bundle_jdk=false) oss = variant == :oss require "stud/temporary" @@ -482,8 +541,19 @@ namespace "artifact" do dir.input("#{path}=/etc/logstash") end + if bundle_jdk + case platform + when "debian", "ubuntu" + arch_suffix = "amd64" + else + arch_suffix = "x86_64" + end + else + arch_suffix = "no-jdk" + end + ensure_logstash_version_constant_defined - package_filename = "logstash#{suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}.TYPE" + package_filename = "logstash#{suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}-#{arch_suffix}.TYPE" File.join(basedir, "config", "startup.options").tap do |path| dir.input("#{path}=/etc/logstash") @@ -529,7 +599,7 @@ namespace "artifact" do out.license = license out.attributes[:deb_user] = "root" out.attributes[:deb_group] = "root" - out.attributes[:deb_suggests] = "java8-runtime-headless" + out.attributes[:deb_suggests] = "java8-runtime-headless" unless bundle_jdk out.config_files << "/etc/logstash/startup.options" out.config_files << "/etc/logstash/jvm.options" out.config_files << "/etc/logstash/log4j2.properties" @@ -553,13 +623,23 @@ namespace "artifact" do # TODO(sissel): Invoke Pleaserun to generate the init scripts/whatever out.name = oss ? "logstash-oss" : "logstash" - out.version = "#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}".gsub(/[.-]([[:alpha:]])/, '~\1') out.architecture = "all" + if bundle_jdk + case platform + when "redhat", "centos" + out.architecture = "x86_64" + when "debian", "ubuntu" + out.architecture = "amd64" + end + else + out.architecture = "all" + end + out.version = "#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}".gsub(/[.-]([[:alpha:]])/, '~\1') # TODO(sissel): Include the git commit hash? out.iteration = "1" # what revision? - out.url = "http://www.elasticsearch.org/overview/logstash/" + out.url = "https://www.elastic.co/logstash" out.description = "An extensible logging pipeline" - out.vendor = "Elasticsearch" + out.vendor = "Elastic" # Because we made a mistake in naming the RC version numbers, both rpm and deb view # "1.5.0.rc1" higher than "1.5.0". Setting the epoch to 1 ensures that we get a kind diff --git a/versions.yml b/versions.yml index ae26e92cb..432795cb6 100644 --- a/versions.yml +++ b/versions.yml @@ -4,6 +4,12 @@ logstash: 8.0.0 logstash-core: 8.0.0 logstash-core-plugin-api: 2.1.16 +bundled_jdk: + # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1 + vendor: "adoptopenjdk" + revision: 11.0.8 + build: 10 + # jruby must reference a *released* version of jruby which can be downloaded from the official download url # *and* for which jars artifacts are published for compile-time jruby: