From c5b6a853d6d2099b0d983be7fcac0a6b5522a231 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 27 May 2020 16:48:39 +0200 Subject: [PATCH] Introduced JDK environment variable to explicitly pass the JAVA_HOME to use and defined .ci/ with OS and JDK preferences (#11934) --- ...atrix-unix-compatibility-linux-distros.yml | 28 +++++++++++++++++++ .ci/matrix-unix-linux-distros.yml | 9 ++++++ .ci/matrix-unix-runtime-javas.yml | 8 +----- .ci/matrix-windows-compatibility-versions.yml | 10 +++++++ .ci/matrix-windows-runtime-javas.yml | 1 + ci/acceptance_tests.sh | 2 +- ci/docker_run.sh | 8 +++++- ci/integration_tests.sh | 6 +++- ci/unit_tests.bat | 9 ++++-- ci/unit_tests.sh | 6 +++- qa/integration/services/logstash_service.rb | 8 ++++-- x-pack/ci/docker_integration_tests.sh | 2 -- x-pack/ci/docker_unit_tests.sh | 1 - x-pack/ci/integration_tests.sh | 6 +++- x-pack/ci/unit_tests.sh | 6 +++- 15 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 .ci/matrix-unix-compatibility-linux-distros.yml create mode 100644 .ci/matrix-unix-linux-distros.yml create mode 100644 .ci/matrix-windows-compatibility-versions.yml diff --git a/.ci/matrix-unix-compatibility-linux-distros.yml b/.ci/matrix-unix-compatibility-linux-distros.yml new file mode 100644 index 000000000..eeb68ab09 --- /dev/null +++ b/.ci/matrix-unix-compatibility-linux-distros.yml @@ -0,0 +1,28 @@ +# This file is used as part of a matrix build in Jenkins where the +# values below are included as an axis of the matrix. + +# This axis of the build matrix represents the Linux distributions on +# which Logstash will be tested. + +#os: +# - amazon +# - centos-6&&immutable +# - centos-7&&immutable +# - debian-8&&immutable +# - debian-9&&immutable +# - debian-10&&immutable +# - fedora-29&&immutable +# - opensuse-15-1&&immutable +# - oraclelinux-6&&immutable +# - oraclelinux-7&&immutable +# - ubuntu-18.04&&immutable +# - ubuntu-20.04&&immutable + +os: + - amazon + - centos&&immutable + - debian&&immutable + - fedora-29&&immutable + - opensuse-15-1&&immutable + - oraclelinux&&immutable + - ubuntu&&immutable diff --git a/.ci/matrix-unix-linux-distros.yml b/.ci/matrix-unix-linux-distros.yml new file mode 100644 index 000000000..fdfb75dea --- /dev/null +++ b/.ci/matrix-unix-linux-distros.yml @@ -0,0 +1,9 @@ +# This file is used as part of a matrix build in Jenkins where the +# values below are included as an axis of the matrix. + +# This axis of the build matrix represents the Linux distributions on +# which Logstash will be tested. + +os: + - centos-7&&immutable + - ubuntu-18.04&&immutable \ No newline at end of file diff --git a/.ci/matrix-unix-runtime-javas.yml b/.ci/matrix-unix-runtime-javas.yml index 1857ed9bc..d97c9d82b 100644 --- a/.ci/matrix-unix-runtime-javas.yml +++ b/.ci/matrix-unix-runtime-javas.yml @@ -6,13 +6,7 @@ # or 'openjdk' followed by the major release number. LS_RUNTIME_JAVA: -# - java8 -# - zulu8 -# - adoptopenjdk8 -# - java11 - openjdk11 - adoptopenjdk11 - - zulu11 - openjdk14 - - adoptopenjdk14 - - zulu14 \ No newline at end of file + - adoptopenjdk14 \ No newline at end of file diff --git a/.ci/matrix-windows-compatibility-versions.yml b/.ci/matrix-windows-compatibility-versions.yml new file mode 100644 index 000000000..b3b9f3585 --- /dev/null +++ b/.ci/matrix-windows-compatibility-versions.yml @@ -0,0 +1,10 @@ +# This file is used as part of a matrix build in Jenkins where the +# values below are included as an axis of the matrix. + +# This axis of the build matrix represents the Linux distributions on +# which Logstash will be tested. + +nodes: + - "windows-2012-r2" + - "windows-2016" + - "windows-2019" diff --git a/.ci/matrix-windows-runtime-javas.yml b/.ci/matrix-windows-runtime-javas.yml index 2a3fc7388..63b11fef2 100644 --- a/.ci/matrix-windows-runtime-javas.yml +++ b/.ci/matrix-windows-runtime-javas.yml @@ -9,4 +9,5 @@ LS_RUNTIME_JAVA: # - zulu8 # - adoptopenjdk8 - zulu11 + - zulu14 # - adoptopenjdk11 diff --git a/ci/acceptance_tests.sh b/ci/acceptance_tests.sh index 67bd1af28..acb556626 100755 --- a/ci/acceptance_tests.sh +++ b/ci/acceptance_tests.sh @@ -6,7 +6,7 @@ set -x # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 export JRUBY_OPTS="-J-Xmx1g" -export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" export OSS=true SELECTED_TEST_SUITE=$1 diff --git a/ci/docker_run.sh b/ci/docker_run.sh index 4963cc05a..f3c088c1c 100755 --- a/ci/docker_run.sh +++ b/ci/docker_run.sh @@ -6,6 +6,7 @@ set -x # We want verbosity here, this mostly runs on CI and we want to easily de #Note - ensure that the -e flag is NOT set, and explicitly check the $? status to allow for clean up REMOVE_IMAGE=false +DOCKER_EXTERNAL_JDK="" if [ -z "$branch_specifier" ]; then # manual REMOVE_IMAGE=true @@ -33,8 +34,13 @@ cleanup() { } trap cleanup EXIT +if [ -n "$JDK" ]; then + echo "JDK to use $JDK" + DOCKER_EXTERNAL_JDK="--mount type=bind,source=$JDK,target=$JDK,readonly --env BUILD_JAVA_HOME=$JDK" +fi + # Run the command, skip the first argument, which is the image name -docker run $DOCKER_ENV_OPTS --cidfile=docker_cid --sig-proxy=true --rm $IMAGE_NAME ${@:2} +docker run $DOCKER_ENV_OPTS --cidfile=docker_cid --sig-proxy=true $DOCKER_EXTERNAL_JDK --rm $IMAGE_NAME ${@:2} exit_code=$? # Remove the container cid since we ran cleanly, no need to force rm it if we got to this point diff --git a/ci/integration_tests.sh b/ci/integration_tests.sh index 4954d4420..5d6576022 100755 --- a/ci/integration_tests.sh +++ b/ci/integration_tests.sh @@ -5,12 +5,16 @@ # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 export JRUBY_OPTS="-J-Xmx1g" -export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" export SPEC_OPTS="--order rand --format documentation" export CI=true export OSS=true +if [ -n "$BUILD_JAVA_HOME" ]; then + GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" +fi + if [[ $1 = "setup" ]]; then echo "Setup only, no tests will be run" exit 0 diff --git a/ci/unit_tests.bat b/ci/unit_tests.bat index f9c9f5481..5d7858657 100644 --- a/ci/unit_tests.bat +++ b/ci/unit_tests.bat @@ -40,9 +40,14 @@ echo Using drive !use_drive! for %WORKSPACE% !use_drive! echo Running core tests.. -if "%BUILD_JAVA_HOME%" == "" ( - GRADLE_OPTS="%GRADLE_OPTS% -Dorg.gradle.java.home=%BUILD_JAVA_HOME%" +if defined BUILD_JAVA_HOME ( + if defined GRADLE_OPTS ( + set GRADLE_OPTS=%GRADLE_OPTS% -Dorg.gradle.java.home=%BUILD_JAVA_HOME% + ) else ( + set GRADLE_OPTS=-Dorg.gradle.java.home=%BUILD_JAVA_HOME% + ) ) +echo Invoking Gradle, GRADLE_OPTS: %GRADLE_OPTS%, BUILD_JAVA_HOME: %BUILD_JAVA_HOME% call .\gradlew.bat test --console=plain --no-daemon --info if errorlevel 1 ( diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index 113d544d6..5968dd7d4 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -5,13 +5,17 @@ # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 export JRUBY_OPTS="-J-Xmx1g" -export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" export SPEC_OPTS="--order rand --format documentation" export CI=true export OSS=true export TEST_DEBUG=true +if [ -n "$BUILD_JAVA_HOME" ]; then + GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" +fi + SELECTED_TEST_SUITE=$1 if [[ $SELECTED_TEST_SUITE == $"java" ]]; then diff --git a/qa/integration/services/logstash_service.rb b/qa/integration/services/logstash_service.rb index 33539b7f3..630822916 100644 --- a/qa/integration/services/logstash_service.rb +++ b/qa/integration/services/logstash_service.rb @@ -118,9 +118,11 @@ class LogstashService < Service # pipe STDOUT and STDERR to a file @process.io.stdout = @process.io.stderr = out @process.duplex = true + java_home = java.lang.System.getProperty('java.home') + @process.environment['JAVA_HOME'] = java_home @process.start wait_for_logstash - puts "Logstash started with PID #{@process.pid}" if alive? + puts "Logstash started with PID #{@process.pid}, JAVA_HOME: #{java_home}" if alive? end end @@ -135,9 +137,11 @@ class LogstashService < Service Bundler.with_clean_env do @process = build_child_process(*args) @env_variables.map { |k, v| @process.environment[k] = v} unless @env_variables.nil? + java_home = java.lang.System.getProperty('java.home') + @process.environment['JAVA_HOME'] = java_home @process.io.inherit! @process.start - puts "Logstash started with PID #{@process.pid}" if @process.alive? + puts "Logstash started with PID #{@process.pid}, JAVA_HOME: #{java_home}" if @process.alive? end end diff --git a/x-pack/ci/docker_integration_tests.sh b/x-pack/ci/docker_integration_tests.sh index 479154337..fdb084a71 100755 --- a/x-pack/ci/docker_integration_tests.sh +++ b/x-pack/ci/docker_integration_tests.sh @@ -3,9 +3,7 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the Elastic License; # you may not use this file except in compliance with the Elastic License. - if [ -n "${ELASTICSEARCH_SNAPSHOT_URL}" ]; then export DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} --env ELASTICSEARCH_SNAPSHOT_URL=${ELASTICSEARCH_SNAPSHOT_URL}" fi - ci/docker_run.sh logstash-xpack-integration-tests x-pack/ci/integration_tests.sh $@ diff --git a/x-pack/ci/docker_unit_tests.sh b/x-pack/ci/docker_unit_tests.sh index 74ed9dc3e..f76b645ce 100755 --- a/x-pack/ci/docker_unit_tests.sh +++ b/x-pack/ci/docker_unit_tests.sh @@ -3,5 +3,4 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the Elastic License; # you may not use this file except in compliance with the Elastic License. - ci/docker_run.sh logstash-xpack-unit-tests x-pack/ci/unit_tests.sh $@ diff --git a/x-pack/ci/integration_tests.sh b/x-pack/ci/integration_tests.sh index 79e616b77..e85c89504 100755 --- a/x-pack/ci/integration_tests.sh +++ b/x-pack/ci/integration_tests.sh @@ -9,7 +9,11 @@ # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 export JRUBY_OPTS="-J-Xmx1g" -export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" export CI=true +if [ -n "$BUILD_JAVA_HOME" ]; then + GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" +fi + ./gradlew runXPackIntegrationTests \ No newline at end of file diff --git a/x-pack/ci/unit_tests.sh b/x-pack/ci/unit_tests.sh index 182f0138b..8ac523c4a 100755 --- a/x-pack/ci/unit_tests.sh +++ b/x-pack/ci/unit_tests.sh @@ -9,7 +9,11 @@ # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 export JRUBY_OPTS="-J-Xmx1g" -export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" export CI=true +if [ -n "$BUILD_JAVA_HOME" ]; then + GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" +fi + ./gradlew runXPackUnitTests \ No newline at end of file