Introduced JDK environment variable to explicitly pass the JAVA_HOME to use and defined .ci/ with OS and JDK preferences (#11934)

.ci/matrix-unix-compatibility-linux-distros.yml

@@ -0,0 +1,28 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the Linux distributions on
+# which Logstash will be tested.
+
+#os:
+#  - amazon
+#  - centos-6&&immutable
+#  - centos-7&&immutable
+#  - debian-8&&immutable
+#  - debian-9&&immutable
+#  - debian-10&&immutable
+#  - fedora-29&&immutable
+#  - opensuse-15-1&&immutable
+#  - oraclelinux-6&&immutable
+#  - oraclelinux-7&&immutable
+#  - ubuntu-18.04&&immutable
+#  - ubuntu-20.04&&immutable
+
+os:
+  - amazon
+  - centos&&immutable
+  - debian&&immutable
+  - fedora-29&&immutable
+  - opensuse-15-1&&immutable
+  - oraclelinux&&immutable
+  - ubuntu&&immutable

.ci/matrix-unix-linux-distros.yml

@@ -0,0 +1,9 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the Linux distributions on
+# which Logstash will be tested.
+
+os:
+  - centos-7&&immutable
+  - ubuntu-18.04&&immutable

.ci/matrix-unix-runtime-javas.yml

@@ -6,13 +6,7 @@
 # or 'openjdk' followed by the major release number.
 
 LS_RUNTIME_JAVA:
-#  - java8
-#  - zulu8
-#  - adoptopenjdk8
-#  - java11
   - openjdk11
   - adoptopenjdk11
-  - zulu11
   - openjdk14
-  - adoptopenjdk14
+  - adoptopenjdk14
-  - zulu14

.ci/matrix-windows-compatibility-versions.yml

@@ -0,0 +1,10 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the Linux distributions on
+# which Logstash will be tested.
+
+nodes:
+  - "windows-2012-r2"
+  - "windows-2016"
+  - "windows-2019"

.ci/matrix-windows-runtime-javas.yml

@@ -9,4 +9,5 @@ LS_RUNTIME_JAVA:
 #  - zulu8
 #  - adoptopenjdk8
   - zulu11
+  - zulu14
 #  - adoptopenjdk11

ci/acceptance_tests.sh

@@ -6,7 +6,7 @@ set -x
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 export OSS=true
 
 SELECTED_TEST_SUITE=$1

ci/docker_run.sh

@@ -6,6 +6,7 @@ set -x # We want verbosity here, this mostly runs on CI and we want to easily de
 #Note - ensure that the -e flag is NOT set, and explicitly check the $? status to allow for clean up
 
 REMOVE_IMAGE=false
+DOCKER_EXTERNAL_JDK=""
 if [ -z "$branch_specifier" ]; then
     # manual
     REMOVE_IMAGE=true
@@ -33,8 +34,13 @@ cleanup() {
 }
 trap cleanup EXIT
 
+if [ -n "$JDK" ]; then
+  echo "JDK to use $JDK"
+  DOCKER_EXTERNAL_JDK="--mount type=bind,source=$JDK,target=$JDK,readonly --env BUILD_JAVA_HOME=$JDK"
+fi
+
 # Run the command, skip the first argument, which is the image name
-docker run $DOCKER_ENV_OPTS --cidfile=docker_cid --sig-proxy=true --rm $IMAGE_NAME ${@:2}
+docker run $DOCKER_ENV_OPTS --cidfile=docker_cid --sig-proxy=true $DOCKER_EXTERNAL_JDK --rm $IMAGE_NAME ${@:2}
 exit_code=$?
 
 # Remove the container cid since we ran cleanly, no need to force rm it if we got to this point

ci/integration_tests.sh

@@ -5,12 +5,16 @@
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
 export OSS=true
 
+if [ -n "$BUILD_JAVA_HOME" ]; then
+  GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
+fi
+
 if [[ $1 = "setup" ]]; then
  echo "Setup only, no tests will be run"
  exit 0

ci/unit_tests.bat

@@ -40,9 +40,14 @@ echo Using drive !use_drive! for %WORKSPACE%
 !use_drive!
 
 echo Running core tests..
-if "%BUILD_JAVA_HOME%" == "" (
+if defined BUILD_JAVA_HOME (
-  GRADLE_OPTS="%GRADLE_OPTS% -Dorg.gradle.java.home=%BUILD_JAVA_HOME%"
+  if defined GRADLE_OPTS (
+    set GRADLE_OPTS=%GRADLE_OPTS% -Dorg.gradle.java.home=%BUILD_JAVA_HOME%
+  ) else (
+    set GRADLE_OPTS=-Dorg.gradle.java.home=%BUILD_JAVA_HOME%
+  )
 )
+echo Invoking Gradle, GRADLE_OPTS: %GRADLE_OPTS%, BUILD_JAVA_HOME: %BUILD_JAVA_HOME%
 call .\gradlew.bat test --console=plain --no-daemon --info
 
 if errorlevel 1 (

ci/unit_tests.sh

@@ -5,13 +5,17 @@
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
 export OSS=true
 export TEST_DEBUG=true
 
+if [ -n "$BUILD_JAVA_HOME" ]; then
+  GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
+fi
+
 SELECTED_TEST_SUITE=$1
 
 if [[ $SELECTED_TEST_SUITE == $"java" ]]; then

qa/integration/services/logstash_service.rb

@@ -118,9 +118,11 @@ class LogstashService < Service
       # pipe STDOUT and STDERR to a file
       @process.io.stdout = @process.io.stderr = out
       @process.duplex = true
+      java_home = java.lang.System.getProperty('java.home')
+      @process.environment['JAVA_HOME'] = java_home
       @process.start
       wait_for_logstash
-      puts "Logstash started with PID #{@process.pid}" if alive?
+      puts "Logstash started with PID #{@process.pid}, JAVA_HOME: #{java_home}" if alive?
     end
   end
 
@@ -135,9 +137,11 @@ class LogstashService < Service
     Bundler.with_clean_env do
       @process = build_child_process(*args)
       @env_variables.map { |k, v|  @process.environment[k] = v} unless @env_variables.nil?
+      java_home = java.lang.System.getProperty('java.home')
+      @process.environment['JAVA_HOME'] = java_home
       @process.io.inherit!
       @process.start
-      puts "Logstash started with PID #{@process.pid}" if @process.alive?
+      puts "Logstash started with PID #{@process.pid}, JAVA_HOME: #{java_home}" if @process.alive?
     end
   end
 

x-pack/ci/docker_integration_tests.sh

@@ -3,9 +3,7 @@
 # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 # or more contributor license agreements. Licensed under the Elastic License;
 # you may not use this file except in compliance with the Elastic License.
-
 if [ -n "${ELASTICSEARCH_SNAPSHOT_URL}" ]; then
   export DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} --env ELASTICSEARCH_SNAPSHOT_URL=${ELASTICSEARCH_SNAPSHOT_URL}"
 fi
-
 ci/docker_run.sh logstash-xpack-integration-tests x-pack/ci/integration_tests.sh $@

x-pack/ci/docker_unit_tests.sh

@@ -3,5 +3,4 @@
 # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 # or more contributor license agreements. Licensed under the Elastic License;
 # you may not use this file except in compliance with the Elastic License.
-
 ci/docker_run.sh logstash-xpack-unit-tests x-pack/ci/unit_tests.sh $@

x-pack/ci/integration_tests.sh

@@ -9,7 +9,11 @@
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
 export CI=true
 
+if [ -n "$BUILD_JAVA_HOME" ]; then
+  GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
+fi
+
 ./gradlew runXPackIntegrationTests

x-pack/ci/unit_tests.sh

@@ -9,7 +9,11 @@
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
 export CI=true
 
+if [ -n "$BUILD_JAVA_HOME" ]; then
+  GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
+fi
+
 ./gradlew runXPackUnitTests