Use jvm catalog for reproducible builds and expose new pipeline to check JDK availability (#16602)

Updates the existing `createElasticCatalogDownloadUrl` method to use the precise version retrieved `versions.yml` to download the JDK instead of using the latest of major version. This makes the build reproducible again.
Defines a new Gradle `checkNewJdkVersion` task to check if there is a new JDK version available from JVM catalog matching the same major of the current branch. 
Creates a new Buildkite pipeline to execute a `bash` script to run the Gradle task; plus it also update the `catalog-info.yaml` with the new pipeline and a trigger to execute every week.

.buildkite/jdk_availability_check_pipeline.yml

@@ -0,0 +1,7 @@
+steps:
+  - label: "JDK Availability check"
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/vm-agent.sh
+      ci/check_jdk_version_availability.sh

build.gradle

@@ -740,46 +740,66 @@ class JDKDetails {
         return createElasticCatalogDownloadUrl()
     }
 
-    private String createElasticCatalogDownloadUrl() {
+    // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog.
-        // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK
+    void checkLocalVersionMatchingLatest() {
-
+        // retrieve the metadata from remote
-        // arch x86_64 never used, only aarch64 if macos
         def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}"
-
-        // Append the cpu's arch only if Mac on aarch64, all the other OSes doesn't have CPU extension
-        if (arch == "aarch64") {
-            url += "_${arch}"
-        }
-        println "Retrieving JDK from catalog..."
         def catalogMetadataUrl = URI.create(url).toURL()
         def catalogConnection = catalogMetadataUrl.openConnection()
         catalogConnection.requestMethod = 'GET'
         assert catalogConnection.responseCode == 200
 
         def metadataRetrieved = catalogConnection.content.text
-        println "Retrieved!"
-
         def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved)
-        return catalogMetadata.url
-    }
 
-    private String createAdoptDownloadUrl() {
+        if (catalogMetadata.version != revision || catalogMetadata.revision != build) {
-        String releaseName = major > 8 ?
+            throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}")
-                "jdk-${revision}+${build}" :
-                "jdk${revision}u${build}"
-        String vendorOsName = vendorOsName(osName)
-        switch (vendor) {
-            case "adoptium":
-                return "https://api.adoptium.net/v3/binary/version/${releaseName}/${vendorOsName}/${arch}/jdk/hotspot/normal/adoptium"
-            default:
-                throw RuntimeException("Can't handle vendor: ${vendor}")
         }
     }
 
-    private String vendorOsName(String osName) {
+    private String createElasticCatalogDownloadUrl() {
-        if (osName == "darwin")
+        // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK
-            return "mac"
+
-        return osName
+        // arch x86_64 is default, aarch64 if macos or linux
+        def url = "https://jvm-catalog.elastic.co/jdk/adoptiumjdk-${revision}+${build}-${osName}"
+
+        // Append the cpu's arch only if not x86_64, which is the default
+        if (arch == "aarch64") {
+            url += "-${arch}"
+        }
+        println "Retrieving JDK from catalog..."
+        def catalogMetadataUrl = URI.create(url).toURL()
+        def catalogConnection = catalogMetadataUrl.openConnection()
+        catalogConnection.requestMethod = 'GET'
+        if (catalogConnection.responseCode != 200) {
+            println "Can't find adoptiumjdk ${revision} for ${osName} on Elastic JVM catalog"
+            throw new GradleException("JVM not present on catalog")
+        }
+
+        def metadataRetrieved = catalogConnection.content.text
+        println "Retrieved!"
+
+        def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved)
+        validateMetadata(catalogMetadata)
+
+        return catalogMetadata.url
+    }
+
+    //Verify that the artifact metadata correspond to the request, if not throws an error
+    private void validateMetadata(Map metadata) {
+        if (metadata.version != revision) {
+            throw new GradleException("Expected to retrieve a JDK for version ${revision} but received: ${metadata.version}")
+        }
+        if (!isSameArchitecture(metadata.architecture)) {
+            throw new GradleException("Expected to retrieve a JDK for architecture ${arch} but received: ${metadata.architecture}")
+        }
+    }
+
+    private boolean isSameArchitecture(String metadataArch) {
+        if (arch == 'x64') {
+            return metadataArch == 'x86_64'
+        }
+        return metadata.architecture == arch
     }
 
     private String parseJdkArchitecture(String jdkArch) {
@@ -791,7 +811,7 @@ class JDKDetails {
                 return "aarch64"
                 break
             default:
-                throw RuntimeException("Can't handle CPU architechture: ${jdkArch}")
+                throw new GradleException("Can't handle CPU architechture: ${jdkArch}")
         }
     }
 }
@@ -840,6 +860,15 @@ tasks.register("downloadJdk", Download) {
     }
 }
 
+tasks.register("checkNewJdkVersion") {
+    def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text)
+
+    // use Linux x86_64 as canary platform
+    def jdkDetails = new JDKDetails(versionYml, "linux", "x86_64")
+    // throws Gradle exception if local and remote doesn't match
+    jdkDetails.checkLocalVersionMatchingLatest()
+}
+
 tasks.register("deleteLocalJdk", Delete) {
     // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin]
     String osName = selectOsType()

catalog-info.yaml

@@ -33,6 +33,7 @@ spec:
     - resource:logstash-windows-jdk-matrix-pipeline
     - resource:logstash-benchmark-pipeline
     - resource:logstash-health-report-tests-pipeline
+    - resource:logstash-jdk-availability-check-pipeline
 
 # ***********************************
 # Declare serverless IT pipeline
@@ -572,6 +573,12 @@ spec:
           env:
             PIPELINES_TO_TRIGGER: 'logstash-exhaustive-tests-pipeline'
             EXCLUDE_BRANCHES: 'main'
+        JDK availability check:
+          branch: main
+          cronline: 0 2 * * 1  # every Monday@2AM UTC
+          message: Weekly trigger of JDK update availability pipeline per branch
+          env:
+            PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline'
       skip_intermediate_builds: true
       provider_settings:
         trigger_mode: none
@@ -745,4 +752,46 @@ spec:
 
 # *******************************
 # SECTION END: Health Report Tests pipeline
+# *******************************
+
+# ***********************************
+# Declare JDK check pipeline
+# ***********************************
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-jdk-availability-check-pipeline
+  description: ":logstash: check availability of new JDK version"
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
+  system: platform-ingest
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-jdk-availability-check-pipeline
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml"
+      maximum_timeout_in_minutes: 10
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+        SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'
+      teams:
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+
+# *******************************
+# SECTION END: JDK check pipeline
 # *******************************

ci/check_jdk_version_availability.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+
+echo "Checking local JDK version against latest remote from JVM catalog"
+./gradlew checkNewJdkVersion