[buildkite] Add most of the remaining periodic pipelines (#98043)

.buildkite/pipelines/periodic-packaging.bwc.template.yml

@@ -0,0 +1,15 @@
+      - label: "{{matrix.image}} / $BWC_VERSION / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v$BWC_VERSION
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: $BWC_VERSION

.buildkite/pipelines/periodic-packaging.template.yml

@@ -0,0 +1,52 @@
+steps:
+  - group: packaging-tests-unix
+    steps:
+      - label: "{{matrix.image}} / packaging-tests-unix"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructivePackagingTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-16-32768
+        env: {}
+  - group: packaging-tests-upgrade
+    steps: $BWC_STEPS
+  - group: packaging-tests-windows
+    steps:
+      - label: "{{matrix.image}} / packaging-tests-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env: {}

.buildkite/pipelines/periodic-platform-support.yml

@@ -0,0 +1,82 @@
+steps:
+  - group: platform-support-unix
+    steps:
+      - label: "{{matrix.image}} / platform-support-unix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+        timeout_in_minutes: 420
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-32-98304
+        env: {}
+  - group: platform-support-windows
+    steps:
+      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 420
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: platform-support-arm
+    steps:
+      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-arm"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true {{matrix.GRADLE_TASK}}
+        timeout_in_minutes: 420
+        matrix:
+          setup:
+            image:
+              - almalinux-8-aarch64
+              - ubuntu-2004-aarch64
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: aws
+          imagePrefix: elasticsearch-{{matrix.image}}
+          instanceType: m6g.8xlarge
+          diskSizeGb: 350
+          diskType: gp3
+          diskName: /dev/sda1
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"

.buildkite/pipelines/periodic.bwc.template.yml

@@ -0,0 +1,10 @@
+      - label: $BWC_VERSION / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: $BWC_VERSION

.buildkite/pipelines/periodic.template.yml

@@ -0,0 +1,105 @@
+steps:
+  - group: bwc
+    steps: $BWC_STEPS
+  - label: concurrent-search-tests
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: encryption-at-rest
+    command: .buildkite/scripts/encryption-at-rest.sh
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: eql-correctness
+    command: .buildkite/scripts/eql-correctness.sh
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - label: example-plugins
+    command: |-
+      cd $$WORKSPACE/plugins/examples
+
+      $$WORKSPACE/.ci/scripts/run-gradle.sh build
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - group: java-fips-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - openjdk17
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - graalvm-ce17
+              - openjdk17
+              - openjdk18
+              - openjdk19
+              - openjdk20
+              - openjdk21
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - label: release-tests
+    command: .buildkite/scripts/release-tests.sh
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: single-processor-node-tests
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      diskSizeGb: 350
+      machineType: custom-32-98304

.buildkite/pipelines/periodic.trigger.yml

@@ -1,16 +0,0 @@
-steps:
-  - trigger: elasticsearch-periodic
-    label: Trigger periodic pipeline for main
-    async: true
-    build:
-      branch: main
-  - trigger: elasticsearch-periodic
-    label: Trigger periodic pipeline for 8.9
-    async: true
-    build:
-      branch: "8.9"
-  - trigger: elasticsearch-periodic
-    label: Trigger periodic pipeline for 7.17
-    async: true
-    build:
-      branch: "7.17"

.buildkite/scripts/branches.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# This determines which branches will have pipelines triggered periodically, for dra workflows.
+BRANCHES=( $(cat branches.json | jq -r '.branches[].branch') )

.buildkite/scripts/encryption-at-rest.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -euo pipefail
+
+#!/bin/bash
+# Configure a dm-crypt volume backed by a file
+set -e
+dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=80GB
+dd if=/dev/urandom of=key.secret bs=2k count=1
+LOOP=$(losetup -f)
+sudo losetup $LOOP dm-crypt.img
+sudo cryptsetup luksFormat -q --key-file key.secret "$LOOP"
+sudo cryptsetup open --key-file key.secret "$LOOP" secret --verbose
+sudo mkfs.ext2 /dev/mapper/secret
+sudo mkdir /mnt/secret
+# Change /mnt/secret with care (at least a test uses this path to detect when encryption at rest is used)
+sudo mount /dev/mapper/secret /mnt/secret
+sudo chown -R buildkite-agent /mnt/secret
+cp -r "$WORKSPACE" /mnt/secret
+cd /mnt/secret/$(basename "$WORKSPACE")
+touch .output.log
+rm -Rf "$WORKSPACE"
+ln -s "$PWD" "$WORKSPACE"
+
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check

.buildkite/scripts/eql-correctness.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+eql_test_credentials_file="$(pwd)/x-pack/plugin/eql/qa/correctness/credentials.gcs.json"
+export eql_test_credentials_file
+vault read -field=credentials.gcs.json secret/ci/elastic-elasticsearch/migrated/eql_test_credentials > "${eql_test_credentials_file}"
+
+.ci/scripts/run-gradle.sh -Dignore.tests.seed :x-pack:plugin:eql:qa:correctness:check

.buildkite/scripts/periodic.trigger.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "steps:"
+
+source .buildkite/scripts/branches.sh
+
+for BRANCH in "${BRANCHES[@]}"; do
+  INTAKE_PIPELINE_SLUG="elasticsearch-intake"
+  BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
+  LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
+
+  cat <<EOF
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+  - trigger: elasticsearch-periodic-packaging
+    label: Trigger periodic-packaging pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+  - trigger: elasticsearch-periodic-platform-support
+    label: Trigger periodic-platform-support pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+EOF
+done

.buildkite/scripts/release-tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Fetch beats artifacts
+export ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
+export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
+
+mkdir -p ${BEATS_DIR}
+curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
+curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
+
+# Fetch ML artifacts
+export ML_IVY_REPO=$(mktemp -d)
+mkdir -p ${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}
+curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-deps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-deps.zip
+curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-nodeps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-nodeps.zip
+curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
+
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
+  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

build.gradle

@@ -72,9 +72,35 @@ tasks.register("updateCIBwcVersions") {
       file << "  - \"$it\"\n"
     }
   }
+
+  def writeBuildkiteSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
+    def outputFile = file(outputFilePath)
+    def pipelineTemplate = file(pipelineTemplatePath)
+    def stepTemplate = file(stepTemplatePath)
+
+    def steps = ""
+    versions.each {
+      steps += "\n" + stepTemplate.text.replaceAll('\\$BWC_VERSION', it.toString())
+    }
+
+    outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipelineTemplate.text.replaceAll(' *\\$BWC_STEPS', steps)
+  }
+
   doLast {
     writeVersions(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
     writeVersions(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
+    writeBuildkiteSteps(
+      ".buildkite/pipelines/periodic.yml",
+      ".buildkite/pipelines/periodic.template.yml",
+      ".buildkite/pipelines/periodic.bwc.template.yml",
+      BuildParams.bwcVersions.allIndexCompatible
+    )
+    writeBuildkiteSteps(
+      ".buildkite/pipelines/periodic-packaging.yml",
+      ".buildkite/pipelines/periodic-packaging.template.yml",
+      ".buildkite/pipelines/periodic-packaging.bwc.template.yml",
+      BuildParams.bwcVersions.allIndexCompatible
+    )
   }
 }
 

catalog-info.yaml

@@ -104,7 +104,7 @@ apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
   name: buildkite-pipeline-elasticsearch-periodic-trigger
-  description: Triggers periodic pipeline for all required branches
+  description: Triggers periodic pipelines for all required branches
   links:
     - title: Pipeline
       url: https://buildkite.com/elastic/elasticsearch-periodic-trigger
@@ -116,11 +116,11 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      description: ":elasticsearch: Triggers periodic pipeline for all required branches"
+      description: ":elasticsearch: Triggers periodic pipelines for all required branches"
       name: elasticsearch / periodic / trigger
     spec:
       repository: elastic/elasticsearch
-      pipeline_file: .buildkite/pipelines/periodic.trigger.yml
+      pipeline_file: .buildkite/scripts/periodic.trigger.sh
       branch_configuration: main
       teams:
         elasticsearch-team: {}