[ci] Allow schedule to trigger several pipelines (#15703)

This commit enhances the functionality introduced in #15668 and #15700
by allowing a single Buildkite scheduling job to trigger several
pipelines, in addition to multiple branches which it already does.

We rename the env var PIPELINE_TO_TRIGGER to PIPELINES_TO_TRIGGER
which now supports comma separate values.

This enhancement can be useful for pipelines like JDK matrix which
have variants (Linux and Windows) that we want to trigger with a single
scheduling job, thus reducing unnecessary entries in catalog-info.

.buildkite/scripts/common/trigger-pipeline-generate-steps.sh

@@ -22,13 +22,19 @@ if ! [[ -x $(which yq) && $(yq --version) == *mikefarah* ]]; then
 fi
 }
 
-if [[ -z $PIPELINE_TO_TRIGGER ]]; then
+function parse_pipelines() {
+  IFS="," read -ra PIPELINES <<< "$PIPELINES_TO_TRIGGER"
+}
+
+if [[ -z $PIPELINES_TO_TRIGGER ]]; then
     echo "^^^ +++"
-    echo "Required environment variable [PIPELINE_TO_TRIGGER] is missing."
+    echo "Required environment variable [PIPELINES_TO_TRIGGER] is missing."
     echo "Exiting now."
     exit 1
 fi
 
+parse_pipelines
+
 set -u
 set +e
 BRANCHES=$(curl --retry-all-errors --retry 5 --retry-delay 5 -fsSL $ACTIVE_BRANCHES_URL | jq -r '.branches[].branch')
@@ -43,14 +49,16 @@ install_yq
 
 echo 'steps:' >pipeline_steps.yaml
 
-for BRANCH in $BRANCHES; do
-    cat >>pipeline_steps.yaml <<EOF
-  - trigger: $PIPELINE_TO_TRIGGER
-    label: ":testexecute: Triggering ${PIPELINE_TO_TRIGGER} / ${BRANCH}"
+for PIPELINE in "${PIPELINES[@]}"; do
+  for BRANCH in $BRANCHES; do
+      cat >>pipeline_steps.yaml <<EOF
+  - trigger: $PIPELINE
+    label: ":testexecute: Triggering ${PIPELINE} / ${BRANCH}"
     build:
       branch: "$BRANCH"
       message: ":testexecute: Scheduled build for ${BRANCH}"
 EOF
+  done
 done
 
 echo "--- Printing generated steps"

.buildkite/trigger_pipelines.yml

@@ -1,5 +1,5 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 steps:
-  - label: ":pipeline: Generate trigger steps for $PIPELINE_TO_TRIGGER"
+  - label: ":pipeline: Generate trigger steps for $PIPELINES_TO_TRIGGER"
     command: ".buildkite/scripts/common/trigger-pipeline-generate-steps.sh"

catalog-info.yaml

@@ -185,7 +185,7 @@ spec:
       description: ':alarm_clock: Scheduled runs of Logstash DRA Snapshot Pipeline per eligible branch'
     spec:
       repository: elastic/logstash
-      pipeline_file: ".buildkite/trigger_pipeline.yml"
+      pipeline_file: ".buildkite/trigger_pipelines.yml"
       maximum_timeout_in_minutes: 120
       schedules:
         Daily main:
@@ -196,7 +196,7 @@ spec:
       provider_settings:
         trigger_mode: none
       env:
-        PIPELINE_TO_TRIGGER: 'logstash-dra-snapshot-pipeline'
+        PIPELINES_TO_TRIGGER: 'logstash-dra-snapshot-pipeline'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#logstash'
         SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'