[ci] Add pipeline to trigger artifacts rebuild (#153818)

This adds a new pipeline that will check if the bundled version of
metricbeat and filebeat in our cloud image is up to date. If it's
outdated, this pipeline will trigger a rebuild.

---------

Co-authored-by: Tiago Costa <tiago.costa@elastic.co>

.buildkite/pipelines/artifacts_trigger.yml

@@ -0,0 +1,6 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/trigger.sh
+    label: Trigger artifacts build
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10

.buildkite/scripts/steps/artifacts/env.sh

@@ -23,9 +23,16 @@ else
   WORKFLOW="snapshot"
 fi
 
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_MANIFEST_FQDN="https://$ARTIFACTS_SUBDOMAIN.elastic.co"
+KIBANA_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/kibana/latest/$FULL_VERSION.json"
+BEATS_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/beats/latest/$FULL_VERSION.json"
+
 export VERSION_QUALIFIER
 export BASE_VERSION
 export QUALIFIER_VERSION
 export FULL_VERSION
 export BUILD_ARGS
 export WORKFLOW
+export KIBANA_MANIFEST_LATEST
+export BEATS_MANIFEST_LATEST

.buildkite/scripts/steps/artifacts/publish.sh

@@ -79,13 +79,12 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
         --dependency "beats:$BEATS_MANIFEST_URL" \
         --artifact-set main
 
-  ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+  KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url')
-  ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
 
   cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
   ### Artifacts Summary
 
-  $ARTIFACTS_SUMMARY
+  $KIBANA_SUMMARY
 EOF
 
 else

.buildkite/scripts/steps/artifacts/trigger.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/artifacts/env.sh
+
+BEATS_MANIFEST_LATEST_URL=$(curl  "$BEATS_MANIFEST_LATEST" | jq -r '.manifest_url')
+KIBANA_MANIFEST_URL=$(curl  "$KIBANA_MANIFEST_LATEST" | jq -r '.manifest_url')
+KIBANA_BEATS_MANIFEST_URL=$(curl $KIBANA_MANIFEST_URL | jq -r '.projects.kibana.dependencies[] | select(.prefix == "beats") | .build_uri')
+
+echo "--- Trigger artifact builds"
+if [ "$BEATS_MANIFEST_LATEST_URL" = "$KIBANA_BEATS_MANIFEST_URL" ]; then
+  echo "Kibana has the latest version of beats, skipping trigger"
+else
+  # Staging builds are not necessary on main
+  if [[ "$BUILDKITE_BRANCH" != "main" ]]; then
+    ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-artifacts-staging "$BUILDKITE_BRANCH"
+  fi
+  ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-artifacts-snapshot "$BUILDKITE_BRANCH"
+fi