[artifacts] Pass dependency manifest to release-manager CLI (#142408)

This passes the beats manifest used to download metricbeat and filebeat
for our cloud image to the release-manager CLI. This will be used to
validate that the bundled versions we use match the released versions.

.buildkite/scripts/steps/artifacts/build.sh

@@ -29,5 +29,6 @@ if [ -d .beats ]; then
   cd .beats
   buildkite-agent artifact upload 'metricbeat-*'
   buildkite-agent artifact upload 'filebeat-*'
+  buildkite-agent artifact upload 'beats_manifest.json'
   cd -
 fi

.buildkite/scripts/steps/artifacts/publish.sh

@@ -58,6 +58,10 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
   export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
   export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   export VAULT_ADDR="https://secrets.elastic.co:8200"
+
+  download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json)
+
   docker run --rm \
     --name release-manager \
     -e VAULT_ADDR \
@@ -72,6 +76,7 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
         --workflow "$WORKFLOW" \
         --version "$BASE_VERSION" \
         --qualifier "$VERSION_QUALIFIER" \
+        --dependency "beats:$BEATS_MANIFEST_URL" \
         --artifact-set main
 
   ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"

src/dev/build/tasks/download_cloud_dependencies.ts

@@ -9,6 +9,7 @@
 import Path from 'path';
 import del from 'del';
 import Axios from 'axios';
+import Fsp from 'fs/promises';
 import { Task, downloadToDisk, downloadToString } from '../lib';
 
 export const DownloadCloudDependencies: Task = {
@@ -38,18 +39,41 @@ export const DownloadCloudDependencies: Task = {
       return Promise.all(downloads);
     };
 
+    const writeManifest = async (manifestUrl: string, manifestJSON: object) => {
+      const destination = config.resolveFromRepo('.beats', 'beats_manifest.json');
+      return Fsp.writeFile(
+        destination,
+        JSON.stringify(
+          {
+            manifest_url: manifestUrl,
+            ...manifestJSON,
+          },
+          null,
+          2
+        )
+      );
+    };
+
     let buildId = '';
+    let manifestUrl = '';
+    let manifestJSON = null;
     const buildUrl = `https://${subdomain}.elastic.co/beats/latest/${config.getBuildVersion()}.json`;
     try {
       const latest = await Axios.get(buildUrl);
       buildId = latest.data.build_id;
+      manifestUrl = latest.data.manifest_url;
+      manifestJSON = (await Axios.get(manifestUrl)).data;
+      if (!(manifestUrl && manifestJSON)) throw new Error('Missing manifest.');
     } catch (e) {
       log.error(`Unable to find Beats artifacts for ${config.getBuildVersion()} at ${buildUrl}.`);
       throw e;
     }
+
     await del([config.resolveFromRepo('.beats')]);
 
     await downloadBeat('metricbeat', buildId);
     await downloadBeat('filebeat', buildId);
+
+    await writeManifest(manifestUrl, manifestJSON);
   },
 };