[ci] Support ARM artifacts when using label ci:build-serverless-image (#163355)

Previously the `ci:build-serverless-image` label was hooked into the
default `Build distribution` step, which only builds the x64 linux
distribution. This limited build is in support of starting functional
tests as quickly as possible.

In order to support ARM artifacts, this starts a non-blocking parallel
build that cross compiles.

There's tradeoffs to this approach. CI time will be kept near our hour
target, at the cost of building x64 artifacts twice. Adding cross
compilation to the primary build step will extend CI time by ~30
minutes.

.buildkite/pipelines/artifacts_container_image.yml

@@ -1,6 +1,6 @@
 steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
-    label: Build default container images
+    label: Build serverless container images
     agents:
       queue: n2-16-spot
     timeout_in_minutes: 60

.buildkite/scripts/build_kibana.sh

@@ -41,32 +41,6 @@ if is_pr_with_label "ci:build-cloud-image"; then
 EOF
 fi
 
-if is_pr_with_label "ci:build-serverless-image"; then
-  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
-  node scripts/build \
-  --skip-initialize \
-  --skip-generic-folders \
-  --skip-platform-folders \
-  --skip-archives \
-  --docker-images \
-  --docker-namespace="kibana-ci" \
-  --docker-tag="pr-$BUILDKITE_PULL_REQUEST-$GIT_ABBREV_COMMIT" \
-  --docker-push \
-  --skip-docker-ubi \
-  --skip-docker-ubuntu \
-  --skip-docker-cloud \
-  --skip-docker-contexts
-  docker logout docker.elastic.co
-
-  SERVERLESS_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-serverless)
-  buildkite-agent meta-data set pr_comment:deploy_cloud:head "* Kibana Serverless Image: \`$SERVERLESS_IMAGE\`"
-  cat << EOF | buildkite-agent annotate --style "info" --context kibana-serverless-image
-
-  Kibana Serverless Image: \`$SERVERLESS_IMAGE\`
-EOF
-fi
-
 echo "--- Archive Kibana Distribution"
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$KIBANA_DIR/install/kibana"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -170,6 +170,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([/.*stor(ies|y).*/])) ||
       GITHUB_PR_LABELS.includes('ci:build-storybooks')

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -7,12 +7,19 @@ set -euo pipefail
 source .buildkite/scripts/steps/artifacts/env.sh
 
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
-KIBANA_IMAGE="docker.elastic.co/kibana-ci/kibana-serverless:git-$GIT_ABBREV_COMMIT"
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
+  KIBANA_IMAGE_TAG="git-$GIT_ABBREV_COMMIT"
+else
+  KIBANA_IMAGE_TAG="pr-$BUILDKITE_PULL_REQUEST-$GIT_ABBREV_COMMIT"
+fi
+
+KIBANA_IMAGE="docker.elastic.co/kibana-ci/kibana-serverless:$KIBANA_IMAGE_TAG"
 
 echo "--- Verify manifest does not already exist"
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
 trap 'docker logout docker.elastic.co' EXIT
 
+echo "Checking manifest for $KIBANA_IMAGE"
 if docker manifest inspect $KIBANA_IMAGE &> /dev/null; then
   echo "Manifest already exists, exiting"
   exit 1
@@ -25,7 +32,7 @@ node scripts/build \
   --docker-cross-compile \
   --docker-images \
   --docker-namespace="kibana-ci" \
-  --docker-tag="git-$GIT_ABBREV_COMMIT" \
+  --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
   --skip-docker-ubi \
   --skip-docker-cloud \
@@ -55,7 +62,7 @@ docker manifest push "$KIBANA_IMAGE"
 docker logout docker.elastic.co
 
 cat << EOF | buildkite-agent annotate --style "info" --context image
-  ### Container Images
+  ### Serverless Images
 
   Manifest: \`$KIBANA_IMAGE\`
 
@@ -64,6 +71,11 @@ cat << EOF | buildkite-agent annotate --style "info" --context image
   ARM64: \`$KIBANA_IMAGE-arm64\`
 EOF
 
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
+  buildkite-agent meta-data set pr_comment:build_serverless:head "* Kibana Serverless Image: \`$KIBANA_IMAGE\`"
+  buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
+fi
+
 echo "--- Build dependencies report"
 node scripts/licenses_csv_report "--csv=target/dependencies-$GIT_ABBREV_COMMIT.csv"
 
@@ -80,7 +92,7 @@ cd -
 # so that new stack instances contain the latest and greatest image of kibana,
 # and the respective stack components of course.
 echo "--- Trigger image tag update"
-if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
+if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] && [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
   cat << EOF | buildkite-agent pipeline upload
 steps:
   - label: ":argo: Update kibana image tag for kibana-controller using gpctl"