[CI] Improve reliability of macos bazel cache jobs (#138554)

.buildkite/pipelines/bazel_cache.yml

@@ -26,10 +26,18 @@ steps:
         concurrency_group: bazel_macos
         concurrency: 1
         concurrency_method: eager
+        retry:
+          automatic:
+          - exit_status: '*'
+            limit: 1
       - command: .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
         label: Bootstrap (MacOS ARM)
         priority: $${PRIORITY}
         agents:
           queue: macos-arm
         timeout_in_minutes: 60
+        retry:
+          automatic:
+          - exit_status: '*'
+            limit: 1
       YAML

.buildkite/scripts/bootstrap.sh

@@ -7,6 +7,11 @@ source .buildkite/scripts/common/setup_bazel.sh
 
 echo "--- yarn install and bootstrap"
 
+BOOTSTRAP_PARAMS=()
+if [[ "${BOOTSTRAP_ALWAYS_FORCE_INSTALL:-}" ]]; then
+  BOOTSTRAP_PARAMS+=(--force-install)
+fi
+
 # Use the node_modules that is baked into the agent image, if it exists, as a cache
 # But only for agents not mounting the workspace on a local ssd or in memory
 # It actually ends up being slower to move all of the tiny files between the disks vs extracting archives from the yarn cache
@@ -15,7 +20,7 @@ if [[ -d ~/.kibana/node_modules && "$(pwd)" != *"/local-ssd/"* && "$(pwd)" != "/
   mv ~/.kibana/node_modules ./
 fi
 
-if ! yarn kbn bootstrap; then
+if ! yarn kbn bootstrap "${BOOTSTRAP_PARAMS[@]}"; then
   echo "bootstrap failed, trying again in 15 seconds"
   sleep 15
 

.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh

@@ -7,6 +7,9 @@ source .buildkite/scripts/common/util.sh
 export BAZEL_CACHE_MODE=populate-local-gcs
 export DISABLE_BOOTSTRAP_VALIDATION=true
 
+# Because we're manually deleting node_modules and bazel directories in-between runs, we need to --force-install
+export BOOTSTRAP_ALWAYS_FORCE_INSTALL=true
+
 # Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
 rm -rf ~/.bazel-cache