[ci] Disable Bazel cache on CI (#115428) (#115660)

This is a short-term solution to reduce our hit count in Buildkite due to the increase in bootstraps so we can move forward with enabling Buildkite across all PRs.

Co-authored-by: Tyler Smalley <tyler.smalley@elastic.co>

.bazelrc

@@ -10,9 +10,6 @@ build --remote_timeout=30
 build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
 build --remote_accept_cached=true
 
-# BuildBuddy
-## Metadata settings
-build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.bazelrc.common

@@ -120,8 +120,8 @@ test --incompatible_strict_action_env
 # collect coverage information from test targets
 coverage --instrument_test_targets
 
-# Settings for CI
-# Bazel flags for CI are in /src/dev/ci_setup/.bazelrc-ci
+# Metadata settings
+build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members

.buildkite/pipelines/on_merge.yml

@@ -10,6 +10,8 @@ steps:
 
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Default Build and Metrics
+    env:
+      BAZEL_CACHE_MODE: read-write
     agents:
       queue: c2-8
     timeout_in_minutes: 60

.buildkite/scripts/bootstrap.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/common/setup_bazel.sh
 
 echo "--- yarn install and bootstrap"
 retry 2 15 yarn kbn bootstrap

.buildkite/scripts/common/persist_bazel_cache.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-source .buildkite/scripts/common/util.sh
-
-KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
-export KIBANA_BUILDBUDDY_CI_API_KEY
-
-# overwrites the file checkout .bazelrc file with the one intended for CI env
-cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc"
-
-###
-### append auth token to buildbuddy into "$KIBANA_DIR/.bazelrc";
-###
-echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc"
-echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc"

.buildkite/scripts/common/setup_bazel.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+echo "[bazel] writing .bazelrc"
+cat <<EOF > $KIBANA_DIR/.bazelrc
+  # Generated by .buildkite/scripts/common/setup_bazel.sh
+
+  import %workspace%/.bazelrc.common
+
+  build --build_metadata=ROLE=CI
+EOF
+
+if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
+  echo "[bazel] enabling caching"
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --bes_results_url=https://app.buildbuddy.io/invocation/
+  build --bes_backend=grpcs://cloud.buildbuddy.io
+  build --remote_cache=grpcs://cloud.buildbuddy.io
+  build --remote_timeout=3600
+  build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
+EOF
+fi
+
+if [[ "${BAZEL_CACHE_MODE:-none}" == "read" ]]; then
+  echo "[bazel] cache set to read-only"
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --noremote_upload_local_results
+EOF
+fi
+
+if [[ "${BAZEL_CACHE_MODE:-none}" != @(read|read-write|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [read,read-write,none]"
+  exit 1
+fi

.buildkite/scripts/common/util.sh

@@ -18,7 +18,7 @@ verify_no_git_changes() {
   RED='\033[0;31m'
   C_RESET='\033[0m' # Reset color
 
-  GIT_CHANGES="$(git ls-files --modified)"
+  GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
   if [ "$GIT_CHANGES" ]; then
     echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
     echo -e "$GIT_CHANGES\n"

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -2,9 +2,6 @@
 
 set -euo pipefail
 
-# Write Bazel cache for Linux
-.buildkite/scripts/common/persist_bazel_cache.sh
-
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/build_kibana.sh
 .buildkite/scripts/post_build_kibana.sh

src/dev/ci_setup/.bazelrc-ci

@@ -1,15 +0,0 @@
-# Used in the on-merge job to persist the Bazel cache to BuildBuddy
-# from: .buildkite/scripts/common/persist_bazel_cache.sh
-
-import %workspace%/.bazelrc.common
-
-# BuildBuddy settings
-build --bes_results_url=https://app.buildbuddy.io/invocation/
-build --bes_backend=grpcs://cloud.buildbuddy.io
-build --remote_cache=grpcs://cloud.buildbuddy.io
-build --remote_timeout=3600
-# --remote_header=x-buildbuddy-api-key= # appended in CI script
-
-# Metadata settings
-build --build_metadata=ROLE=CI
-build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"