From 48ca660c5044350f6f6388910bd957de5cc6ae7f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 18 Apr 2022 02:01:38 +0100 Subject: [PATCH] chore(NA): use new and more performant BuildBuddy servers (#130350) * chore(NA): use new and more performant BuildBuddy servers * chore(NA): simple upgrade mechanism for new remote address servers * docs(NA): missing note for future removal Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .bazelrc | 8 ----- .buildkite/scripts/common/setup_bazel.sh | 4 +-- packages/kbn-pm/dist/index.js | 31 ++++++++++++++++-- .../src/utils/bazel/setup_remote_cache.ts | 32 +++++++++++++++++-- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/.bazelrc b/.bazelrc index 9278352e686e..c4029cd0bab4 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,14 +2,6 @@ # Import shared settings first so we can override below import %workspace%/.bazelrc.common -# Remote cache settings for local env -# build --remote_cache=grpcs://cloud.buildbuddy.io -# build --incompatible_remote_results_ignore_disk=true -# build --noremote_upload_local_results -# build --remote_timeout=30 -# build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze -# build --remote_accept_cached=true - # Enable this in case you want to share your build info # build --build_metadata=VISIBILITY=PUBLIC build --build_metadata=TEST_GROUPS=//packages diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh index f9877b16cd42..96cd04fa612f 100755 --- a/.buildkite/scripts/common/setup_bazel.sh +++ b/.buildkite/scripts/common/setup_bazel.sh @@ -15,8 +15,8 @@ if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then echo "[bazel] enabling caching" cat <> $KIBANA_DIR/.bazelrc build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://cloud.buildbuddy.io - build --remote_cache=grpcs://cloud.buildbuddy.io + build --bes_backend=grpcs://remote.buildbuddy.io + build --remote_cache=grpcs://remote.buildbuddy.io build --remote_timeout=3600 build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY EOF diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 503e9f119a49..93108f2d85c9 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -55910,6 +55910,7 @@ __webpack_require__.r(__webpack_exports__); /* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__); /* harmony import */ var _child_process__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(342); /* harmony import */ var _log__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(341); +/* harmony import */ var _fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(352); /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -55923,6 +55924,7 @@ __webpack_require__.r(__webpack_exports__); + async function isVaultAvailable() { try { await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['--version'], { @@ -55947,6 +55949,23 @@ async function isElasticCommitter() { } } +async function migrateToNewServersIfNeeded(settingsPath) { + if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(settingsPath))) { + return false; + } + + const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["readFile"])(settingsPath, 'utf8'); + const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io'); + + if (newReadSettingsFile === readSettingsFile) { + return false; + } + + Object(_fs__WEBPACK_IMPORTED_MODULE_5__["writeFile"])(settingsPath, newReadSettingsFile); + _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] upgrade remote cache settings to use new server address`); + return true; +} + async function setupRemoteCache(repoRootPath) { // The remote cache is only for Elastic employees working locally (CI cache settings are handled elsewhere) if (process.env.CI || !(await isElasticCommitter())) { @@ -55954,7 +55973,13 @@ async function setupRemoteCache(repoRootPath) { } _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] setting up remote cache settings if necessary`); - const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); + const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache + // + // NOTE: this can be removed in the future once everyone is migrated into the new servers + + if (await migrateToNewServersIfNeeded(settingsPath)) { + return; + } if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) { _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already exist, skipping`); @@ -55990,8 +56015,8 @@ async function setupRemoteCache(repoRootPath) { # V1 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://cloud.buildbuddy.io - build --remote_cache=grpcs://cloud.buildbuddy.io + build --bes_backend=grpcs://remote.buildbuddy.io + build --remote_cache=grpcs://remote.buildbuddy.io build --remote_timeout=3600 build --remote_header=${apiKey} `; diff --git a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts index c1379f8da3bc..0c5213e2dacc 100644 --- a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts +++ b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts @@ -10,6 +10,7 @@ import { existsSync, writeFileSync } from 'fs'; import { resolve } from 'path'; import { spawn } from '../child_process'; import { log } from '../log'; +import { isFile, readFile, writeFile } from '../fs'; async function isVaultAvailable() { try { @@ -33,6 +34,26 @@ async function isElasticCommitter() { } } +async function migrateToNewServersIfNeeded(settingsPath: string) { + if (!(await isFile(settingsPath))) { + return false; + } + + const readSettingsFile = await readFile(settingsPath, 'utf8'); + const newReadSettingsFile = readSettingsFile.replace( + /cloud\.buildbuddy\.io/g, + 'remote.buildbuddy.io' + ); + + if (newReadSettingsFile === readSettingsFile) { + return false; + } + + writeFile(settingsPath, newReadSettingsFile); + log.info(`[bazel_tools] upgrade remote cache settings to use new server address`); + return true; +} + export async function setupRemoteCache(repoRootPath: string) { // The remote cache is only for Elastic employees working locally (CI cache settings are handled elsewhere) if (process.env.CI || !(await isElasticCommitter())) { @@ -43,6 +64,13 @@ export async function setupRemoteCache(repoRootPath: string) { const settingsPath = resolve(repoRootPath, '.bazelrc.cache'); + // Checks if we should upgrade the servers used on .bazelrc.cache + // + // NOTE: this can be removed in the future once everyone is migrated into the new servers + if (await migrateToNewServersIfNeeded(settingsPath)) { + return; + } + if (existsSync(settingsPath)) { log.debug(`[bazel_tools] remote cache settings already exist, skipping`); return; @@ -82,8 +110,8 @@ export async function setupRemoteCache(repoRootPath: string) { # V1 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://cloud.buildbuddy.io - build --remote_cache=grpcs://cloud.buildbuddy.io + build --bes_backend=grpcs://remote.buildbuddy.io + build --remote_cache=grpcs://remote.buildbuddy.io build --remote_timeout=3600 build --remote_header=${apiKey} `;