mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
# Backport This will backport the following commits from `main` to `7.17`: - [[Ops] Refactor env loading & fix agent-targeting rewrite (#178320)](https://github.com/elastic/kibana/pull/178320) <!--- Backport version: 8.9.8 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Alex Szabo","email":"alex.szabo@elastic.co"},"sourceCommit":{"committedDate":"2024-03-12T15:31:26Z","message":"[Ops] Refactor env loading & fix agent-targeting rewrite (#178320)\n\n## Summary\r\nThis PR refactors a bit of the pre-command env setup, separating parts,\r\nso they can be individually skipped. Then it removes the setup-avoidance\r\nbased on agent types, as this won't be useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the agent-targeting rewrite used for the\r\nmigration, where the `provider: 'gcp'` was missing, and adds an optional\r\ntargeting for the script.\r\n\r\n- add gcp as provider to all rewritten agent targeting rules\r\n- add option to target specific pipelines\r\n- refactor env-var loading to a separated file\r\n- refactor node installs so it can be switched by a flag\r\n- skip node installing in (some) jobs that don't require it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","backport missing","v8.14.0","v8.13.1"],"number":178320,"url":"https://github.com/elastic/kibana/pull/178320","mergeCommit":{"message":"[Ops] Refactor env loading & fix agent-targeting rewrite (#178320)\n\n## Summary\r\nThis PR refactors a bit of the pre-command env setup, separating parts,\r\nso they can be individually skipped. Then it removes the setup-avoidance\r\nbased on agent types, as this won't be useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the agent-targeting rewrite used for the\r\nmigration, where the `provider: 'gcp'` was missing, and adds an optional\r\ntargeting for the script.\r\n\r\n- add gcp as provider to all rewritten agent targeting rules\r\n- add option to target specific pipelines\r\n- refactor env-var loading to a separated file\r\n- refactor node installs so it can be switched by a flag\r\n- skip node installing in (some) jobs that don't require it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.14.0","labelRegex":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/178320","number":178320,"mergeCommit":{"message":"[Ops] Refactor env loading & fix agent-targeting rewrite (#178320)\n\n## Summary\r\nThis PR refactors a bit of the pre-command env setup, separating parts,\r\nso they can be individually skipped. Then it removes the setup-avoidance\r\nbased on agent types, as this won't be useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the agent-targeting rewrite used for the\r\nmigration, where the `provider: 'gcp'` was missing, and adds an optional\r\ntargeting for the script.\r\n\r\n- add gcp as provider to all rewritten agent targeting rules\r\n- add option to target specific pipelines\r\n- refactor env-var loading to a separated file\r\n- refactor node installs so it can be switched by a flag\r\n- skip node installing in (some) jobs that don't require it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d"}},{"branch":"8.13","label":"v8.13.1","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/178533","number":178533,"state":"OPEN"}]}] BACKPORT-->
This commit is contained in:
Alex Szabo
GitHub
parent
5636a8e103
commit
c9d7ee0739
9 changed files with 355 additions and 112 deletions
|
|
@ -1,3 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
.buildkite/scripts/lifecycle/post_command.sh
|
||||
source .buildkite/scripts/lifecycle/post_command.sh
|
||||
|
|
|
|||
88
.buildkite/scripts/common/activate_service_account.sh
Normal file
88
.buildkite/scripts/common/activate_service_account.sh
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
|
||||
|
||||
CALL_ARGUMENT="${1:-}"
|
||||
GCLOUD_EMAIL_POSTFIX="elastic-kibana-ci.iam.gserviceaccount.com"
|
||||
GCLOUD_SA_PROXY_EMAIL="kibana-ci-sa-proxy@$GCLOUD_EMAIL_POSTFIX"
|
||||
|
||||
if [[ -z "$CALL_ARGUMENT" ]]; then
|
||||
echo "Usage: $0 <bucket_name|email>"
|
||||
exit 1
|
||||
elif [[ "$CALL_ARGUMENT" == "--unset-impersonation" ]]; then
|
||||
echo "Unsetting impersonation"
|
||||
gcloud config unset auth/impersonate_service_account
|
||||
exit 0
|
||||
elif [[ "$CALL_ARGUMENT" == "--logout-gcloud" ]]; then
|
||||
echo "Logging out of gcloud"
|
||||
if [[ -x "$(command -v gcloud)" ]] && [[ "$(gcloud auth list 2>/dev/null | grep $GCLOUD_SA_PROXY_EMAIL)" != "" ]]; then
|
||||
gcloud auth revoke $GCLOUD_SA_PROXY_EMAIL --no-user-output-enabled
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CURRENT_GCLOUD_USER=$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")
|
||||
|
||||
# Verify that the service account proxy is activated
|
||||
if [[ "$CURRENT_GCLOUD_USER" != "$GCLOUD_SA_PROXY_EMAIL" ]]; then
|
||||
if [[ -x "$(command -v gcloud)" ]]; then
|
||||
if [[ -z "${KIBANA_SERVICE_ACCOUNT_PROXY_KEY:-}" ]]; then
|
||||
echo "KIBANA_SERVICE_ACCOUNT_PROXY_KEY is not set, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
AUTH_RESULT=$(gcloud auth activate-service-account --key-file="$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" || "FAILURE")
|
||||
if [[ "$AUTH_RESULT" == "FAILURE" ]]; then
|
||||
echo "Failed to activate service account $GCLOUD_SA_PROXY_EMAIL."
|
||||
exit 1
|
||||
else
|
||||
echo "Activated service account $GCLOUD_SA_PROXY_EMAIL"
|
||||
fi
|
||||
else
|
||||
echo "gcloud is not installed, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if the arg is a service account e-mail or a bucket name
|
||||
EMAIL=""
|
||||
if [[ "$CALL_ARGUMENT" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then
|
||||
EMAIL="$CALL_ARGUMENT"
|
||||
elif [[ "$CALL_ARGUMENT" =~ ^gs://* ]]; then
|
||||
BUCKET_NAME="${CALL_ARGUMENT:5}"
|
||||
else
|
||||
BUCKET_NAME="$CALL_ARGUMENT"
|
||||
fi
|
||||
|
||||
if [[ -z "$EMAIL" ]]; then
|
||||
case "$BUCKET_NAME" in
|
||||
"elastic-kibana-coverage-live")
|
||||
EMAIL="kibana-ci-access-coverage@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
"kibana-ci-es-snapshots-daily")
|
||||
EMAIL="kibana-ci-access-es-daily@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
"kibana-ci-es-snapshots-permanent")
|
||||
EMAIL="kibana-ci-access-es-permanent@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
"kibana-so-types-snapshots")
|
||||
EMAIL="kibana-ci-access-so-snapshots@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
"kibana-performance")
|
||||
EMAIL="kibana-ci-access-perf-stats@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
"ci-artifacts.kibana.dev")
|
||||
EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
*)
|
||||
EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Activate the service account
|
||||
echo "Impersonating $EMAIL"
|
||||
gcloud config set auth/impersonate_service_account "$EMAIL"
|
||||
echo "Activated service account $EMAIL"
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
echo '--- Setup environment vars'
|
||||
|
||||
export CI=true
|
||||
|
||||
KIBANA_DIR=$(pwd)
|
||||
|
|
|
|||
9
.buildkite/scripts/common/setup_buildkite_deps.sh
Normal file
9
.buildkite/scripts/common/setup_buildkite_deps.sh
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo '--- Install/build buildkite dependencies'
|
||||
|
||||
cd '.buildkite'
|
||||
retry 5 15 npm ci
|
||||
cd -
|
||||
175
.buildkite/scripts/common/setup_job_env.sh
Normal file
175
.buildkite/scripts/common/setup_job_env.sh
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo '--- Job Environment Setup'
|
||||
|
||||
if [[ "$(type -t vault_get)" != "function" ]]; then
|
||||
source .buildkite/scripts/common/vault_fns.sh
|
||||
fi
|
||||
|
||||
# Set up general-purpose tokens and credentials
|
||||
{
|
||||
BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
|
||||
export BUILDKITE_TOKEN
|
||||
|
||||
GITHUB_TOKEN=$(vault_get kibanamachine github_token)
|
||||
export GITHUB_TOKEN
|
||||
|
||||
KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
|
||||
export KIBANA_CI_GITHUB_TOKEN
|
||||
|
||||
KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
|
||||
export KIBANA_DOCKER_USERNAME
|
||||
|
||||
KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
|
||||
export KIBANA_DOCKER_PASSWORD
|
||||
}
|
||||
|
||||
# Set up a custom ES Snapshot Manifest if one has been specified for this build
|
||||
{
|
||||
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')}
|
||||
export ES_SNAPSHOT_MANIFEST
|
||||
|
||||
if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
|
||||
cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest
|
||||
This build is running using a custom Elasticsearch snapshot.
|
||||
|
||||
ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST
|
||||
|
||||
To use this locally, simply prefix your commands with:
|
||||
|
||||
\`\`\`
|
||||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST"
|
||||
\`\`\`
|
||||
|
||||
e.g.
|
||||
|
||||
\`\`\`
|
||||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js
|
||||
\`\`\`
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
|
||||
{
|
||||
if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
|
||||
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
|
||||
export ES_SNAPSHOT_MANIFEST
|
||||
echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup CI Stats
|
||||
{
|
||||
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
|
||||
export CI_STATS_BUILD_ID
|
||||
|
||||
CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
|
||||
export CI_STATS_TOKEN
|
||||
|
||||
CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
|
||||
export CI_STATS_HOST
|
||||
|
||||
if [[ "$CI_STATS_BUILD_ID" ]]; then
|
||||
echo "CI Stats Build ID: $CI_STATS_BUILD_ID"
|
||||
|
||||
KIBANA_CI_STATS_CONFIG=$(jq -n \
|
||||
--arg buildId "$CI_STATS_BUILD_ID" \
|
||||
--arg apiUrl "https://$CI_STATS_HOST" \
|
||||
--arg apiToken "$CI_STATS_TOKEN" \
|
||||
'{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \
|
||||
)
|
||||
export KIBANA_CI_STATS_CONFIG
|
||||
fi
|
||||
}
|
||||
|
||||
# Set up misc keys
|
||||
{
|
||||
KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
|
||||
export KIBANA_CI_REPORTER_KEY
|
||||
|
||||
EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
|
||||
export EC_API_KEY
|
||||
|
||||
PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
|
||||
export PROJECT_API_KEY
|
||||
|
||||
PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
|
||||
export PROJECT_API_DOMAIN
|
||||
|
||||
SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
|
||||
export SYNTHETICS_SERVICE_USERNAME
|
||||
|
||||
SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
|
||||
export SYNTHETICS_SERVICE_PASSWORD
|
||||
|
||||
SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
|
||||
export SYNTHETICS_SERVICE_MANIFEST
|
||||
|
||||
SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
|
||||
export SYNTHETICS_REMOTE_KIBANA_USERNAME
|
||||
|
||||
SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
|
||||
export SYNTHETICS_REMOTE_KIBANA_PASSWORD
|
||||
|
||||
SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
|
||||
export SYNTHETICS_REMOTE_KIBANA_URL
|
||||
|
||||
DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
|
||||
export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
|
||||
|
||||
SONAR_LOGIN=$(vault_get sonarqube token)
|
||||
export SONAR_LOGIN
|
||||
}
|
||||
|
||||
# Set up GCS Service Account for CDN
|
||||
{
|
||||
GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)"
|
||||
export GCS_SA_CDN_KEY
|
||||
|
||||
GCS_SA_CDN_EMAIL="$(vault_get gcs-sa-cdn-prod email)"
|
||||
export GCS_SA_CDN_EMAIL
|
||||
|
||||
GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)"
|
||||
export GCS_SA_CDN_BUCKET
|
||||
|
||||
GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)"
|
||||
export GCS_SA_CDN_URL
|
||||
}
|
||||
|
||||
# Setup Failed Test Reporter Elasticsearch credentials
|
||||
{
|
||||
TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
|
||||
export TEST_FAILURES_ES_CLOUD_ID
|
||||
|
||||
TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
|
||||
export TEST_FAILURES_ES_USERNAME
|
||||
|
||||
TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
|
||||
export TEST_FAILURES_ES_PASSWORD
|
||||
}
|
||||
|
||||
# Setup Bazel Remote/Local Cache Credentials
|
||||
{
|
||||
BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
|
||||
export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
|
||||
vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
|
||||
|
||||
BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json"
|
||||
export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
|
||||
vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE"
|
||||
}
|
||||
|
||||
# Setup GCS Service Account Proxy for CI
|
||||
{
|
||||
KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json"
|
||||
export KIBANA_SERVICE_ACCOUNT_PROXY_KEY
|
||||
vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY"
|
||||
}
|
||||
|
||||
PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
|
||||
if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
|
||||
source "$PIPELINE_PRE_COMMAND"
|
||||
fi
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
|
||||
|
||||
checks-reporter-with-killswitch() {
|
||||
if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
|
||||
yarn run github-checks-reporter "$@"
|
||||
|
|
|
|||
67
.buildkite/scripts/common/vault_fns.sh
Normal file
67
.buildkite/scripts/common/vault_fns.sh
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
#!/bin/bash
|
||||
|
||||
# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
|
||||
if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
|
||||
VAULT_PATH_PREFIX="secret/kibana-issues/dev"
|
||||
VAULT_KV_PREFIX="secret/kibana-issues/dev"
|
||||
IS_LEGACY_VAULT_ADDR=true
|
||||
else
|
||||
VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
|
||||
VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
|
||||
IS_LEGACY_VAULT_ADDR=false
|
||||
fi
|
||||
export IS_LEGACY_VAULT_ADDR
|
||||
|
||||
retry() {
|
||||
local retries=$1; shift
|
||||
local delay=$1; shift
|
||||
local attempts=1
|
||||
|
||||
until "$@"; do
|
||||
retry_exit_status=$?
|
||||
echo "Exited with $retry_exit_status" >&2
|
||||
if (( retries == "0" )); then
|
||||
return $retry_exit_status
|
||||
elif (( attempts == retries )); then
|
||||
echo "Failed $attempts retries" >&2
|
||||
return $retry_exit_status
|
||||
else
|
||||
echo "Retrying $((retries - attempts)) more times..." >&2
|
||||
attempts=$((attempts + 1))
|
||||
sleep "$delay"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
vault_get() {
|
||||
key_path=${1:-}
|
||||
field=${2:-}
|
||||
|
||||
fullPath="$VAULT_PATH_PREFIX/$key_path"
|
||||
|
||||
if [[ -z "$field" || "$field" =~ ^-.* ]]; then
|
||||
retry 5 5 vault read "$fullPath" "${@:2}"
|
||||
else
|
||||
retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
|
||||
fi
|
||||
}
|
||||
|
||||
vault_set() {
|
||||
key_path=$1
|
||||
shift
|
||||
fields=("$@")
|
||||
|
||||
|
||||
fullPath="$VAULT_PATH_PREFIX/$key_path"
|
||||
|
||||
# shellcheck disable=SC2068
|
||||
retry 5 5 vault write "$fullPath" ${fields[@]}
|
||||
}
|
||||
|
||||
vault_kv_set() {
|
||||
kv_path=$1
|
||||
shift
|
||||
fields=("$@")
|
||||
|
||||
vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
|
||||
}
|
||||
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
echo '--- Log out of gcloud'
|
||||
./.buildkite/scripts/common/activate_service_account.sh --unset-impersonation || echo "Failed to unset impersonation"
|
||||
./.buildkite/scripts/common/activate_service_account.sh --logout-gcloud || echo "Failed to log out of gcloud"
|
||||
|
||||
echo '--- Agent Debug Info'
|
||||
node .buildkite/scripts/lifecycle/print_agent_links.js || true
|
||||
|
||||
|
|
|
|||
|
|
@ -3,24 +3,16 @@
|
|||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/common/env.sh
|
||||
source .buildkite/scripts/common/setup_job_env.sh
|
||||
|
||||
# By default, all steps should set up these things to get a full environment before running
|
||||
# It can be skipped for pipeline upload steps though, to make job start time a little faster
|
||||
if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
|
||||
if [[ -d .buildkite/scripts && "${BUILDKITE_COMMAND:-}" != "buildkite-agent pipeline upload"* ]]; then
|
||||
source .buildkite/scripts/common/env.sh
|
||||
source .buildkite/scripts/common/setup_node.sh
|
||||
fi
|
||||
if [[ "${SKIP_CI_SETUP:-}" == "true" || "${SKIP_NODE_SETUP:-}" =~ ^(1|true)$ ]]; then
|
||||
echo "Skipping node setup (SKIP_NODE_SETUP=$SKIP_NODE_SETUP)"
|
||||
else
|
||||
source .buildkite/scripts/common/setup_node.sh
|
||||
source .buildkite/scripts/common/setup_buildkite_deps.sh
|
||||
fi
|
||||
|
||||
BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
|
||||
export BUILDKITE_TOKEN
|
||||
|
||||
echo '--- Install buildkite dependencies'
|
||||
cd '.buildkite'
|
||||
retry 5 15 npm ci
|
||||
cd ..
|
||||
|
||||
echo '--- Agent Debug/SSH Info'
|
||||
node .buildkite/scripts/lifecycle/print_agent_links.js || true
|
||||
|
||||
|
|
@ -30,99 +22,3 @@ if [[ "$(curl -is metadata.google.internal || true)" ]]; then
|
|||
echo "gcloud compute ssh --tunnel-through-iap --project elastic-kibana-ci --zone \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone)\" \"$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/name)\""
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
echo '--- Job Environment Setup'
|
||||
|
||||
# Set up a custom ES Snapshot Manifest if one has been specified for this build
|
||||
{
|
||||
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')}
|
||||
export ES_SNAPSHOT_MANIFEST
|
||||
|
||||
if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
|
||||
cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest
|
||||
This build is running using a custom Elasticsearch snapshot.
|
||||
|
||||
ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST
|
||||
|
||||
To use this locally, simply prefix your commands with:
|
||||
|
||||
\`\`\`
|
||||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST"
|
||||
\`\`\`
|
||||
|
||||
e.g.
|
||||
|
||||
\`\`\`
|
||||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js
|
||||
\`\`\`
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
|
||||
{
|
||||
if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
|
||||
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
|
||||
export ES_SNAPSHOT_MANIFEST
|
||||
echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup CI Stats
|
||||
{
|
||||
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
|
||||
export CI_STATS_BUILD_ID
|
||||
|
||||
CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
|
||||
export CI_STATS_TOKEN
|
||||
|
||||
CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
|
||||
export CI_STATS_HOST
|
||||
|
||||
if [[ "$CI_STATS_BUILD_ID" ]]; then
|
||||
echo "CI Stats Build ID: $CI_STATS_BUILD_ID"
|
||||
|
||||
KIBANA_CI_STATS_CONFIG=$(jq -n \
|
||||
--arg buildId "$CI_STATS_BUILD_ID" \
|
||||
--arg apiUrl "https://$CI_STATS_HOST" \
|
||||
--arg apiToken "$CI_STATS_TOKEN" \
|
||||
'{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \
|
||||
)
|
||||
export KIBANA_CI_STATS_CONFIG
|
||||
fi
|
||||
}
|
||||
|
||||
GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
|
||||
export GITHUB_TOKEN
|
||||
|
||||
KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
|
||||
export KIBANA_DOCKER_USERNAME
|
||||
|
||||
KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
|
||||
export KIBANA_DOCKER_PASSWORD
|
||||
|
||||
KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
|
||||
export KIBANA_CI_REPORTER_KEY
|
||||
|
||||
|
||||
BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
|
||||
export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
|
||||
retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
|
||||
|
||||
# Setup Failed Test Reporter Elasticsearch credentials
|
||||
{
|
||||
TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
|
||||
export TEST_FAILURES_ES_CLOUD_ID
|
||||
|
||||
TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
|
||||
export TEST_FAILURES_ES_USERNAME
|
||||
|
||||
TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
|
||||
export TEST_FAILURES_ES_PASSWORD
|
||||
}
|
||||
|
||||
PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
|
||||
if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
|
||||
source "$PIPELINE_PRE_COMMAND"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue