[Ops] Switch to using the migrated secret paths for vault secrets (#172175)

## Summary
We're moving to a different vault address/instance when we're on the
elastic-wide buildkite infra. While the migration is in progress, we can
bridge between using this solution.

 Tested the status quo by running the PR pipeline (tests all the loads
from `pre-command`) and by using `ci:cloud-deploy` (tests vault
writing).

🟠 Tested the new vault provider on this PR:
https://github.com/elastic/kibana/pull/171317
The secrets can be accessed, *but they can't be written* (neither by me
nor) by the PR pipeline. Change requested here:
https://elasticco.atlassian.net/browse/ENGPRD-414

However, this PR can be merged without figuring out write access to
secrets, this will work as long as we're on the `kibana-buildkite`
infra.

---
Closes: https://github.com/elastic/kibana-operations/issues/28

Based on: https://github.com/elastic/kibana/pull/157220

---------

Co-authored-by: Jon <jon@budzenski.me>
This commit is contained in:
Alex Szabo 2023-12-06 10:40:44 +01:00 committed by GitHub
parent b45c1890e2
commit 1022ccdf78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 93 additions and 44 deletions

View file

@ -170,3 +170,34 @@ npm_install_global() {
download_artifact() {
retry 3 1 timeout 3m buildkite-agent artifact download "$@"
}
vault_get() {
path=$1
field=$2
fullPath="secret/ci/elastic-kibana/$path"
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
fullPath="secret/kibana-issues/dev/$path"
fi
if [[ -z "${2:-}" ]]; then
retry 5 5 vault read "$fullPath"
else
retry 5 5 vault read -field="$field" "$fullPath"
fi
}
vault_set() {
path=$1
shift
fields=("$@")
fullPath="secret/ci/elastic-kibana/$path"
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
fullPath="secret/kibana-issues/dev/$path"
fi
# shellcheck disable=SC2068
retry 5 5 vault write "$fullPath" ${fields[@]}
}

View file

@ -8,7 +8,7 @@ echo '--- Setup environment vars'
source .buildkite/scripts/common/env.sh
source .buildkite/scripts/common/setup_node.sh
BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
export BUILDKITE_TOKEN
echo '--- Install/build buildkite dependencies'
@ -77,11 +77,11 @@ EOF
{
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
export CI_STATS_BUILD_ID
CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
export CI_STATS_TOKEN
CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
export CI_STATS_HOST
if [[ "$CI_STATS_BUILD_ID" ]]; then
@ -97,66 +97,66 @@ EOF
fi
}
GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
GITHUB_TOKEN=$(vault_get kibanamachine github_token)
export GITHUB_TOKEN
KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
export KIBANA_CI_GITHUB_TOKEN
KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
export KIBANA_CI_REPORTER_KEY
KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
export KIBANA_DOCKER_USERNAME
KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
export KIBANA_DOCKER_PASSWORD
EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
export EC_API_KEY
PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
export PROJECT_API_KEY
PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
export PROJECT_API_DOMAIN
SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
export SYNTHETICS_SERVICE_USERNAME
SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
export SYNTHETICS_SERVICE_PASSWORD
SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
export SYNTHETICS_SERVICE_MANIFEST
SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
export SYNTHETICS_REMOTE_KIBANA_USERNAME
SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
export SYNTHETICS_REMOTE_KIBANA_PASSWORD
SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
export SYNTHETICS_REMOTE_KIBANA_URL
DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
# Setup Failed Test Reporter Elasticsearch credentials
{
TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
export TEST_FAILURES_ES_CLOUD_ID
TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
export TEST_FAILURES_ES_USERNAME
TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
export TEST_FAILURES_ES_PASSWORD
}
BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then

View file

@ -14,7 +14,7 @@ echo "--- Serverless Security Second Quality Gate"
cd x-pack/test/security_solution_api_integration
set +e
QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
# Generate a random 5-digit number
random_number=$((10000 + $RANDOM % 90000))
@ -47,7 +47,7 @@ while : ; do
echo "Sleeping for 40s to wait for ES status to be green..."
sleep 40
else
echo "Elasticsearch has status green."
echo "Elasticsearch has status green."
break
fi
done
@ -59,17 +59,17 @@ while : ; do
echo "Sleeping for 15s to wait for Kibana to be available..."
sleep 15
else
echo "Kibana is available."
echo "Kibana is available."
break
fi
done
# Removing the https:// part of the url provided in order to use it in the command below.
FORMATTED_ES_URL="${ES_URL/https:\/\//}"
FORMATTED_ES_URL="${ES_URL/https:\/\//}"
FORMATTED_KB_URL="${KB_URL/https:\/\//}"
# Find a way to remove this in the future
# This is used in order to wait for the environment to be ready.
# This is used in order to wait for the environment to be ready.
sleep 150
TEST_CLOUD=1 TEST_ES_URL="https://elastic:$PASSWORD@$FORMATTED_ES_URL:443" TEST_KIBANA_URL="https://elastic:$PASSWORD@$FORMATTED_KB_URL:443" yarn run $1
@ -79,4 +79,4 @@ echo "Exit code with status: $cmd_status"
curl --location --request DELETE "https://global.qa.cld.elstc.co/api/v1/serverless/projects/security/$ID" \
--header "Authorization: ApiKey $QA_API_KEY"
exit $cmd_status
exit $cmd_status

View file

@ -19,6 +19,6 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true
cd x-pack/test/security_solution_cypress
set +e
QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status
CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

View file

@ -86,7 +86,7 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
retry 5 30 vault login -no-print "$VAULT_TOKEN"
retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
echo "Enabling Stack Monitoring..."
jq '
@ -121,6 +121,12 @@ fi
CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
VAULT_PATH_PREFIX="secret/kibana-issues/dev"
else
VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
fi
cat << EOF | buildkite-agent annotate --style "info" --context cloud
### Cloud Deployment
@ -128,7 +134,7 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
Kibana image: \`$KIBANA_CLOUD_IMAGE\`

View file

@ -10,6 +10,9 @@ import { execSync } from 'child_process';
const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
const { deployments } = JSON.parse(deploymentsListJson);
const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
? 'secret/kibana-issues/dev'
: 'secret/ci/elastic-kibana';
const prNumber = parseInt(
process.env.KIBANA_PULL_REQUEST ||
@ -27,7 +30,7 @@ if (!prNumber || !deployment) {
console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
try {
execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
execSync(`vault delete secret/kibana-issues/dev/cloud-deploy/${deployment.name}`, {
execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
stdio: 'inherit',
});
} catch (ex) {

View file

@ -10,6 +10,9 @@ import { execSync } from 'child_process';
const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
const { deployments } = JSON.parse(deploymentsListJson);
const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
? 'secret/kibana-issues/dev'
: 'secret/ci/elastic-kibana';
const prDeployments = deployments.filter((deployment: any) =>
deployment.name.startsWith('kibana-pr-')
@ -67,7 +70,7 @@ for (const deployment of deploymentsToPurge) {
console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
try {
execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
execSync(`vault delete secret/kibana-issues/dev/cloud-deploy/${deployment.name}`, {
execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
stdio: 'inherit',
});
} catch (ex) {

View file

@ -4,7 +4,7 @@ set -euo pipefail
source .buildkite/scripts/common/util.sh
APM_CYPRESS_RECORD_KEY="$(retry 5 5 vault read -field=CYPRESS_RECORD_KEY secret/kibana-issues/dev/apm-cypress-dashboard-record-key)"
APM_CYPRESS_RECORD_KEY="$(vault_get apm-cypress-dashboard-record-key CYPRESS_RECORD_KEY)"
.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh

View file

@ -5,8 +5,8 @@ set -euo pipefail
source .buildkite/scripts/common/util.sh
# TODO: Add new user and change lines accordingly
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/ci_stats_performance_metrics)"
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/ci_stats_performance_metrics)"
USER_FROM_VAULT="$(vault_get ci_stats_performance_metrics username)"
PASS_FROM_VAULT="$(vault_get ci_stats_performance_metrics password)"
APM_SERVER_URL="https://kibana-ops-e2e-perf.kb.us-central1.gcp.cloud.es.io:9243/internal/apm"
BUILD_ID=${BUILDKITE_BUILD_ID}

View file

@ -4,8 +4,8 @@ set -euo pipefail
source .buildkite/scripts/common/util.sh
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
USER_FROM_VAULT="$(vault_get apm_parser_performance username)"
PASS_FROM_VAULT="$(vault_get apm_parser_performance password)"
ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
BUILD_ID="${BUILDKITE_BUILD_ID}"
GCS_BUCKET="gs://kibana-performance/scalability-tests"

View file

@ -72,7 +72,7 @@ deploy() {
VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
retry 5 30 vault login -no-print "$VAULT_TOKEN"
retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
else
echo "Updating project..."
curl -s \
@ -86,6 +86,12 @@ deploy() {
PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
VAULT_PATH_PREFIX="secret/kibana-issues/dev"
else
VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
fi
cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
### $PROJECT_TYPE_LABEL Deployment
@ -93,7 +99,7 @@ Kibana: $PROJECT_KIBANA_LOGIN_URL
Elasticsearch: $PROJECT_ELASTICSEARCH_URL
Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME\`
Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME\`
Kibana image: \`$KIBANA_IMAGE\`
EOF

View file

@ -21,7 +21,7 @@ kibanaPipeline(timeoutMinutes: 210) {
runErrorReporter: false,
bootstrapped: false
) {
withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') {
withGcpServiceAccount.fromVaultSecret('secret/ci/elastic-kibana/ci-artifacts-key', 'value') {
withEnv([
'DISABLE_BOOTSTRAP_VALIDATION=true',
]) {