[Ops] Use vault kv store when on new Buildkite infra (#174915)

## Summary
On the new Buildkite infra, our agents won't have write access to the
paths we used to write the deployment information upon deployment.

We're allowed to use KV writes if we enable it
(https://docs.elastic.dev/ci/using-secrets#using-shared-secrets,
enabling PRs below).

I've built this in a way that we can enable the feature before the final
rollout, and we can clear up the branches once done with the rollout.

Cloud deployment works on the old infra as well as the new, tested on
this PR and #171317

Enabled by: https://github.com/elastic/ci/pull/2594 &
https://github.com/elastic/ci/pull/2553
Part of: https://github.com/elastic/kibana-operations/issues/15
Related: https://elasticco.atlassian.net/browse/ENGPRD-414

.buildkite/scripts/common/util.sh

@@ -171,15 +171,23 @@ download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
 
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  VAULT_KV_PREFIX="secret/kibana-issues/dev"
+  IS_LEGACY_VAULT_ADDR=true
+else
+  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
+  IS_LEGACY_VAULT_ADDR=false
+fi
+export IS_LEGACY_VAULT_ADDR
 
 vault_get() {
   key_path=$1
   field=$2
 
-  fullPath="secret/ci/elastic-kibana/$key_path"
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$key_path"
-  fi
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
 
   if [[ -z "${2:-}" || "${2:-}" =~ ^-.* ]]; then
     retry 5 5 vault read "$fullPath" "${@:2}"
@@ -193,11 +201,17 @@ vault_set() {
   shift
   fields=("$@")
 
-  fullPath="secret/ci/elastic-kibana/$key_path"
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$key_path"
-  fi
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
 
   # shellcheck disable=SC2068
   retry 5 5 vault write "$fullPath" ${fields[@]}
 }
+
+vault_kv_set() {
+  kv_path=$1
+  shift
+  fields=("$@")
+
+  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
+}

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -86,7 +86,13 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
-  vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+
+  # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  else
+    vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  fi
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -121,10 +127,11 @@ fi
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
 
-if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+  VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 fi
 
 cat << EOF | buildkite-agent annotate --style "info" --context cloud
@@ -134,7 +141,7 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-  Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
+  Credentials: \`$VAULT_READ_COMMAND\`
 
   Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -77,7 +77,14 @@ deploy() {
     VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
     VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
     retry 5 30 vault login -no-print "$VAULT_TOKEN"
-    vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+
+    # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+    if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+      vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    else
+      vault_kv_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    fi
+
   else
     echo "Updating project..."
     curl -s \
@@ -91,10 +98,11 @@ deploy() {
   PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
   PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)
 
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME"
   else
-    VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+    VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$PROJECT_NAME"
   fi
 
   cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
@@ -104,7 +112,7 @@ Kibana: $PROJECT_KIBANA_LOGIN_URL
 
 Elasticsearch: $PROJECT_ELASTICSEARCH_URL
 
-Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME\`
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_IMAGE\`
 EOF