[CI] Use legacy vault for deployment credentials (#184944)

## Summary
With the migration to the shared buildkite infra, we've also switched to
using the ci-prod vault (https://vault-ci-prod.elastic.dev) for all
CI-related secrets. We found it reasonable then, to also switch the
storage of the credentials for the deployments there. It's since been
proven unnecessary, even confusing for developers, as they might not be
adequately set up for accessing the two vaults. We've also learned, that
both of these vault instances are here to stay, so there's no push to
migrate everything to the ci-prod instance.

So, this PR switches back to using the legacy vault in all cases for
storing deployment keys, as it fits better with the developers' daily
secret handling duties.

Also, adds a cleanup part to the purge routine.

- [x] extract vault read / write to a parametric shell script, because
the typescript invocations to vault won't have an easy access to the
`set_in_legacy_vault`

.buildkite/scripts/common/deployment_credentials.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/vault_fns.sh
+
+## Usage
+# ./deployment_credentials.sh set <key-path> <key=value> <key=value> ...
+# ./deployment_credentials.sh unset <key-path>
+# ./deployment_credentials.sh print <key-path>
+
+if [[ "${1:-}" == "set" ]]; then
+  set_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "unset" ]]; then
+  unset_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "print" ]]; then
+  print_legacy_vault_read "${2}"
+else
+  echo "Unknown command: $1"
+  exit 1
+fi

.buildkite/scripts/common/vault_fns.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
-if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+# TODO: rewrite after https://github.com/elastic/kibana-operations/issues/15 is done
+export LEGACY_VAULT_ADDR="https://secrets.elastic.co:8200"
+if [[ "${VAULT_ADDR:-}" == "$LEGACY_VAULT_ADDR" ]]; then
   VAULT_PATH_PREFIX="secret/kibana-issues/dev"
   VAULT_KV_PREFIX="secret/kibana-issues/dev"
   IS_LEGACY_VAULT_ADDR=true
@@ -85,3 +86,56 @@ function get_vault_secret_id() {
 
     echo "$VAULT_SECRET_ID"
 }
+
+function set_in_legacy_vault() {
+  key_path=$1
+  shift
+  fields=("$@")
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  # shellcheck disable=SC2068
+  vault write -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path" ${fields[@]}
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function unset_in_legacy_vault() {
+  key_path=$1
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  vault delete -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path"
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function print_legacy_vault_read() {
+  key_path=$1
+
+  echo "vault read -address=$LEGACY_VAULT_ADDR secret/kibana-issues/dev/cloud-deploy/$key_path"
+}

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -82,7 +82,9 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
 
   echo "Writing to vault..."
 
-  vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  set_in_legacy_vault "$CLOUD_DEPLOYMENT_NAME" \
+    username="$CLOUD_DEPLOYMENT_USERNAME" \
+    password="$CLOUD_DEPLOYMENT_PASSWORD"
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -114,6 +116,7 @@ else
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
+VAULT_READ_COMMAND=$(print_legacy_vault_read "$CLOUD_DEPLOYMENT_NAME")
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
@@ -125,9 +128,7 @@ Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL
 
 Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-Credentials: \`vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
-
-(Stored in the production vault: VAULT_ADDR=https://vault-ci-prod.elastic.dev, more info: https://docs.elastic.dev/ci/using-secrets)
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -7,12 +7,10 @@
  */
 
 import { execSync } from 'child_process';
+import { getKibanaDir } from '#pipeline-utils';
 
 const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
 const { deployments } = JSON.parse(deploymentsListJson);
-const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
-  ? 'secret/kibana-issues/dev'
-  : 'secret/ci/elastic-kibana';
 
 const prDeployments = deployments.filter((deployment: any) =>
   deployment.name.startsWith('kibana-pr-')
@@ -70,7 +68,9 @@ for (const deployment of deploymentsToPurge) {
   console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
   try {
     execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
-    execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${deployment.name}`, {
+      cwd: getKibanaDir(),
       stdio: 'inherit',
     });
   } catch (ex) {

.buildkite/scripts/steps/cloud/purge_projects.ts

@@ -8,6 +8,7 @@
 
 import { execSync } from 'child_process';
 import axios from 'axios';
+import { getKibanaDir } from '#pipeline-utils';
 
 async function getPrProjects() {
   const match = /^(keep.?)?kibana-pr-([0-9]+)-(elasticsearch|security|observability)$/;
@@ -43,12 +44,19 @@ async function getPrProjects() {
 async function deleteProject({
   type,
   id,
+  name,
 }: {
   type: 'elasticsearch' | 'observability' | 'security';
   id: number;
+  name: string;
 }) {
   try {
     await projectRequest.delete(`/api/v1/serverless/projects/${type}/${id}`);
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${name}`, {
+      cwd: getKibanaDir(),
+      stdio: 'inherit',
+    });
   } catch (e) {
     if (e.isAxiosError) {
       const message =
@@ -61,7 +69,7 @@ async function deleteProject({
 
 async function purgeProjects() {
   const prProjects = await getPrProjects();
-  const projectsToPurge = [];
+  const projectsToPurge: typeof prProjects = [];
   for (const project of prProjects) {
     const NOW = new Date().getTime() / 1000;
     const DAY_IN_SECONDS = 60 * 60 * 24;

.buildkite/scripts/steps/serverless/deploy.sh

@@ -88,7 +88,10 @@ deploy() {
 
     echo "Write to vault..."
 
-    vault_kv_set "cloud-deploy/$VAULT_KEY_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    set_in_legacy_vault "$VAULT_KEY_NAME" \
+     username="$PROJECT_USERNAME" \
+     password="$PROJECT_PASSWORD" \
+     id="$PROJECT_ID"
 
   else
     echo "Updating project..."
@@ -109,6 +112,8 @@ deploy() {
   PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
   PROJECT_ELASTICSEARCH_URL=$(jq -r '.endpoints.elasticsearch' $PROJECT_INFO_LOGS)
 
+  VAULT_READ_COMMAND=$(print_legacy_vault_read "$VAULT_KEY_NAME")
+
   cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
 ### $PROJECT_TYPE_LABEL Deployment
 
@@ -116,9 +121,7 @@ Kibana: $PROJECT_KIBANA_LOGIN_URL
 
 Elasticsearch: $PROJECT_ELASTICSEARCH_URL
 
-Credentials: \`vault kv get $VAULT_KV_PREFIX/cloud-deploy/$VAULT_KEY_NAME\`
-
-(Stored in the production vault: VAULT_ADDR=https://vault-ci-prod.elastic.dev, more info: https://docs.elastic.dev/ci/using-secrets)
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_IMAGE\`
 EOF