[Ops] Buildkite job for serverless deployment (#170655)

## Summary
Connected to: https://github.com/elastic/kibana-operations/issues/18
Pre-requisite for:
https://github.com/elastic/kibana-operations/issues/30

You can test the current assistant from the branch:
https://buildkite.com/elastic/kibana-serverless-release-1/builds?branch=buildkite-job-for-deployment
- use `DRY_RUN=1` in the runtime params to not trigger an actual release
:)

This PR creates the contents of a Buildkite job to assist the Kibana
Serverless Release initiation process at the very beginning and lay some
groundwork for further additions to the release management.

At the end of the day, we would like to create a tag deploy@<timestamp>
which will be picked up by another job that listens to these tags:
https://buildkite.com/elastic/kibana-serverless-release. However,
several parts of the preparation for release require manual research,
collecting information about target releases, running scripts, etc.

Any further addition to what would be useful for someone wanting to
start a release could be contained here.

Furthermore, we could also trigger downstream jobs from here. e.g.:
https://buildkite.com/elastic/kibana-serverless-release is currently set
up to listen for a git tag, but we may as well just trigger the job
after we've created a tag.

Check out an example run at:
https://buildkite.com/elastic/kibana-serverless-release-1/builds/72
(visible only if you're a
member of @ elastic/kibana-release-operators) 

Missing features compared to the git action:

- [x] Slack notification about the started deploy
- [x] full "useful links" section

Missing features:
- [x] there's a bit of useful context that should be integrated to the
display of the FTR results (*)
- [x] skip listing and analysis if a commit sha is passed in env


(*) - Currently, we display the next FTR test suite that ran after the
merge of the PR. However, the next FTR that will contain the changes,
and show useful info related to the changeset is ONLY in the FTR that's
ran after the first successful onMerge after the merge commit. Meaning:
if main is failing when the change is merged, an FTR suite won't pick up
the change right after.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Thomas Watson <w@tson.dk>
Co-authored-by: Thomas Watson <watson@elastic.co>

.buildkite/pipeline-utils/buildkite/client.ts

@@ -7,16 +7,21 @@
  */
 
 import axios, { AxiosInstance } from 'axios';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 import { dump } from 'js-yaml';
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';
 import { Job, JobState } from './types/job';
 
+type ExecType =
+  | ((command: string, execOpts: ExecSyncOptions) => Buffer | null)
+  | ((command: string, execOpts: ExecSyncOptions) => string | null);
+
 export interface BuildkiteClientConfig {
   baseUrl?: string;
   token?: string;
+  exec?: ExecType;
 }
 
 export interface BuildkiteGroup {
@@ -24,7 +29,9 @@ export interface BuildkiteGroup {
   steps: BuildkiteStep[];
 }
 
-export interface BuildkiteStep {
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+
+export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
@@ -43,6 +50,50 @@ export interface BuildkiteStep {
   env?: { [key: string]: string };
 }
 
+interface BuildkiteInputTextField {
+  text: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+}
+
+interface BuildkiteInputSelectField {
+  select: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+  multiple?: boolean;
+  options: Array<{
+    label: string;
+    value: string;
+  }>;
+}
+
+export interface BuildkiteInputStep {
+  input: string;
+  prompt?: string;
+  fields: Array<BuildkiteInputTextField | BuildkiteInputSelectField>;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  branches?: string;
+  parallelism?: number;
+  agents?: {
+    queue: string;
+  };
+  timeout_in_minutes?: number;
+  key?: string;
+  depends_on?: string | string[];
+  retry?: {
+    automatic: Array<{
+      exit_status: string;
+      limit: number;
+    }>;
+  };
+  env?: { [key: string]: string };
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;
@@ -61,6 +112,7 @@ export interface BuildkiteTriggerBuildParams {
 
 export class BuildkiteClient {
   http: AxiosInstance;
+  exec: ExecType;
 
   constructor(config: BuildkiteClientConfig = {}) {
     const BUILDKITE_BASE_URL =
@@ -78,6 +130,8 @@ export class BuildkiteClient {
       },
     });
 
+    this.exec = config.exec ?? execSync;
+
     // this.agentHttp = axios.create({
     //   baseURL: BUILDKITE_AGENT_BASE_URL,
     //   headers: {
@@ -97,6 +151,32 @@ export class BuildkiteClient {
     return resp.data as Build;
   };
 
+  getBuildsAfterDate = async (
+    pipelineSlug: string,
+    date: string,
+    numberOfBuilds: number
+  ): Promise<Build[]> => {
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?created_from=${date}&per_page=${numberOfBuilds}`
+    );
+    return response.data as Build[];
+  };
+
+  getBuildForCommit = async (pipelineSlug: string, commit: string): Promise<Build | null> => {
+    if (commit.length !== 40) {
+      throw new Error(`Invalid commit hash: ${commit}, this endpoint works with full SHAs only`);
+    }
+
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?commit=${commit}`
+    );
+    const builds = response.data as Build[];
+    if (builds.length === 0) {
+      return null;
+    }
+    return builds[0];
+  };
+
   getCurrentBuild = (includeRetriedJobs = false) => {
     if (!process.env.BUILDKITE_PIPELINE_SLUG || !process.env.BUILDKITE_BUILD_NUMBER) {
       throw new Error(
@@ -235,31 +315,46 @@ export class BuildkiteClient {
   };
 
   setMetadata = (key: string, value: string) => {
-    execSync(`buildkite-agent meta-data set '${key}'`, {
+    this.exec(`buildkite-agent meta-data set '${key}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
+  getMetadata(key: string, defaultValue: string | null = null): string | null {
+    try {
+      const stdout = this.exec(`buildkite-agent meta-data get '${key}'`, {
+        stdio: ['pipe'],
+      });
+      return stdout?.toString().trim() || defaultValue;
+    } catch (e) {
+      if (e.message.includes('404 Not Found')) {
+        return defaultValue;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
     value: string
   ) => {
-    execSync(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
+    this.exec(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
   uploadArtifacts = (pattern: string) => {
-    execSync(`buildkite-agent artifact upload '${pattern}'`, {
+    this.exec(`buildkite-agent artifact upload '${pattern}'`, {
       stdio: ['ignore', 'inherit', 'inherit'],
     });
   };
 
   uploadSteps = (steps: Array<BuildkiteStep | BuildkiteGroup>) => {
-    execSync(`buildkite-agent pipeline upload`, {
+    this.exec(`buildkite-agent pipeline upload`, {
       input: dump({ steps }),
       stdio: ['pipe', 'inherit', 'inherit'],
     });

.buildkite/pipeline-utils/github/github.ts

@@ -91,3 +91,7 @@ export const doAnyChangesMatch = async (
 
   return anyFilesMatchRequired;
 };
+
+export function getGithubClient() {
+  return github;
+}

.buildkite/pipeline-utils/index.ts

@@ -10,3 +10,4 @@ export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
+export * from './utils';

.buildkite/pipeline-utils/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+
+const getKibanaDir = (() => {
+  let kibanaDir: string | undefined;
+  return () => {
+    if (!kibanaDir) {
+      kibanaDir = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' })
+        .toString()
+        .trim();
+    }
+
+    return kibanaDir;
+  };
+})();
+
+export { getKibanaDir };

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -0,0 +1,33 @@
+## Creates deploy@<timestamp> tag on Kibana
+
+agents:
+ queue: kibana-default
+
+steps:
+  - label: "List potential commits"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state initialize
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commits
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 25
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_selection
+    key: select_commit
+
+  - wait: ~
+
+  - label: "Collect commit info"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commit_info
+      - bash .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_confirmation
+    key: collect_data
+    depends_on: select_commit
+
+  - wait: ~
+
+  - label: ":ship: Create Deploy Tag"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state create_deploy_tag
+      - bash .buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state tag_created
+    env:
+      DRY_RUN: $DRY_RUN

.buildkite/scripts/lifecycle/pre_command.sh

@@ -139,6 +139,9 @@ export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
+DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
+export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# SO migration comparison lives in the Kibana dev app code, needs bootstrapping
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Collecting commit info"
+ts-node .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts
+
+cat << EOF | buildkite-agent pipeline upload
+  steps:
+    - block: "Confirm deployment"
+      prompt: "Are you sure you want to deploy to production? (dry run: ${DRY_RUN:-false})"
+      depends_on: collect_data
+EOF

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { COMMIT_INFO_CTX, exec } from './shared';
+import {
+  toGitCommitExtract,
+  getCurrentQARelease,
+  getSelectedCommitHash,
+  getCommitByHash,
+  makeCommitInfoHtml,
+} from './info_sections/commit_info';
+import {
+  getArtifactBuild,
+  getOnMergePRBuild,
+  getQAFBuildContainingCommit,
+  makeBuildkiteBuildInfoHtml,
+} from './info_sections/build_info';
+import {
+  compareSOSnapshots,
+  makeSOComparisonBlockHtml,
+  makeSOComparisonErrorHtml,
+} from './info_sections/so_snapshot_comparison';
+import { makeUsefulLinksHtml } from './info_sections/useful_links';
+
+async function main() {
+  const previousSha = await getCurrentQARelease();
+  const selectedSha = getSelectedCommitHash();
+
+  // Current commit info
+  const previousCommit = await getCommitByHash(previousSha);
+  const previousCommitInfo = toGitCommitExtract(previousCommit);
+  addBuildkiteInfoSection(makeCommitInfoHtml('Current commit on QA:', previousCommitInfo));
+
+  // Target commit info
+  const selectedCommit = await getCommitByHash(selectedSha);
+  const selectedCommitInfo = toGitCommitExtract(selectedCommit);
+  addBuildkiteInfoSection(makeCommitInfoHtml('Target commit to deploy:', selectedCommitInfo));
+
+  // Buildkite build info
+  const buildkiteBuild = await getOnMergePRBuild(selectedSha);
+  const nextBuildContainingCommit = await getQAFBuildContainingCommit(
+    selectedSha,
+    selectedCommitInfo.date!
+  );
+  const artifactBuild = await getArtifactBuild(selectedSha);
+  addBuildkiteInfoSection(
+    makeBuildkiteBuildInfoHtml('Relevant build info:', {
+      'Merge build': buildkiteBuild,
+      'Artifact container build': artifactBuild,
+      'Next QAF test build containing this commit': nextBuildContainingCommit,
+    })
+  );
+
+  // Save Object migration comparison
+  const comparisonResult = compareSOSnapshots(previousSha, selectedSha);
+  if (comparisonResult) {
+    addBuildkiteInfoSection(makeSOComparisonBlockHtml(comparisonResult));
+  } else {
+    addBuildkiteInfoSection(makeSOComparisonErrorHtml());
+  }
+
+  // Useful links
+  addBuildkiteInfoSection(
+    makeUsefulLinksHtml('Useful links:', {
+      previousCommitHash: previousSha,
+      selectedCommitHash: selectedSha,
+    })
+  );
+}
+
+function addBuildkiteInfoSection(html: string) {
+  exec(`buildkite-agent annotate --append --style 'info' --context '${COMMIT_INFO_CTX}'`, {
+    input: html + '<br />',
+  });
+}
+
+main()
+  .then(() => {
+    console.log('Commit-related information added.');
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  })
+  .finally(() => {
+    // When running locally, we can see what calls were made to execSync to debug
+    if (!process.env.CI) {
+      // @ts-ignore
+      console.log(exec.calls);
+    }
+  });

.buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DEPLOY_TAG="deploy@$(date +%s)"
+KIBANA_COMMIT_SHA=$(buildkite-agent meta-data get selected-commit-hash)
+
+if [[ -z "$KIBANA_COMMIT_SHA" ]]; then
+  echo "Commit sha is not set, exiting."
+  exit 1
+fi
+
+echo "--- Creating deploy tag $DEPLOY_TAG at $KIBANA_COMMIT_SHA"
+
+# Set git identity to whomever triggered the buildkite job
+git config user.email "$BUILDKITE_BUILD_CREATOR_EMAIL"
+git config user.name "$BUILDKITE_BUILD_CREATOR"
+
+# Create a tag for the deploy
+git tag -a "$DEPLOY_TAG" "$KIBANA_COMMIT_SHA" \
+ -m "Tagging release $KIBANA_COMMIT_SHA as: $DEPLOY_TAG, by $BUILDKITE_BUILD_CREATOR_EMAIL"
+
+# Set meta-data for the deploy tag
+buildkite-agent meta-data set deploy-tag "$DEPLOY_TAG"
+
+# Push the tag to GitHub
+if [[ -z "${DRY_RUN:-}" ]]; then
+  echo "Pushing tag to GitHub..."
+  git push origin --tags
+else
+  echo "Skipping tag push to GitHub due to DRY_RUN=$DRY_RUN"
+fi
+
+echo "Created deploy tag: $DEPLOY_TAG - your QA release should start @ https://buildkite.com/elastic/kibana-serverless-release/builds?branch=$DEPLOY_TAG"

.buildkite/scripts/serverless/create_deploy_tag/info_sections/build_info.ts

@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { components } from '@octokit/openapi-types';
+import { buildkite, buildkiteBuildStateToEmoji, CommitWithStatuses, octokit } from '../shared';
+import { Build } from '#pipeline-utils/buildkite';
+
+const QA_FTR_TEST_SLUG = 'appex-qa-serverless-kibana-ftr-tests';
+const KIBANA_ARTIFACT_BUILD_SLUG = 'kibana-artifacts-container-image';
+const KIBANA_PR_BUILD_SLUG = 'kibana-on-merge';
+
+export interface BuildkiteBuildExtract {
+  success: boolean;
+  stateEmoji: string;
+  url: string;
+  buildNumber: number;
+  slug: string;
+  commit: string;
+  startedAt: string;
+  finishedAt: string;
+  kibanaCommit: string;
+}
+
+export async function getOnMergePRBuild(commitSha: string): Promise<BuildkiteBuildExtract | null> {
+  const buildkiteBuild = await buildkite.getBuildForCommit(KIBANA_PR_BUILD_SLUG, commitSha);
+
+  if (!buildkiteBuild) {
+    return null;
+  }
+
+  const stateEmoji = buildkiteBuildStateToEmoji(buildkiteBuild.state);
+
+  return {
+    success: buildkiteBuild.state === 'passed',
+    stateEmoji,
+    slug: KIBANA_PR_BUILD_SLUG,
+    url: buildkiteBuild.web_url,
+    buildNumber: buildkiteBuild.number,
+    commit: commitSha,
+    kibanaCommit: buildkiteBuild.commit,
+    startedAt: buildkiteBuild.started_at,
+    finishedAt: buildkiteBuild.finished_at,
+  };
+}
+
+export async function getArtifactBuild(commitSha: string): Promise<BuildkiteBuildExtract | null> {
+  const build = await buildkite.getBuildForCommit(KIBANA_ARTIFACT_BUILD_SLUG, commitSha);
+
+  if (!build) {
+    return null;
+  }
+
+  return {
+    success: build.state === 'passed',
+    stateEmoji: buildkiteBuildStateToEmoji(build.state),
+    url: build.web_url,
+    slug: KIBANA_ARTIFACT_BUILD_SLUG,
+    buildNumber: build.number,
+    commit: build.commit,
+    kibanaCommit: build.commit,
+    startedAt: build.started_at,
+    finishedAt: build.finished_at,
+  };
+}
+
+export async function getQAFBuildContainingCommit(
+  commitSha: string,
+  date: string
+): Promise<BuildkiteBuildExtract | null> {
+  // List of commits
+  const commitShaList = await getCommitListCached();
+
+  // List of QAF builds
+  const qafBuilds = await buildkite.getBuildsAfterDate(QA_FTR_TEST_SLUG, date, 30);
+
+  // Find the first build that contains this commit
+  const build = qafBuilds.find((kbBuild) => {
+    // Check if build.commit is after commitSha?
+    const kibanaCommitSha = tryGetKibanaBuildHashFromQAFBuild(kbBuild);
+    const buildkiteBuildShaIndex = commitShaList.findIndex((c) => c.sha === kibanaCommitSha);
+    const commitShaIndex = commitShaList.findIndex((c) => c.sha === commitSha);
+
+    return (
+      commitShaIndex !== -1 &&
+      buildkiteBuildShaIndex !== -1 &&
+      buildkiteBuildShaIndex < commitShaIndex
+    );
+  });
+
+  if (!build) {
+    return null;
+  }
+
+  return {
+    success: build.state === 'passed',
+    stateEmoji: buildkiteBuildStateToEmoji(build.state),
+    url: build.web_url,
+    slug: QA_FTR_TEST_SLUG,
+    buildNumber: build.number,
+    commit: build.commit,
+    kibanaCommit: tryGetKibanaBuildHashFromQAFBuild(build),
+    startedAt: build.started_at,
+    finishedAt: build.finished_at,
+  };
+}
+function tryGetKibanaBuildHashFromQAFBuild(build: Build) {
+  try {
+    const metaDataKeys = Object.keys(build.meta_data || {});
+    const anyKibanaProjectKey =
+      metaDataKeys.find((key) => key.startsWith('project::bk-serverless')) || 'missing';
+    const kibanaBuildInfo = JSON.parse(build.meta_data[anyKibanaProjectKey]);
+    return kibanaBuildInfo?.kibana_build_hash;
+  } catch (e) {
+    console.error(e);
+    return null;
+  }
+}
+
+let _commitListCache: Array<components['schemas']['commit']> | null = null;
+async function getCommitListCached() {
+  if (!_commitListCache) {
+    const resp = await octokit.request<'GET /repos/{owner}/{repo}/commits'>(
+      'GET /repos/{owner}/{repo}/commits',
+      {
+        owner: 'elastic',
+        repo: 'kibana',
+        headers: {
+          accept: 'application/vnd.github.v3+json',
+          'X-GitHub-Api-Version': '2022-11-28',
+        },
+      }
+    );
+    _commitListCache = resp.data;
+  }
+  return _commitListCache;
+}
+
+function makeBuildInfoSnippetHtml(name: string, build: BuildkiteBuildExtract | null) {
+  if (!build) {
+    return `[❓] ${name} - no build found`;
+  } else {
+    const statedAt = build.startedAt
+      ? `started at <strong>${new Date(build.startedAt).toUTCString()}</strong>`
+      : 'not started yet';
+    const finishedAt = build.finishedAt
+      ? `finished at <strong>${new Date(build.finishedAt).toUTCString()}</strong>`
+      : 'not finished yet';
+    return `[${build.stateEmoji}] <a href="${build.url}">${name} #${build.buildNumber}</a> - ${statedAt}, ${finishedAt}`;
+  }
+}
+
+export function makeBuildkiteBuildInfoHtml(
+  heading: string,
+  builds: Record<string, BuildkiteBuildExtract | null>
+): string {
+  let html = `<div><h4>${heading}</h4>`;
+  for (const [name, build] of Object.entries(builds)) {
+    html += `<div> | ${makeBuildInfoSnippetHtml(name, build)}</div>\n`;
+  }
+  html += '</div>';
+
+  return html;
+}
+
+export function makeCommitInfoWithBuildResultsHtml(commits: CommitWithStatuses[]) {
+  const commitWithBuildResultsHtml = commits.map((commitInfo) => {
+    const checks = commitInfo.checks;
+    const prBuildSnippet = makeBuildInfoSnippetHtml('on merge job', checks.onMergeBuild);
+    const ftrBuildSnippet = makeBuildInfoSnippetHtml('qaf/ftr tests', checks.ftrBuild);
+    const artifactBuildSnippet = makeBuildInfoSnippetHtml('artifact build', checks.artifactBuild);
+    const titleWithLink = commitInfo.title.replace(
+      /#(\d{4,6})/,
+      `<a href="${commitInfo.prLink}">$&</a>`
+    );
+
+    return `<div>
+  <div>
+      <div><strong><a href="${commitInfo.link}">${commitInfo.sha}</a></strong></div>
+      <div><strong>${titleWithLink}</strong><i> by ${commitInfo.author} on ${commitInfo.date}</i></div>
+      <div>| ${prBuildSnippet}</div>
+      <div>| ${artifactBuildSnippet}</div>
+      <div>| ${ftrBuildSnippet}</div>
+  </div>
+  <hr />
+</div>`;
+  });
+
+  return commitWithBuildResultsHtml.join('\n');
+}

.buildkite/scripts/serverless/create_deploy_tag/info_sections/commit_info.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods/dist-types/generated/parameters-and-response-types';
+import { buildkite, octokit, SELECTED_COMMIT_META_KEY, CURRENT_COMMIT_META_KEY } from '../shared';
+
+export type GithubCommitType = RestEndpointMethodTypes['repos']['getCommit']['response']['data'];
+export type ListedGithubCommitType =
+  RestEndpointMethodTypes['repos']['listCommits']['response']['data'][0];
+
+const KIBANA_PR_BASE = 'https://github.com/elastic/kibana/pull';
+
+export interface GitCommitExtract {
+  sha: string;
+  title: string;
+  message: string;
+  link: string;
+  date: string | undefined;
+  author: string | undefined;
+  prLink: string | undefined;
+}
+
+export async function getCurrentQARelease() {
+  const releasesFile = await octokit.request(`GET /repos/{owner}/{repo}/contents/{path}`, {
+    owner: 'elastic',
+    repo: 'serverless-gitops',
+    path: 'services/kibana/versions.yaml',
+  });
+
+  // @ts-ignore
+  const fileContent = Buffer.from(releasesFile.data.content, 'base64').toString('utf8');
+
+  const sha = fileContent.match(`qa: "([a-z0-9]+)"`)?.[1];
+
+  if (!sha) {
+    throw new Error('Could not find QA hash in current releases file');
+  } else {
+    buildkite.setMetadata(CURRENT_COMMIT_META_KEY, sha);
+    return sha;
+  }
+}
+
+export function getSelectedCommitHash() {
+  const commitHash = buildkite.getMetadata(SELECTED_COMMIT_META_KEY);
+  if (!commitHash) {
+    throw new Error(
+      `Could not find selected commit (by '${SELECTED_COMMIT_META_KEY}' in buildkite meta-data)`
+    );
+  }
+  return commitHash;
+}
+
+export async function getCommitByHash(hash: string): Promise<GithubCommitType> {
+  const commit = await octokit.repos.getCommit({
+    owner: 'elastic',
+    repo: 'kibana',
+    ref: hash,
+  });
+
+  return commit.data;
+}
+
+export async function getRecentCommits(commitCount: number): Promise<GitCommitExtract[]> {
+  const kibanaCommits: ListedGithubCommitType[] = (
+    await octokit.repos.listCommits({
+      owner: 'elastic',
+      repo: 'kibana',
+      per_page: Number(commitCount),
+    })
+  ).data;
+
+  return kibanaCommits.map(toGitCommitExtract);
+}
+
+export function toGitCommitExtract(
+  commit: GithubCommitType | ListedGithubCommitType
+): GitCommitExtract {
+  const title = commit.commit.message.split('\n')[0];
+  const prNumber = title.match(/#(\d{4,6})/)?.[1];
+  const prLink = prNumber ? `${KIBANA_PR_BASE}/${prNumber}` : undefined;
+
+  return {
+    sha: commit.sha,
+    message: commit.commit.message,
+    title,
+    link: commit.html_url,
+    date: commit.commit.author?.date || commit.commit.committer?.date,
+    author: commit.author?.login || commit.committer?.login,
+    prLink,
+  };
+}
+
+export function makeCommitInfoHtml(sectionTitle: string, commitInfo: GitCommitExtract): string {
+  const titleWithLink = commitInfo.title.replace(
+    /#(\d{4,6})/,
+    `<a href="${commitInfo.prLink}">$&</a>`
+  );
+
+  const commitDateUTC = new Date(commitInfo.date!).toUTCString();
+
+  return `<div>
+<div><h4>${sectionTitle}</h4></div>
+<div><a href="${commitInfo.link}">
+${commitInfo.sha}</a>
+ by <strong>${commitInfo.author}</strong>
+ on <strong>${commitDateUTC}</strong>
+</div>
+<div><small>:merged-pr: ${titleWithLink}</small></div>
+</div>`;
+}

.buildkite/scripts/serverless/create_deploy_tag/info_sections/so_snapshot_comparison.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import path from 'path';
+import { readFileSync } from 'fs';
+import { exec } from '../shared';
+import { BuildkiteClient, getKibanaDir } from '#pipeline-utils';
+
+export function compareSOSnapshots(
+  previousSha: string,
+  selectedSha: string
+): null | {
+  hasChanges: boolean;
+  changed: string[];
+  command: string;
+} {
+  assertValidSha(previousSha);
+  assertValidSha(selectedSha);
+
+  const command = `node scripts/snapshot_plugin_types compare --from ${previousSha} --to ${selectedSha}`;
+  const outputPath = path.resolve(getKibanaDir(), 'so_comparison.json');
+
+  try {
+    exec(`${command} --outputPath ${outputPath}`, { stdio: 'inherit' });
+
+    const soComparisonResult = JSON.parse(readFileSync(outputPath).toString());
+
+    const buildkite = new BuildkiteClient({ exec });
+    buildkite.uploadArtifacts(outputPath);
+
+    return {
+      hasChanges: soComparisonResult.hasChanges,
+      changed: soComparisonResult.changed,
+      command,
+    };
+  } catch (ex) {
+    console.error(ex);
+    return null;
+  }
+}
+
+export function makeSOComparisonBlockHtml(comparisonResult: {
+  hasChanges: boolean;
+  changed: string[];
+  command: string;
+}): string {
+  if (comparisonResult.hasChanges) {
+    return `<div>
+<h4>Plugin Saved Object migration changes: *yes, ${comparisonResult.changed.length} plugin(s)*</h4>
+<div>Changed plugins: ${comparisonResult.changed.join(', ')}</div>
+<i>Find detailed info in the archived artifacts, or run the command yourself: </i>
+<div><pre>${comparisonResult.command}</pre></div>
+</div>`;
+  } else {
+    return `<div>
+<h4>Plugin Saved Object migration changes: none</h4>
+<i>No changes between targets, you can run the command yourself to verify: </i>
+<div><pre>${comparisonResult.command}</pre></div>
+</div>`;
+  }
+}
+
+export function makeSOComparisonErrorHtml(): string {
+  return `<div>
+<h4>Plugin Saved Object migration changes: N/A</h4>
+<div>Could not compare plugin migrations. Check the logs for more info.</div>
+</div>`;
+}
+
+function assertValidSha(sha: string) {
+  if (!sha.match(/^[a-f0-9]{8,40}$/)) {
+    throw new Error(`Invalid sha: ${sha}`);
+  }
+}

.buildkite/scripts/serverless/create_deploy_tag/info_sections/useful_links.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+function link(text: string, url: string) {
+  return `<a href="${url}">${text}</a>`;
+}
+
+function getLinkForGPCTLNonProd(commit: string) {
+  return `https://overview.qa.cld.elstc.co/app/dashboards#/view/serverless-tooling-gpctl-deployment-status?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&service-name=kibana&_a=(controlGroupInput:(chainingSystem:HIERARCHICAL,controlStyle:oneLine,ignoreParentSettings:(ignoreFilters:!f,ignoreQuery:!f,ignoreTimerange:!f,ignoreValidations:!f),panels:('18201b8e-3aae-4459-947d-21e007b6a3a5':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:commit-hash,id:'18201b8e-3aae-4459-947d-21e007b6a3a5',selectedOptions:!('${commit}'),title:commit-hash),grow:!t,order:1,type:optionsListControl,width:medium),'41060e65-ce4c-414e-b8cf-492ccb19245f':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:service-name,id:'41060e65-ce4c-414e-b8cf-492ccb19245f',selectedOptions:!(kibana),title:service-name),grow:!t,order:0,type:optionsListControl,width:medium),ed96828e-efe9-43ad-be3f-0e04218f79af:(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:to-env,id:ed96828e-efe9-43ad-be3f-0e04218f79af,selectedOptions:!(qa),title:to-env),grow:!t,order:2,type:optionsListControl,width:medium))))`;
+}
+
+function getLinkForGPCTLProd(commit: string) {
+  return `https://overview.elastic-cloud.com/app/dashboards#/view/serverless-tooling-gpctl-deployment-status?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&service-name=kibana&_a=(controlGroupInput:(chainingSystem:HIERARCHICAL,controlStyle:oneLine,ignoreParentSettings:(ignoreFilters:!f,ignoreQuery:!f,ignoreTimerange:!f,ignoreValidations:!f),panels:('18201b8e-3aae-4459-947d-21e007b6a3a5':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:commit-hash,id:'18201b8e-3aae-4459-947d-21e007b6a3a5',selectedOptions:!('${commit}'),title:commit-hash),grow:!t,order:1,type:optionsListControl,width:medium),'41060e65-ce4c-414e-b8cf-492ccb19245f':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:service-name,id:'41060e65-ce4c-414e-b8cf-492ccb19245f',selectedOptions:!(kibana),title:service-name),grow:!t,order:0,type:optionsListControl,width:medium),ed96828e-efe9-43ad-be3f-0e04218f79af:(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:to-env,id:ed96828e-efe9-43ad-be3f-0e04218f79af,selectedOptions:!(production),title:to-env),grow:!t,order:2,type:optionsListControl,width:medium))))`;
+}
+
+export function getUsefulLinks({
+  selectedCommitHash,
+  previousCommitHash,
+}: {
+  previousCommitHash: string;
+  selectedCommitHash: string;
+}): Record<string, string> {
+  return {
+    'Commits contained in deploy': `https://github.com/elastic/kibana/compare/${previousCommitHash}...${selectedCommitHash}`,
+    'Argo Workflow (use Elastic Cloud Staging VPN)': `https://argo-workflows.cd.internal.qa.elastic.cloud/workflows?label=hash%3D${selectedCommitHash}`,
+    'GPCTL Deployment Status dashboard for nonprod': getLinkForGPCTLNonProd(selectedCommitHash),
+    'GPCTL Deployment Status dashboard for prod': getLinkForGPCTLProd(selectedCommitHash),
+    'Quality Gate pipeline': `https://buildkite.com/elastic/kibana-tests/builds?branch=main`,
+    'Kibana Serverless Release pipeline': `https://buildkite.com/elastic/kibana-serverless-release/builds?commit=${selectedCommitHash}`,
+  };
+}
+
+export function makeUsefulLinksHtml(
+  heading: string,
+  data: {
+    previousCommitHash: string;
+    selectedCommitHash: string;
+  }
+) {
+  return (
+    `<h4>${heading}</h4>` +
+    Object.entries(getUsefulLinks(data))
+      .map(([name, url]) => `<div>:link: ${link(name, url)}</div>`)
+      .join('\n')
+  );
+}

.buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  buildkite,
+  COMMIT_INFO_CTX,
+  CommitWithStatuses,
+  exec,
+  SELECTED_COMMIT_META_KEY,
+} from './shared';
+import {
+  getArtifactBuild,
+  getOnMergePRBuild,
+  getQAFBuildContainingCommit,
+  makeCommitInfoWithBuildResultsHtml,
+} from './info_sections/build_info';
+import { getRecentCommits, GitCommitExtract } from './info_sections/commit_info';
+import { BuildkiteInputStep } from '#pipeline-utils';
+
+async function main(commitCountArg: string) {
+  console.log('--- Listing commits');
+  const commitCount = parseInt(commitCountArg, 10);
+  const commitData = await collectAvailableCommits(commitCount);
+  const commitsWithStatuses = await enrichWithStatuses(commitData);
+
+  console.log('--- Updating buildkite context with listed commits');
+  const commitListWithBuildResultsHtml = makeCommitInfoWithBuildResultsHtml(commitsWithStatuses);
+  exec(`buildkite-agent annotate --style 'info' --context '${COMMIT_INFO_CTX}'`, {
+    input: commitListWithBuildResultsHtml,
+  });
+
+  console.log('--- Generating buildkite input step');
+  addBuildkiteInputStep();
+}
+
+async function collectAvailableCommits(commitCount: number): Promise<GitCommitExtract[]> {
+  console.log('--- Collecting recent kibana commits');
+
+  const recentCommits = await getRecentCommits(commitCount);
+
+  if (!recentCommits) {
+    throw new Error('Could not find any, while listing recent commits');
+  }
+
+  return recentCommits;
+}
+
+async function enrichWithStatuses(commits: GitCommitExtract[]): Promise<CommitWithStatuses[]> {
+  console.log('--- Enriching with build statuses');
+
+  const commitsWithStatuses: CommitWithStatuses[] = await Promise.all(
+    commits.map(async (commit) => {
+      const onMergeBuild = await getOnMergePRBuild(commit.sha);
+
+      if (!commit.date) {
+        return {
+          ...commit,
+          checks: {
+            onMergeBuild,
+            ftrBuild: null,
+            artifactBuild: null,
+          },
+        };
+      }
+
+      const nextFTRBuild = await getQAFBuildContainingCommit(commit.sha, commit.date);
+      const artifactBuild = await getArtifactBuild(commit.sha);
+
+      return {
+        ...commit,
+        checks: {
+          onMergeBuild,
+          ftrBuild: nextFTRBuild,
+          artifactBuild,
+        },
+      };
+    })
+  );
+
+  return commitsWithStatuses;
+}
+
+function addBuildkiteInputStep() {
+  const inputStep: BuildkiteInputStep = {
+    input: 'Select commit to deploy',
+    prompt: 'Select commit to deploy.',
+    key: 'select-commit',
+    fields: [
+      {
+        text: 'Enter the release candidate commit SHA',
+        key: SELECTED_COMMIT_META_KEY,
+      },
+    ],
+  };
+
+  buildkite.uploadSteps([inputStep]);
+}
+
+main(process.argv[2])
+  .then(() => {
+    console.log('Commit selector generated, added as a buildkite input step.');
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });

.buildkite/scripts/serverless/create_deploy_tag/mock_exec.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * This file has a wrapper for exec, that stores answers for queries from a file, to be able to use it in tests.
+ */
+
+import { execSync, ExecSyncOptions } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { getKibanaDir } from '#pipeline-utils';
+
+const PREPARED_RESPONSES_PATH =
+  '.buildkite/scripts/serverless/create_deploy_tag/prepared_responses.json';
+
+/**
+ * This module allows for a stand-in for execSync that stores calls, and responds from a file of recorded responses.
+ * Most of the components in this module are lazy, so that they are only initialized if needed.
+ * @param fake - if set to true, it will use the fake, prepared exec, if false, it will use child_process.execSync
+ * @param id - an optional ID, used to distinguish between different instances of exec.
+ */
+const getExec = (fake = false, id: string = randomId()) => {
+  return fake ? makeMockExec(id) : exec;
+};
+
+/**
+ * Lazy getter for a storage for calls to the mock exec.
+ */
+const getCallStorage: () => Record<string, Array<{ command: string; opts: any }>> = (() => {
+  let callStorage: Record<string, Array<{ command: string; opts: any }>> | null = null;
+
+  return () => {
+    if (!callStorage) {
+      callStorage = new Proxy<Record<string, Array<{ command: string; opts: any }>>>(
+        {},
+        {
+          get: (target, prop: string) => {
+            if (!target[prop]) {
+              target[prop] = [];
+            }
+            return target[prop];
+          },
+        }
+      );
+    }
+    return callStorage;
+  };
+})();
+
+/**
+ * Lazy getter for the responses file.
+ */
+const loadFakeResponses = (() => {
+  let responses: any;
+  return () => {
+    if (!responses) {
+      const responsesFile = path.resolve(getKibanaDir(), PREPARED_RESPONSES_PATH);
+      if (fs.existsSync(responsesFile)) {
+        const responsesContent = fs.readFileSync(responsesFile).toString();
+        responses = JSON.parse(responsesContent);
+      } else {
+        fs.writeFileSync(responsesFile, '{}');
+        console.log(responsesFile, 'created');
+        responses = {};
+      }
+    }
+
+    return responses;
+  };
+})();
+
+const makeMockExec = (id: string) => {
+  console.warn("--- Using mock exec, don't use this on CI. ---");
+  const callStorage = getCallStorage();
+  const calls = callStorage[id];
+
+  const mockExecInstance = (command: string, opts: ExecSyncOptions = {}): string | null => {
+    const responses = loadFakeResponses();
+    calls.push({ command, opts });
+
+    if (typeof responses[command] !== 'undefined') {
+      return responses[command];
+    } else {
+      console.warn(`No response for command: ${command}`);
+      responses[command] = '<missing>';
+      fs.writeFileSync(
+        path.resolve(getKibanaDir(), PREPARED_RESPONSES_PATH),
+        JSON.stringify(responses, null, 2)
+      );
+      return exec(command, opts);
+    }
+  };
+
+  mockExecInstance.id = id;
+  mockExecInstance.calls = calls;
+
+  return mockExecInstance;
+};
+
+const exec = (command: string, opts: any = {}) => {
+  const result = execSync(command, { encoding: 'utf-8', cwd: getKibanaDir(), ...opts });
+  if (result) {
+    return result.toString().trim();
+  } else {
+    return null;
+  }
+};
+
+const randomId = () => (Math.random() * 10e15).toString(36);
+
+export { getExec, getCallStorage };

.buildkite/scripts/serverless/create_deploy_tag/prepared_responses.json

@@ -0,0 +1,13 @@
+{
+  "buildkite-agent annotate --append --style 'info' --context 'commit-info'": "ok",
+  "buildkite-agent meta-data get \"commit-sha\"": "906987c2860b53b91d449bc164957857adddc06a",
+  "node scripts/snapshot_plugin_types compare --from b5aa37525578 --to 906987c2860b53b91d449bc164957857adddc06a --outputPath 'so_comparison.json'": "ok",
+  "buildkite-agent artifact upload 'so_comparison.json'": "ok",
+  "buildkite-agent meta-data get 'release_state'": "",
+  "buildkite-agent meta-data get 'state_data'": "",
+  "buildkite-agent meta-data set 'release_state'": "ok",
+  "buildkite-agent meta-data set 'state_data'": "ok",
+  "buildkite-agent annotate --context 'wizard-main' --style 'info'": "ok",
+  "buildkite-agent annotate --context 'wizard-instruction' --style 'info'": "ok",
+  "buildkite-agent annotate --context 'wizard-instruction' --style 'warning'": "ok"
+}

.buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts

@@ -0,0 +1,372 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  buildkite,
+  COMMIT_INFO_CTX,
+  CURRENT_COMMIT_META_KEY,
+  DEPLOY_TAG_META_KEY,
+  octokit,
+  SELECTED_COMMIT_META_KEY,
+  sendSlackMessage,
+} from './shared';
+import { GithubCommitType } from './info_sections/commit_info';
+import { getUsefulLinks } from './info_sections/useful_links';
+
+const WIZARD_CTX_INSTRUCTION = 'wizard-instruction';
+const WIZARD_CTX_DEFAULT = 'wizard-main';
+
+type StateNames =
+  | 'start'
+  | 'initialize'
+  | 'collect_commits'
+  | 'wait_for_selection'
+  | 'collect_commit_info'
+  | 'wait_for_confirmation'
+  | 'create_deploy_tag'
+  | 'tag_created'
+  | 'end'
+  | 'error_generic'
+  | string;
+
+interface StateShape {
+  name: string;
+  description: string;
+  instruction?: string;
+  instructionStyle?: 'success' | 'warning' | 'error' | 'info';
+  display: boolean;
+  pre?: (state: StateShape) => Promise<void | boolean>;
+  post?: (state: StateShape) => Promise<void | boolean>;
+}
+
+const states: Record<StateNames, StateShape> = {
+  start: {
+    name: 'Starting state',
+    description: 'No description',
+    display: false,
+    post: async () => {
+      buildkite.setAnnotation(COMMIT_INFO_CTX, 'info', `<h4>:kibana: Release candidates</h4>`);
+    },
+  },
+  initialize: {
+    name: 'Initializing',
+    description: 'The job is starting up.',
+    instruction: 'Wait while we bootstrap. Follow the instructions displayed in this block.',
+    instructionStyle: 'info',
+    display: true,
+  },
+  collect_commits: {
+    name: 'Collecting commits',
+    description: 'Collecting potential commits for the release.',
+    instruction: `Please wait, while we're collecting the list of available commits.`,
+    instructionStyle: 'info',
+    display: true,
+  },
+  wait_for_selection: {
+    name: 'Waiting for selection',
+    description: 'Waiting for the Release Manager to select a release candidate commit.',
+    instruction: `Please find, copy and enter a commit SHA to the buildkite input box to proceed.`,
+    instructionStyle: 'warning',
+    display: true,
+  },
+  collect_commit_info: {
+    name: 'Collecting commit info',
+    description: 'Collecting supplementary info about the selected commit.',
+    instruction: `Please wait, while we're collecting data about the commit, and the release candidate.`,
+    instructionStyle: 'info',
+    display: true,
+    pre: async () => {
+      buildkite.setAnnotation(
+        COMMIT_INFO_CTX,
+        'info',
+        `<h4>:kibana: Selected release candidate info:</h4>`
+      );
+    },
+  },
+  wait_for_confirmation: {
+    name: 'Waiting for confirmation',
+    description: 'Waiting for the Release Manager to confirm the release.',
+    instruction: `Please review the collected information above and unblock the release on Buildkite, if you're satisfied.`,
+    instructionStyle: 'warning',
+    display: true,
+  },
+  create_deploy_tag: {
+    name: 'Creating deploy tag',
+    description: 'Creating the deploy tag, this will be picked up by another pipeline.',
+    instruction: `Please wait, while we're creating the deploy@timestamp tag.`,
+    instructionStyle: 'info',
+    display: true,
+  },
+  tag_created: {
+    name: 'Release tag created',
+    description: 'The initial step release is completed, follow up jobs will be triggered soon.',
+    instruction: `<h3>Deploy tag successfully created!</h3>`,
+    post: async () => {
+      // The deployTag here is only for communication, if it's missing, it's not a big deal, but it's an error
+      const deployTag =
+        buildkite.getMetadata(DEPLOY_TAG_META_KEY) ||
+        (console.error(`${DEPLOY_TAG_META_KEY} not found in buildkite meta-data`), 'unknown');
+      const selectedCommit = buildkite.getMetadata(SELECTED_COMMIT_META_KEY);
+      const currentCommitSha = buildkite.getMetadata(CURRENT_COMMIT_META_KEY);
+
+      buildkite.setAnnotation(
+        WIZARD_CTX_INSTRUCTION,
+        'success',
+        `<h3>Deploy tag successfully created!</h3><br/>
+Your deployment will appear <a href='https://buildkite.com/elastic/kibana-serverless-release/builds?branch=${deployTag}'>here on buildkite.</a>`
+      );
+
+      if (!selectedCommit) {
+        // If we get here with no selected commit set, it's either an unsynced change in keys, or some weird error.
+        throw new Error(
+          `Couldn't find selected commit in buildkite meta-data (with key '${SELECTED_COMMIT_META_KEY}').`
+        );
+      }
+
+      const targetCommitData = (
+        await octokit.repos.getCommit({
+          owner: 'elastic',
+          repo: 'kibana',
+          ref: selectedCommit,
+        })
+      ).data;
+
+      await sendReleaseSlackAnnouncement({
+        targetCommitData,
+        currentCommitSha,
+        deployTag,
+      });
+    },
+    instructionStyle: 'success',
+    display: true,
+  },
+  end: {
+    name: 'End of the release process',
+    description: 'The release process has ended.',
+    display: false,
+  },
+  error_generic: {
+    name: 'Encountered an error',
+    description: 'An error occurred during the release process.',
+    instruction: `<h4>Please check the build logs for more information.</h4>`,
+    instructionStyle: 'error',
+    display: false,
+  },
+};
+
+/**
+ * This module is a central interface for updating the messaging interface for the wizard.
+ * It's implemented as a state machine that updates the wizard state as we transition between states.
+ * Use: `node <dir>/release_wizard_messaging.ts --state <state_name> [--data <data>]`
+ */
+export async function main(args: string[]) {
+  if (!args.includes('--state')) {
+    throw new Error('Missing --state argument');
+  }
+  const targetState = args.slice(args.indexOf('--state') + 1)[0];
+
+  let data: any;
+  if (args.includes('--data')) {
+    data = args.slice(args.indexOf('--data') + 1)[0];
+  }
+
+  const resultingTargetState = await transition(targetState, data);
+  if (resultingTargetState === 'tag_created') {
+    return await transition('end');
+  } else {
+    return resultingTargetState;
+  }
+}
+
+export async function transition(targetStateName: StateNames, data?: any) {
+  // use the buildkite agent to find what state we are in:
+  const currentStateName = buildkite.getMetadata('release_state') || 'start';
+  const stateData = JSON.parse(buildkite.getMetadata('state_data') || '{}');
+
+  if (!currentStateName) {
+    throw new Error('Could not find current state in buildkite meta-data');
+  }
+
+  // find the index of the current state in the core flow
+  const currentStateIndex = Object.keys(states).indexOf(currentStateName);
+  const targetStateIndex = Object.keys(states).indexOf(targetStateName);
+
+  if (currentStateIndex === -1) {
+    throw new Error(`Could not find current state '${currentStateName}' in core flow`);
+  }
+  const currentState = states[currentStateName];
+
+  if (targetStateIndex === -1) {
+    throw new Error(`Could not find target state '${targetStateName}' in core flow`);
+  }
+  const targetState = states[targetStateName];
+
+  if (currentStateIndex + 1 !== targetStateIndex) {
+    await tryCall(currentState.post, stateData);
+    stateData[currentStateName] = 'nok';
+  } else {
+    const result = await tryCall(currentState.post, stateData);
+    stateData[currentStateName] = result ? 'ok' : 'nok';
+  }
+  stateData[targetStateName] = 'pending';
+
+  await tryCall(targetState.pre, stateData);
+
+  buildkite.setMetadata('release_state', targetStateName);
+  buildkite.setMetadata('state_data', JSON.stringify(stateData));
+
+  updateWizardState(stateData);
+  updateWizardInstruction(targetStateName, stateData);
+
+  return targetStateName;
+}
+
+function updateWizardState(stateData: Record<string, 'ok' | 'nok' | 'pending' | undefined>) {
+  const wizardHeader = `<h3>:kibana: Kibana Serverless deployment wizard :mage:</h3>`;
+
+  const wizardSteps = Object.keys(states)
+    .filter((stateName) => states[stateName].display)
+    .map((stateName) => {
+      const stateInfo = states[stateName];
+      const stateStatus = stateData[stateName];
+      const stateEmoji = {
+        ok: ':white_check_mark:',
+        nok: ':x:',
+        pending: ':hourglass_flowing_sand:',
+        missing: ':white_circle:',
+      }[stateStatus || 'missing'];
+
+      if (stateStatus === 'pending') {
+        return `<div>[${stateEmoji}] ${stateInfo.name}<br />&nbsp; - ${stateInfo.description}</div>`;
+      } else {
+        return `<div>[${stateEmoji}] ${stateInfo.name}</div>`;
+      }
+    });
+
+  const wizardHtml = `<section>
+${wizardHeader}
+${wizardSteps.join('\n')}
+</section>`;
+
+  buildkite.setAnnotation(WIZARD_CTX_DEFAULT, 'info', wizardHtml);
+}
+
+function updateWizardInstruction(targetState: string, stateData: any) {
+  const { instructionStyle, instruction } = states[targetState];
+
+  if (instruction) {
+    buildkite.setAnnotation(
+      WIZARD_CTX_INSTRUCTION,
+      instructionStyle || 'info',
+      `<strong>${instruction}</strong>`
+    );
+  }
+}
+
+async function tryCall(fn: any, ...args: any[]) {
+  if (typeof fn === 'function') {
+    try {
+      const result = await fn(...args);
+      return result !== false;
+    } catch (error) {
+      console.error(error);
+      return false;
+    }
+  } else {
+    return true;
+  }
+}
+
+async function sendReleaseSlackAnnouncement({
+  targetCommitData,
+  currentCommitSha,
+  deployTag,
+}: {
+  targetCommitData: GithubCommitType;
+  currentCommitSha: string | undefined | null;
+  deployTag: string;
+}) {
+  const textBlock = (...str: string[]) => ({ type: 'mrkdwn', text: str.join('\n') });
+  const buildShortname = `kibana-serverless-release #${process.env.BUILDKITE_BUILD_NUMBER}`;
+
+  const isDryRun = process.env.DRY_RUN?.match('(1|true)');
+  const mergedAtDate = targetCommitData.commit?.committer?.date;
+  const mergedAtUtcString = mergedAtDate ? new Date(mergedAtDate).toUTCString() : 'unknown';
+  const targetCommitSha = targetCommitData.sha;
+  const targetCommitShort = targetCommitSha.slice(0, 12);
+  const compareResponse = (
+    await octokit.repos.compareCommits({
+      owner: 'elastic',
+      repo: 'kibana',
+      base: currentCommitSha || 'main',
+      head: targetCommitSha,
+    })
+  ).data;
+  const compareLink = currentCommitSha
+    ? `<${compareResponse.html_url}|${compareResponse.total_commits} new commits>`
+    : 'a new release candidate';
+
+  const mainMessage = [
+    `:ship_it_parrot: Promotion of ${compareLink} to QA has been <${process.env.BUILDKITE_BUILD_URL}|initiated>!\n`,
+    `*Remember:* Promotion to Staging is currently a manual process and will proceed once the build is signed off in QA.\n`,
+  ];
+  if (isDryRun) {
+    mainMessage.unshift(
+      `*:memo:This is a dry run - no commit will actually be promoted. Please ignore!*\n`
+    );
+  } else {
+    mainMessage.push(`cc: @kibana-serverless-promotion-notify`);
+  }
+
+  const linksSection = {
+    'Initiated by': process.env.BUILDKITE_BUILD_CREATOR || 'unknown',
+    'Pre-release job': `<${process.env.BUILDKITE_BUILD_URL}|${buildShortname}>`,
+    'Git tag': `<https://github.com/elastic/kibana/releases/tag/${deployTag}|${deployTag}>`,
+    Commit: `<https://github.com/elastic/kibana/commit/${targetCommitShort}|${targetCommitShort}>`,
+    'Merged at': mergedAtUtcString,
+  };
+
+  const usefulLinksSection = getUsefulLinks({
+    previousCommitHash: currentCommitSha || 'main',
+    selectedCommitHash: targetCommitSha,
+  });
+
+  return sendSlackMessage({
+    blocks: [
+      {
+        type: 'section',
+        text: textBlock(...mainMessage),
+      },
+      {
+        type: 'section',
+        fields: Object.entries(linksSection).map(([name, link]) => textBlock(`*${name}*:`, link)),
+      },
+      {
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text:
+            '*Useful links:*\n\n' +
+            Object.entries(usefulLinksSection)
+              .map(([name, link]) => ` • <${link}|${name}>`)
+              .join('\n'),
+        },
+      },
+    ],
+  });
+}
+
+main(process.argv.slice(2)).then(
+  (targetState) => {
+    console.log('Transition completed to: ' + targetState);
+  },
+  (error) => {
+    console.error(error);
+    process.exit(1);
+  }
+);

.buildkite/scripts/serverless/create_deploy_tag/shared.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import axios from 'axios';
+
+import { getExec } from './mock_exec';
+import { GitCommitExtract } from './info_sections/commit_info';
+import { BuildkiteBuildExtract } from './info_sections/build_info';
+import { BuildkiteClient, getGithubClient } from '#pipeline-utils';
+
+const SELECTED_COMMIT_META_KEY = 'selected-commit-hash';
+const CURRENT_COMMIT_META_KEY = 'current-commit-hash';
+
+const DEPLOY_TAG_META_KEY = 'deploy-tag';
+const COMMIT_INFO_CTX = 'commit-info';
+
+const octokit = getGithubClient();
+
+const exec = getExec(!process.env.CI);
+
+const buildkite = new BuildkiteClient({ exec });
+
+const buildkiteBuildStateToEmoji = (state: string) => {
+  return (
+    {
+      running: '⏳',
+      scheduled: '⏳',
+      passed: '✅',
+      failed: '❌',
+      blocked: '❌',
+      canceled: '❌',
+      canceling: '❌',
+      skipped: '❌',
+      not_run: '❌',
+      finished: '✅',
+    }[state] || '❓'
+  );
+};
+
+export {
+  octokit,
+  exec,
+  buildkite,
+  buildkiteBuildStateToEmoji,
+  SELECTED_COMMIT_META_KEY,
+  COMMIT_INFO_CTX,
+  DEPLOY_TAG_META_KEY,
+  CURRENT_COMMIT_META_KEY,
+};
+
+export interface CommitWithStatuses extends GitCommitExtract {
+  title: string;
+  author: string | undefined;
+  checks: {
+    onMergeBuild: BuildkiteBuildExtract | null;
+    ftrBuild: BuildkiteBuildExtract | null;
+    artifactBuild: BuildkiteBuildExtract | null;
+  };
+}
+
+export function sendSlackMessage(payload: any) {
+  if (!process.env.DEPLOY_TAGGER_SLACK_WEBHOOK_URL) {
+    console.log('No SLACK_WEBHOOK_URL set, not sending slack message');
+    return Promise.resolve();
+  } else {
+    return axios
+      .post(
+        process.env.DEPLOY_TAGGER_SLACK_WEBHOOK_URL,
+        typeof payload === 'string' ? payload : JSON.stringify(payload)
+      )
+      .catch((error) => {
+        if (axios.isAxiosError(error) && error.response) {
+          console.error(
+            "Couldn't send slack message.",
+            error.response.status,
+            error.response.statusText,
+            error.message
+          );
+        } else {
+          console.error("Couldn't send slack message.", error.message);
+        }
+      });
+  }
+}

src/dev/so_migration/compare_snapshots.ts

@@ -45,7 +45,7 @@ async function compareSnapshots({
 
   log.info(
     `Snapshots compared: ${from} <=> ${to}. ` +
-      `${result.hasChanges ? 'No changes' : 'Changed: ' + result.changed.join(', ')}`
+      `${result.hasChanges ? 'Changed: ' + result.changed.join(', ') : 'No changes'}`
   );
 
   if (outputPath) {