[OAS] Publish OAS bundles to bump.sh (#197482)

## Summary

Publish OAS docs to bump.sh on merge to `main` or `8.x`

## To reviewers
* For now actual publication requires a manual step on bump.sh (so
things aren't going live immediately)
* Will get to serverless OAS docs next!

## Blockers

* Address vulnerable deps before merging:
https://github.com/bump-sh/cli/issues/583

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/package-lock.json

@@ -1515,9 +1515,9 @@
       }
     },
     "node_modules/tslib": {
-      "version": "2.4.0",
+      "version": "2.8.0",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz",
-      "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ=="
+      "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA=="
     },
     "node_modules/type-detect": {
       "version": "4.0.8",
@@ -2742,9 +2742,9 @@
       }
     },
     "tslib": {
-      "version": "2.4.0",
+      "version": "2.8.0",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz",
-      "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ=="
+      "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA=="
     },
     "type-detect": {
       "version": "4.0.8",

.buildkite/pipelines/on_merge.yml

@@ -493,6 +493,21 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/openapi_publishing/publish_oas_docs.sh
+    label: 'Publish OAS docs to bump.sh'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
+    timeout_in_minutes: 60
+    continue_on_failure: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
     label: 'Populate local dev bazel cache (Linux)'
     agents:

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -4,8 +4,16 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
-echo --- Merge Kibana OpenAPI specs
+cur_dir=$(pwd)
+cd oas_docs
 
-(cd oas_docs && make api-docs && make api-docs-lint)
+echo --- Installing NPM modules
+npm install
+
+echo --- Merge Kibana OpenAPI specs
+make api-docs
+# make api-docs-lint <-- Relies on JSONPath version with RCE vulnerabilities based on `npm audit`, https://github.com/advisories/GHSA-pppg-cpfq-h7wr
+
+cd "$cur_dir"
 
 check_for_changed_files "make api-docs" true

.buildkite/scripts/steps/openapi_publishing/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "serverless",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "bump-cli": "^2.8.4"
+  }
+}

.buildkite/scripts/steps/openapi_publishing/publish_oas_docs.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo "--- Publish OAS docs"
+
+echo "--- Installing NPM modules"
+
+deploy_to_bump() {
+  local file_path="${1:-}"
+  local doc_name="${2:-}"
+  local doc_token="${3:-}"
+  local branch="${4:-}"
+  local cur_dir=$(pwd)
+
+  cd "$(dirname "${BASH_SOURCE[0]}")"
+  npm install
+
+  echo "Checking diff for doc '$doc_name' against file '$file_path' on branch '$branch'..."
+  local result=$(npx bump diff $file_path --doc $doc_name --token $doc_token --branch $branch --format=json)
+  local change_count=$(jq '. | length' <<<$result)
+  if [[ ! -z $change_count && $change_count -gt 0 ]]; then
+    echo "Found $change_count changes..."
+    echo "About to deploy file '$file_path' to doc '$doc_name' to '$branch' on bump.sh..."
+    npx bump deploy $file_path \
+      --branch $branch \
+      --doc $doc_name \
+      --token $doc_token ;
+    echo ""
+    echo "Note: if there is a warning of unchanged docs we probably have unpublished deployments waiting."
+    echo "Go to https://bump.sh/elastic/dashboard to see all the docs in the hub."
+  else
+    echo "Did not detect changes for '$file_path'; not deploying. Got response: '$result'"
+  fi
+
+  echo "Switch back to dir '$cur_dir'"
+  cd $cur_dir
+}
+
+if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
+  BUMP_KIBANA_DOC_NAME="$(vault_get kibana-bump-sh kibana-doc-name)"
+  BUMP_KIBANA_DOC_TOKEN="$(vault_get kibana-bump-sh kibana-token)"
+  deploy_to_bump "$(pwd)/oas_docs/output/kibana.yaml" $BUMP_KIBANA_DOC_NAME $BUMP_KIBANA_DOC_TOKEN main;
+  exit 0;
+fi
+
+if [[ "$BUILDKITE_BRANCH" == "8.x" ]]; then
+  BUMP_KIBANA_DOC_NAME="$(vault_get kibana-bump-sh kibana-doc-name)"
+  BUMP_KIBANA_DOC_TOKEN="$(vault_get kibana-bump-sh kibana-token)"
+  deploy_to_bump "$(pwd)/oas_docs/output/kibana.yaml" $BUMP_KIBANA_DOC_NAME $BUMP_KIBANA_DOC_TOKEN 8x-unreleased;
+  exit 0;
+fi
+
+echo "No branches found to push to; stopping here."

oas_docs/README.md

@@ -39,13 +39,4 @@ The `oas_docs/output` folder contains the final resulting Kibana OpenAPI bundles
 
 ## Bundling commands
 
-Besides the scripts in the `oas_docs/scripts` folder, there is an `oas_docs/makefile` to simplify the workflow. The following makefile targets are available:
+Besides the scripts in the `oas_docs/scripts` folder, there is an `oas_docs/makefile` to simplify the workflow. Use `make help` to see available commands.
-
-| Command                    | Description                                                                                                                                                                                                        |
-| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `api-docs`                 | Builds ESS Kibana OpenAPI bundle                                                                                                                                                                                   |
-| `api-docs-serverless`      | Builds Serverless Kibana OpenAPI bundle                                                                                                                                                                            |
-| `api-docs-lint`            | Lints built result bundles              |
-| `api-docs-preview`         | Generates (ESS + Serverless) Kibana OpenAPI bundles preview                                                                                                                                                        |
-| `api-docs-overlay`         | Applies [overlays](https://docs.bump.sh/help/specification-support/overlays/) from `overlays` folder to the Kibana OpenAPI bundles and generate `*.new.yaml` files. Overlays help to fine tune the result bundles. |
-| `api-docs-overlay-preview` | Generates a preview for bundles produced by `api-docs-overlay`                                                                                                                                                     |

oas_docs/makefile

@@ -14,16 +14,21 @@
 # permission is obtained from Elasticsearch B.V.
 
 .PHONY: api-docs
-api-docs: ## Generate Serverless and ESS Kibana OpenAPI bundles with kbn-openapi-bundler
+api-docs: ## Generate Serverless and ESS Kibana OpenAPI bundles
+	$(MAKE) merge-api-docs
+	$(MAKE) api-docs-overlay
+
+.PHONY: merge-api-docs
+merge-api-docs: ## Merge Serverless and ESS Kibana OpenAPI bundles with kbn-openapi-bundler
 	@node scripts/merge_serverless_oas.js
 	@node scripts/merge_ess_oas.js
 
-.PHONY: api-docs-stateful
+.PHONY: merge-api-docs-stateful
-api-docs-stateful: ## Generate only kibana.yaml
+merge-api-docs-stateful: ## Merge only kibana.yaml
 	@node scripts/merge_ess_oas.js
 
-.PHONY: api-docs-serverless
+.PHONY: merge-api-docs-serverless
-api-docs-serverless: ## Generate only kibana.serverless.yaml
+merge-api-docs-serverless: ## Merge only kibana.serverless.yaml
 	@node scripts/merge_serverless_oas.js
 
 .PHONY: api-docs-lint
@@ -39,7 +44,7 @@ api-docs-lint-serverless: ## Run redocly API docs linter on kibana.serverless.ya
 	@npx @redocly/cli lint "output/kibana.serverless.yaml" --config "linters/redocly.yaml" --format stylish --max-problems 500
 
 .PHONY: api-docs-overlay
-api-docs-overlay: ## Run spectral API docs linter on kibana.serverless.yaml
+api-docs-overlay: ## Apply all overlays
 	@npx bump-cli overlay "output/kibana.serverless.yaml" "overlays/kibana.overlays.serverless.yaml" > "output/kibana.serverless.tmp1.yaml"
 	@npx bump-cli overlay "output/kibana.serverless.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.serverless.tmp2.yaml"
 	@npx bump-cli overlay "output/kibana.serverless.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.serverless.tmp3.yaml"
@@ -48,24 +53,20 @@ api-docs-overlay: ## Run spectral API docs linter on kibana.serverless.yaml
 	@npx bump-cli overlay "output/kibana.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.tmp2.yaml"
 	@npx bump-cli overlay "output/kibana.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.tmp3.yaml"
 	@npx bump-cli overlay "output/kibana.tmp3.yaml" "overlays/kibana.overlays.shared.yaml" > "output/kibana.tmp4.yaml"
-	@npx @redocly/cli bundle output/kibana.serverless.tmp4.yaml --ext yaml -o output/kibana.serverless.new.yaml
+	@npx @redocly/cli bundle output/kibana.serverless.tmp4.yaml --ext yaml -o output/kibana.serverless.yaml
-	@npx @redocly/cli bundle output/kibana.tmp4.yaml --ext yaml -o output/kibana.new.yaml
+	@npx @redocly/cli bundle output/kibana.tmp4.yaml --ext yaml -o output/kibana.yaml
 	rm output/kibana.tmp*.yaml
 	rm output/kibana.serverless.tmp*.yaml
 
 .PHONY: api-docs-preview
 api-docs-preview: ## Generate a preview for kibana.yaml and kibana.serverless.yaml
+	@echo "Rendering stateful docs preview..."
 	@npx bump-cli preview "output/kibana.yaml"
+	@echo "Rendering serverless docs preview..."
 	@npx bump-cli preview "output/kibana.serverless.yaml"
 
-.PHONY: api-docs-overlay-preview
-api-docs-overlay-preview: ## Generate a preview for kibana.new.yaml and kibana.serverless.new.yaml
-	@npx bump-cli preview "output/kibana.new.yaml"
-	@npx bump-cli preview "output/kibana.serverless.new.yaml"
-
 help:  ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 #------------- <https://suva.sh/posts/well-documented-makefiles> --------------
 
 .DEFAULT_GOAL := help
-

oas_docs/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "oas_docs",
+  "version": "1.0.0",
+  "description": "Documentation about our OpenAPI bundling workflow and configuration. See Kibana's hosted [stateful](https://www.elastic.co/docs/api/doc/kibana) and [serverless](https://www.elastic.co/docs/api/doc/serverless) docs.",
+  "main": "index.js",
+  "directories": {
+    "example": "examples"
+  },
+  "dependencies": {
+    "bump-cli": "^2.8.4",
+    "@redocly/cli": "^1.25.7"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC"
+}