[HTTP/OAS] Merge OpenAPI specs by using kbn-openapi-bundler (#189262)

**Addresses:** https://github.com/elastic/kibana/issues/186356
**Relates to:** https://github.com/elastic/kibana/issues/184428

## Summary

This PR adds a merging JS script based on the utility implemented in https://github.com/elastic/kibana/issues/186356. Resulted OpenAPI bundle as committed in `oas_docs/output/kibana.serverless.bundled.yaml`.

## Details

https://github.com/elastic/kibana/pull/188110 implements and exposes `merge` utility design to merge source OpenAPI specs without processing. It's has only a programmatic API. To merge OpenAPI specs it's required to add a JS script like below

```js
const { merge } = require('@kbn/openapi-bundler');

(async () => {
  await merge({
   sourceGlobs: [/* a list of source globs goes here */],
   outputFilePath: 'path/to/the/output/file.yaml',
  });
})();
```

The JS script added in this PR includes source OpenAPI specs presented in `oas_docs/makefile` plus Security Solution OpenAPI specs based on https://github.com/elastic/kibana/issues/184428.

**To run** the script use the following command from Kibana root folder

```bash
node ./oas_docs/scripts/merge_serverless_oas.js 
```

## Known linting issues with Security Solution OpenAPI specs

Running Spectral OpenAPI linter on the result bundle shows a number of errors caused by `no-$ref-siblings` rule. This caused by the current code generator implementation which requires `default` property to be set next to `$ref` though it's not correct for OpenAPI `3.0.3` while it's allowed in `3.1`. It seems that Bump.sh handles such cases properly though by properly showing a default value.

We need to analyze the problem and decide if/when we should fix it.

The rest of warnings look fixable and will be addressed in the next stage after setting up linter rules.

## Next steps

Since `@kbn/openapi-bundler` package is tailored specifically for Kibana it should replace Redocly currently used to merge OpenAPI specs. It also means `oas_docs/makefile` should be superseded by JS script(s) using `merge` utility form `@kbn/openapi-bundler` package.

`@kbn/openapi-bundler` SHOULD NOT replace OpenAPI linters since it doesn't perform thorough linting. It's good if we continue adopting `spectral-cli` for linting purposes.

.buildkite/scripts/steps/checks.sh

@@ -15,3 +15,4 @@ fi
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/openapi_bundling/security_solution_openapi_bundling.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
+.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Merge Kibana OpenAPI specs
+
+(cd oas_docs && make api-docs && make api-docs-lint)

oas_docs/.spectral.yaml

@@ -1,6 +1,6 @@
-extends: ["spectral:oas"]
+extends: ['spectral:oas']
 rules:
-# Built-in rules
+  # Built-in rules
   # Descriptions
   oas3-parameter-description: warn
   oas2-parameter-description: warn
@@ -13,10 +13,10 @@ rules:
   oas3-valid-media-example: false
   oas3-valid-schema-example: false
   oas2-valid-media-example: false
-  # Operations 
-  operation-operationId: warn
-  operation-operationId-unique: warn
-  operation-operationId-valid-in-url: warn
+  # Operations
+  operation-operationId: error
+  operation-operationId-unique: error
+  operation-operationId-valid-in-url: error
   operation-tag-defined: warn
   operation-tags: warn
   # Responses
@@ -25,28 +25,30 @@ rules:
   oas3-schema: error
   oas2-schema: error
   array-items: false
+  # Bump.sh handles $ref siblings. Documentation wise it's convenient to have properties like descriptions next to $ref.
+  no-$ref-siblings: off
   # Tags
   openapi-tags: warn
   openapi-tags-alphabetical: info
   # Turn off some built-in rules
   operation-description: false
   operation-singular-tag: false
-# Custom rules
+  # Custom rules
   # Descriptions
   avoid-problematic-words:
     description: Ban certain words from descriptions
-    message: "Use appropriate replacements for problematic terms"
+    message: 'Use appropriate replacements for problematic terms'
     severity: warn
-    given: "$..*.description"
+    given: '$..*.description'
     then:
       function: pattern
       functionOptions:
         notMatch: /(blacklist|whitelist|execute|kill)/i
   # Examples
   operation-success-examples:
-    formats: ["oas3_1"]
+    formats: ['oas3_1']
     description: Response code 200 should have at least one example.
-    message: "Each response body should have a realistic example. It must not contain any sensitive or confidential data."
+    message: 'Each response body should have a realistic example. It must not contain any sensitive or confidential data.'
     severity: info
     given: $.paths[*][*].responses.[200].content.[application/json]
     then:
@@ -55,7 +57,7 @@ rules:
   # Extensions
   internal-extension:
     description: Operations should not have x-internal extension.
-    message: "Do not publish x-internal operations"
+    message: 'Do not publish x-internal operations'
     severity: error
     given: $.paths[*][*]
     then:
@@ -64,16 +66,16 @@ rules:
   # Operations
   operation-summary:
     description: Operations should have summaries.
-    message: "Each operation should have a summary"
+    message: 'Each operation should have a summary'
     severity: error
     recommended: true
     given: $.paths[*][*]
     then:
       field: summary
-      function: defined   
+      function: defined
   operation-summary-length:
     description: Operation summary should be between 5 and 45 characters
-    given: "$.paths[*][*]"
+    given: '$.paths[*][*]'
     then:
       field: summary
       function: length
@@ -83,14 +85,14 @@ rules:
     severity: warn
   simple-verbs-in-summary:
     given:
-      - "$.paths[*][*].summary"
+      - '$.paths[*][*].summary'
     then:
       function: pattern
       functionOptions:
-        notMatch: "Retrieve|Return|List *"
+        notMatch: 'Retrieve|Return|List *'
     severity: warn
     description: Summaries should use common verbs.
-    message: "Summaries should use common verbs like Get, Update, Delete whenever possible"
+    message: 'Summaries should use common verbs like Get, Update, Delete whenever possible'
   # NOTE: This one hiccups on acronyms so perhaps too noisy
   # docs-operation-summary-sentence-case:
   #   description: Operation summary should be sentence cased
@@ -101,4 +103,3 @@ rules:
   #     functionOptions:
   #       match: /^[A-Z]+[^A-Z]+$/
   #   severity: warn
-

oas_docs/kibana.info.yaml

@@ -7,15 +7,15 @@ info:
     Each request that you make happens in isolation from other calls and must include all of the necessary information for Kibana to fulfill the
     request.
     API requests return JSON output, which is a format that is machine-readable and works well for automation.
-    
+
     To interact with Kibana APIs, use the following operations:
-    
+
     - GET: Fetches the information.
     - PATCH: Applies partial modifications to the existing information.
     - POST: Adds new information.
     - PUT: Updates the existing information.
     - DELETE: Removes the information.
-    
+
     You can prepend any Kibana API endpoint with `kbn:` and run the request in **Dev Tools → Console**.
     For example:
 
@@ -24,26 +24,26 @@ info:
     ```
 
     For more information about the console, refer to [Run API requests](https://www.elastic.co/guide/en/kibana/current/console-kibana.html).
-  version: "1.0.2"
+  version: '1.0.2'
   license:
     name: Elastic License 2.0
     url: https://www.elastic.co/licensing/elastic-license
-  contact: 
+  contact:
     name: Kibana Team
-# servers:
-#   - url: https://{kibana_url}
-#     variables:
-#       kibana_url:
-#         default: localhost:5601
-# security:
-#   - apiKeyAuth: []
-# components:
-#   securitySchemes:
-#     apiKeyAuth:
-#       type: apiKey
-#       in: header
-#       name: Authorization
-#       description: >
-#         These APIs use key-based authentication.
-#         You must create an API key and use the encoded value in the request header.
-#         For example: `Authorization: ApiKey base64AccessApiKey`
+servers:
+  - url: https://{kibana_url}
+    variables:
+      kibana_url:
+        default: localhost:5601
+security:
+  - apiKeyAuth: []
+components:
+  securitySchemes:
+    apiKeyAuth:
+      type: apiKey
+      in: header
+      name: Authorization
+      description: >
+        These APIs use key-based authentication.
+        You must create an API key and use the encoded value in the request header.
+        For example: `Authorization: ApiKey base64AccessApiKey`

oas_docs/makefile

@@ -14,7 +14,12 @@
 # permission is obtained from Elasticsearch B.V.
 
 .PHONY: api-docs
-api-docs: ## Generate kibana.serverless.yaml and kibana.yaml
+api-docs: ## Generate Serverless and ESS Kibana OpenAPI bundles with kbn-openapi-bundler
+	@node scripts/merge_serverless_oas.js
+	@node scripts/merge_ess_oas.js
+
+.PHONY: api-docs-redocly
+api-docs-redocly: ## Generate kibana.serverless.yaml and kibana.yaml with Redocly CLI
 	@npx @redocly/cli join "kibana.info.serverless.yaml" "../x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml" "../x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml" "../src/plugins/data_views/docs/openapi/bundled.yaml" "../x-pack/plugins/ml/common/openapi/ml_apis_serverless.yaml" "../packages/core/saved-objects/docs/openapi/bundled_serverless.yaml" "../x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml"  "bundle.serverless.json" -o "output/kibana.serverless.yaml" --prefix-components-with-info-prop title
 	@npx @redocly/cli join "kibana.info.yaml" "../x-pack/plugins/alerting/docs/openapi/bundled.yaml" "../x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml"  "../x-pack/plugins/cases/docs/openapi/bundled.yaml" "../x-pack/plugins/actions/docs/openapi/bundled.yaml" "../src/plugins/data_views/docs/openapi/bundled.yaml" "../x-pack/plugins/ml/common/openapi/ml_apis.yaml" "../packages/core/saved-objects/docs/openapi/bundled.yaml"  "bundle.json" -o "output/kibana.yaml" --prefix-components-with-info-prop title
 

oas_docs/scripts/merge_ess_oas.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../../src/setup_node_env');
+const { merge } = require('@kbn/openapi-bundler');
+const { REPO_ROOT } = require('@kbn/repo-info');
+
+(async () => {
+  await merge({
+    sourceGlobs: [
+      `${REPO_ROOT}/oas_docs/bundle.json`,
+      `${REPO_ROOT}/x-pack/plugins/actions/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/src/plugins/data_views/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/ml/common/openapi/ml_apis.yaml`,
+      `${REPO_ROOT}/packages/core/saved-objects/docs/openapi/bundled.yaml`,
+
+      // Observability Solution
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml`,
+
+      // Security solution
+      `${REPO_ROOT}/x-pack/plugins/security_solution/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-lists-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-exceptions-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/packages/kbn-elastic-assistant-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/osquery/docs/openapi/ess/*.schema.yaml`,
+    ],
+    outputFilePath: `${REPO_ROOT}/oas_docs/output/kibana.yaml`,
+    options: {
+      prototypeDocument: `${REPO_ROOT}/oas_docs/kibana.info.yaml`,
+    },
+  });
+})();

oas_docs/scripts/merge_serverless_oas.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../../src/setup_node_env');
+const { merge } = require('@kbn/openapi-bundler');
+const { REPO_ROOT } = require('@kbn/repo-info');
+
+(async () => {
+  await merge({
+    sourceGlobs: [
+      `${REPO_ROOT}/oas_docs/bundle.serverless.json`,
+      `${REPO_ROOT}/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml`,
+      `${REPO_ROOT}/src/plugins/data_views/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/ml/common/openapi/ml_apis_serverless.yaml`,
+      `${REPO_ROOT}/packages/core/saved-objects/docs/openapi/bundled_serverless.yaml`,
+
+      // Observability Solution
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml`,
+
+      // Security solution
+      `${REPO_ROOT}/x-pack/plugins/security_solution/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-exceptions-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/packages/kbn-elastic-assistant-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/osquery/docs/openapi/serverless/*.schema.yaml`,
+    ],
+    outputFilePath: `${REPO_ROOT}/oas_docs/output/kibana.serverless.yaml`,
+    options: {
+      prototypeDocument: `${REPO_ROOT}/oas_docs/kibana.info.yaml`,
+    },
+  });
+})();

packages/kbn-openapi-bundler/src/bundler/merge_documents/enrich_with_version_mime_param.ts

@@ -12,9 +12,12 @@ import { isRefNode } from '../process_document';
 import { getOasDocumentVersion } from '../../utils/get_oas_document_version';
 import { KNOWN_HTTP_METHODS } from './http_methods';
 
+const DEFAULT_API_VERSION = '2023-10-31';
+const VERSION_REGEX = /\d{4}-\d{2}-\d{2}/;
+
 export function enrichWithVersionMimeParam(resolvedDocuments: ResolvedDocument[]): void {
   for (const resolvedDocument of resolvedDocuments) {
-    const version = getOasDocumentVersion(resolvedDocument);
+    const version = extractApiVersion(resolvedDocument);
     const paths = resolvedDocument.document.paths as OpenAPIV3.PathsObject;
 
     for (const path of Object.keys(paths ?? {})) {
@@ -80,3 +83,13 @@ function enrichContentWithVersion(
     delete content[mimeType];
   }
 }
+
+function extractApiVersion(resolvedDocument: ResolvedDocument): string {
+  const version = getOasDocumentVersion(resolvedDocument);
+
+  if (!VERSION_REGEX.test(version)) {
+    return DEFAULT_API_VERSION;
+  }
+
+  return version < DEFAULT_API_VERSION ? DEFAULT_API_VERSION : version;
+}

x-pack/plugins/security_solution/common/api/detection_engine/rule_management/bulk_crud/bulk_delete_rules/bulk_delete_rules_route.schema.yaml

@@ -60,6 +60,7 @@ paths:
       x-codegen-enabled: true
       operationId: BulkDeleteRulesPost
       deprecated: true
+      summary: Delete multiple detection rules
       description: Deletes multiple rules.
       tags:
         - Bulk API

x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml

@@ -474,6 +474,7 @@ paths:
               schema:
                 $ref: '#/components/schemas/SiemErrorResponse'
           description: Internal server error response
+      summary: Delete multiple detection rules
       tags:
         - Security Solution Detections API
         - Bulk API