[BK] Migrate batch 6 (api-docs, fleet-packages, secsol-qg-api) (#180784)

## Summary
Migrates 3 pipelines:
- kibana / api-docs / daily
[kibana-api-docs.yml](https://buildkite.com/elastic/kibana-api-docs-daily)
- kibana / serverless / security-solution-quality-gate / api-integration
[kibana-serverless-security-solution-quality-gate-api-integration.yml](https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-api-integration)
- kibana / fleet-packages
[kibana-fleet-packages-daily.yml](https://buildkite.com/elastic/kibana-fleet-packages)

Verification:
 - [x] locally tested the RREs for validity
 - [x] pipelines tested through the migration staging pipeline:
- [x] API-docs -
https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/80
✅
- [x] serverless security solution api-integration -
https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/48#018ef1ed-853d-4649-b008-3a38b9f97923
✅
- [x] fleet packages -
https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/49
✅

Part of: https://github.com/elastic/kibana-operations/issues/79

.buildkite/pipeline-resource-definitions/kibana-api-docs.yml

@@ -0,0 +1,51 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-api-docs-daily
+  description: Builds api_docs daily and creates a PR with the changes
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-api-docs-daily'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / api-docs / daily
+      description: Builds api_docs daily and creates a PR with the changes
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      cancel_intermediate_builds: true
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/build_api_docs.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 0 * * * America/New_York
+          message: Daily build
+          branch: main

.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-fleet-packages
+  description: Installs all fleet packages into Kibana to ensure the install step works
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-fleet-packages'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / fleet-packages
+      description: Installs all fleet packages into Kibana to ensure the install step works
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#fleet-notifications'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/fleet/packages_daily.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        publish_commit_status: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        fleet:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Single user daily test:
+          cronline: 0 9 * * * America/New_York
+          message: Single user daily test
+          env: {}
+          branch: main

.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml

@@ -0,0 +1,46 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-serverless-secsol-qg-api-integration
+  description: Runs the serverless security solution api integration tests for the Quality Gate
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-api-integration'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / serverless / security-solution-quality-gate / api-integration
+      description: Runs the serverless security solution api integration tests for the Quality Gate
+    spec:
+      env: {}
+      allow_rebuilds: true
+      branch_configuration: ''
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/security_solution/api_integration.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        security-engineering-productivity:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -7,20 +7,23 @@ metadata:
 spec:
   type: url
   targets:
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
-    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml'
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml

.buildkite/pipelines/build_api_docs.yml

@@ -1,13 +1,46 @@
 env:
   PUBLISH_API_DOCS_CHANGES: 'true'
 steps:
-  - command: .buildkite/scripts/steps/build_api_docs.sh
+  - command: .buildkite/scripts/steps/api_docs/type_check_with_status.sh
+    label: Run scripts/type_check
+    key: type_check
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    timeout_in_minutes: 50
+
+  - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     key: build_api_docs
-    timeout_in_minutes: 60
+    timeout_in_minutes: 50
     retry:
       automatic:
         - exit_status: '-1'
           limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/api_docs/publish_api_docs.sh
+    label: 'Publish API Docs'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    key: publish_api_docs
+    timeout_in_minutes: 50
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+    if: env('PUBLISH_API_DOCS_CHANGES') == 'true'

.buildkite/pipelines/fleet/packages_daily.yml

@@ -3,14 +3,21 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/fleet/install_all_packages.sh
     label: Install All Packages
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      machineType: n2-standard-2
     env:
       # ensure that the FTR logs all output for these tests
       DISABLE_CI_LOG_OUTPUT_CAPTURE: 'true'
@@ -24,4 +31,7 @@ steps:
   - label: Post-Build
     command: .buildkite/scripts/lifecycle/post_build.sh
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2

.buildkite/pipelines/pull_request/base.yml

@@ -91,8 +91,8 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/build_api_docs.sh
+  - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
-    label: 'Check Types and Build API Docs'
+    label: 'Build API Docs'
     agents:
       queue: n2-4-spot
     key: build_api_docs

.buildkite/pipelines/security_solution/api_integration.yml

@@ -3,7 +3,11 @@ steps:
     label: 'Upload runtime info'
     key: upload_runtime_info
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 300
     retry:
       automatic:
@@ -17,7 +21,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_workflows:qa:serverless
         key: exception_workflows:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -28,7 +36,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_date_numeric_types:qa:serverless
         key: exception_operators_date_numeric_types:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -39,7 +51,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_keyword:qa:serverless
         key: exception_operators_keyword:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -50,7 +66,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_ips:qa:serverless
         key: exception_operators_ips:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -61,7 +81,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_long:qa:serverless
         key: exception_operators_long:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -72,7 +96,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_text:qa:serverless
         key: exception_operators_text:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -83,7 +111,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh alerts:qa:serverless
         key: alerts:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -94,7 +126,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh alerts:essentials:qa:serverless
         key: alerts:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -105,7 +141,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh actions:qa:serverless
         key: actions:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -116,7 +156,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh genai:qa:serverless
         key: genai:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -127,7 +171,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_execution_logic:qa:serverless
         key: rule_execution_logic:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -138,7 +186,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_patch:qa:serverless
         key: rule_patch:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -149,7 +201,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_patch:essentials:qa:serverless
         key: rule_patch:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -160,7 +216,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_update:qa:serverless
         key: rule_update:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -171,7 +231,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_update:essentials:qa:serverless
         key: rule_update:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -182,7 +246,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rules_management:essentials:qa:serverless
         key: rules_management:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -193,7 +261,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_management:qa:serverless
         key: prebuilt_rules_management:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -204,7 +276,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_bundled_prebuilt_rules_package:qa:serverless
         key: prebuilt_rules_bundled_prebuilt_rules_package:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -215,7 +291,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_large_prebuilt_rules_package:qa:serverless
         key: prebuilt_rules_large_prebuilt_rules_package:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -226,7 +306,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_update_prebuilt_rules_package:qa:serverless
         key: prebuilt_rules_update_prebuilt_rules_package:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -237,7 +321,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_bulk_actions:qa:serverless
         key: rule_bulk_actions:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -248,7 +336,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_read:qa:serverless
         key: rule_read:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -259,7 +351,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_import_export:essentials:qa:serverless
         key: rule_import_export:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -270,7 +366,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_import_export:qa:serverless
         key: rule_import_export:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -282,7 +382,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_management:qa:serverless
         key: rule_management:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -293,7 +397,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_read:essentials:qa:serverless
         key: rule_read:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -304,7 +412,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_creation:qa:serverless
         key: rule_creation:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -315,7 +427,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_creation:essentials:qa:serverless
         key: rule_creation:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -326,7 +442,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_delete:qa:serverless
         key: rule_delete:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -337,7 +457,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_delete:essentials:qa:serverless
         key: rule_delete:essentials:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -348,7 +472,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_lists_items:qa:serverless
         key: exception_lists_items:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -359,7 +487,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh lists_items:qa:serverless
         key: lists_items:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -370,7 +502,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh user_roles:qa:serverless
         key: user_roles:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -381,7 +517,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh telemetry:qa:serverless
         key: telemetry:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -391,7 +531,11 @@ steps:
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh entity_analytics:qa:serverless
         key: entity_analytics:qa:serverless
         agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/scripts/steps/api_docs/build_api_docs.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Build API Docs"
+node --max-old-space-size=12000 scripts/build_api_docs
+
+if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then
+  echo "--- Store API Docs changes in Buildkite"
+
+  git diff > api_docs_changes.diff
+  buildkite-agent artifact upload api_docs_changes.diff
+
+  echo "API Docs changes uploaded"
+fi

.buildkite/scripts/steps/api_docs/type_check_with_status.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+
+echo --- Check Types
+set +e
+node scripts/type_check
+TYPE_CHECK_RESULT=$?
+echo Type check exited with status $TYPE_CHECK_RESULT
+set -e
+
+if [[ $TYPE_CHECK_RESULT -ne 0 ]]; then
+  echo "Type check failed - setting status to failure"
+  buildkite-agent meta-data set "type_check_status" "failure"
+else
+  echo "Type check passed - setting status to success"
+  buildkite-agent meta-data set "type_check_status" "success"
+fi
+
+exit $TYPE_CHECK_RESULT

.buildkite/scripts/steps/build_api_docs.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-.buildkite/scripts/bootstrap.sh
-
-echo "--- Run scripts/type_check to ensure that all build available"
-node scripts/type_check
-
-echo "--- Build API Docs"
-node --max-old-space-size=12000 scripts/build_api_docs
-
-if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then
-  echo "--- Publish API Docs"
-
-  git config --global user.name kibanamachine
-  git config --global user.email '42973632+kibanamachine@users.noreply.github.com'
-
-  branch="api_docs_$(date +%F_%H-%M-%S)"
-  git checkout -b "$branch"
-  git add ./*.docnav.json
-  git add api_docs
-  git commit -m "[api-docs] Daily api_docs build"
-
-  git remote add kibanamachine https://github.com/kibanamachine/kibana.git
-  git push -u kibanamachine "$branch"
-  prUrl=$(gh pr create --repo elastic/kibana --title "[api-docs] $(date +%F) Daily api_docs build" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "docs")
-  echo "Opened PR: $prUrl"
-  gh pr merge --repo elastic/kibana --auto --squash "$prUrl"
-
-  GH_TOKEN="$KIBANA_CI_GITHUB_TOKEN" gh pr review --repo elastic/kibana --approve -b "Automated review from $BUILDKITE_BUILD_URL" "$prUrl"
-fi