[BK] Migrate Batch 4 (ES verification) (#180346)

## Summary Creates new Buildkite RRE definitions for batch 4 (ES snapshot verification + ES serverless image verification). Updates agent targeting rules in affected pipeline implementations. - [x] RREs validated with `docker.elastic.co/ci-agent-images/pipelib`'s scripts locally - [x] Tested pipelines through the pipeline staging job - [x] Serverless suite: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/22 - [x] ES Snapshot build: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/26 - [x] ES Snapshot verify: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/27 (basically started running, but failed due to test failures - doesn't seem to be related to the infra change) - [x] ES Snapshot promote: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/28 Part of: https://github.com/elastic/kibana-operations/issues/79 Migration of: - kibana / elasticsearch serverless verify and promote [kibana-es-snapshots.yml](https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote) - kibana / elasticsearch snapshot build [kibana-es-snapshots.yml](https://buildkite.com/elastic/kibana-elasticsearch-snapshot-build) - kibana / elasticsearch snapshot promote [kibana-es-snapshots.yml](https://buildkite.com/elastic/kibana-elasticsearch-snapshot-promote) - kibana / elasticsearch snapshot verify [kibana-es-snapshots.yml](https://buildkite.com/elastic/kibana-elasticsearch-snapshot-verify) --------- Co-authored-by: Jon <jon@budzenski.me>
2025-04-23 09:19:04 -04:00 · 2024-04-10 11:00:32 +02:00 · 2024-04-10 11:00:32 +02:00 · 731174bcf8
commit 731174bcf8
parent 92ca1b65a2
9 changed files with 321 additions and 38 deletions
--- a/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
+++ b/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-elasticsearch-serverless-verify-and-promote
+  description: Verify & promote ElasticSearch Serverless images that pass Kibana's test suite
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / elasticsearch serverless verify and promote
+      description: Verify & promote ElasticSearch Serverless images that pass Kibana's test suite
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ES_SERVERLESS_IMAGE: latest
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          env:
+            PUBLISH_DOCKER_TAG: 'true'
+          branch: main
--- a/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
+++ b/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
@ -0,0 +1,153 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-elasticsearch-snapshot-build
+  description: Build new Elasticsearch snapshots for use by kbn-es / FTR
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-elasticsearch-snapshot-build'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / elasticsearch snapshot build
+      description: Build new Elasticsearch snapshots for use by kbn-es / FTR
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (main):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: main
+        Daily build (8.13):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: '8.13'
+        Daily build (7.17):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: '7.17'
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-elasticsearch-snapshot-promote
+  description: Promote Elasticsearch snapshots for use by kbn-es / FTR
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-elasticsearch-snapshot-promote'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / elasticsearch snapshot promote
+      description: Promote Elasticsearch snapshots for use by kbn-es / FTR
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-elasticsearch-snapshot-verify
+  description: Verify Elasticsearch snapshots for use by kbn-es / FTR
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-elasticsearch-snapshot-verify'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / elasticsearch snapshot verify
+      description: Verify Elasticsearch snapshots for use by kbn-es / FTR
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
--- a/.buildkite/pipeline-resource-definitions/locations.yml
+++ b/.buildkite/pipeline-resource-definitions/locations.yml
@ -6,16 +6,18 @@ metadata:
 spec:
  type: url
  targets:
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml'
--- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
+++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@ -9,7 +9,10 @@
 # BUILDKITE_COMMIT: the commit hash of the kibana branch to test

 agents:
-  queue: kibana-default
+  image: family/kibana-ubuntu-2004
+  imageProject: elastic-images-qa
+  provider: gcp
+  machineType: n2-standard-2

 steps:
  - label: "Annotate runtime parameters"
@ -25,12 +28,18 @@ steps:
        key: pre-build
        timeout_in_minutes: 10
        agents:
-          queue: kibana-default
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-2

      - label: "Build Kibana Distribution and Plugins"
        command: .buildkite/scripts/steps/build_kibana.sh
        agents:
-          queue: n2-16-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-16
        key: build
        depends_on: pre-build
        timeout_in_minutes: 60
@ -41,8 +50,6 @@ steps:

      - label: "Pick Test Group Run Order (FTR + Integration)"
        command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
-        agents:
-          queue: kibana-default
        depends_on: build
        timeout_in_minutes: 10
        env:
@ -60,7 +67,11 @@ steps:
        label: 'Serverless Entity Analytics - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 2
@ -73,7 +84,11 @@ steps:
        label: 'Serverless Explore - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 4
@ -86,7 +101,11 @@ steps:
        label: 'Serverless Investigations - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 8
@ -99,7 +118,11 @@ steps:
        label: 'Serverless Rule Management - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 5
@ -112,7 +135,11 @@ steps:
        label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 2
@ -125,7 +152,11 @@ steps:
        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 5
@ -138,7 +169,11 @@ steps:
        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 2
@ -151,7 +186,11 @@ steps:
        label: 'Serverless AI Assistant - Security Solution Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 1
@ -164,7 +203,13 @@ steps:
        label: 'Defend Workflows Cypress Tests on Serverless'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-virt
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 12
@ -177,7 +222,11 @@ steps:
        label: 'Serverless Osquery Cypress Tests'
        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
        agents:
-          queue: n2-4-spot
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          provider: gcp
+          machineType: n2-standard-4
+          preemptible: true
        depends_on: build
        timeout_in_minutes: 60
        parallelism: 7
@ -196,5 +245,3 @@ steps:
  - label: 'Post-Build'
    command: .buildkite/scripts/lifecycle/post_build.sh
    timeout_in_minutes: 10
-    agents:
-      queue: kibana-default
--- a/.buildkite/pipelines/es_snapshots/build.yml
+++ b/.buildkite/pipelines/es_snapshots/build.yml
@ -3,4 +3,9 @@ steps:
    label: Build ES Snapshot
    timeout_in_minutes: 30
    agents:
-      queue: c2-8
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-8
--- a/.buildkite/pipelines/es_snapshots/promote.yml
+++ b/.buildkite/pipelines/es_snapshots/promote.yml
@ -11,4 +11,7 @@ steps:
  - label: Promote Snapshot
    command: .buildkite/scripts/steps/es_snapshots/promote.sh
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
--- a/.buildkite/pipelines/es_snapshots/verify.yml
+++ b/.buildkite/pipelines/es_snapshots/verify.yml
@ -15,14 +15,21 @@ steps:
    label: Pre-Build
    timeout_in_minutes: 10
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2

  - wait

  - command: .buildkite/scripts/steps/build_kibana.sh
    label: Build Kibana Distribution and Plugins
    agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
    key: build
    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
    timeout_in_minutes: 60
@ -34,7 +41,10 @@ steps:
  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
    label: 'Pick Test Group Run Order'
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
    timeout_in_minutes: 10
    env:
      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@ -50,7 +60,10 @@ steps:
    label: Trigger promotion
    timeout_in_minutes: 10
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
    depends_on:
      - ftr-configs
      - jest-integration
@ -62,4 +75,7 @@ steps:
    label: Post-Build
    timeout_in_minutes: 10
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
--- a/.buildkite/pipelines/upload_pipeline.yml
+++ b/.buildkite/pipelines/upload_pipeline.yml
@ -2,4 +2,4 @@

 steps:
  - label: Upload tested pipeline
-    command: buildkite-agent pipeline upload .buildkite/pipelines/pipeline_to_test.yml
+    command: buildkite-agent pipeline upload ${TESTED_PIPELINE_PATH:-.buildkite/pipelines/pipeline_to_test.yml}
--- a/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts
+++ b/src/dev/buildkite_migration/rewrite_buildkite_agent_rules.ts
@ -40,6 +40,7 @@ interface KBAgentDef {
  spot?: boolean;
  zones?: string[];
  nestedVirtualization?: boolean;
+  serviceAccount?: string;
 }
 type KibanaBuildkiteAgentLookup = Record<string, KBAgentDef>;

@ -216,8 +217,10 @@ function getFullAgentTargetingRule(queue: string): GobldGCPConfig {
    diskType: agent.diskType,
    enableNestedVirtualization: agent.nestedVirtualization,
    localSsds: agent.localSsds,
+    localSsdInterface: !!agent.localSsds ? 'nvme' : undefined,
    machineType: agent.machineType,
    preemptible: agent.spot,
+    serviceAccount: agent.serviceAccount,
  });
 }