[BK] Migrate batch 1 (Artifact builds) (#182582)

## Summary
Migrates batch 1 - artifact builds. The upload aspect wasn't tested,
because it's programmed only to run from `main`, and we didn't want to
interfere with the ongoing releases. This can be tested after the merge.

Verification:
- [x] RREs tested locally
- [x] kibana / artifacts trigger
(https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/87)
- [x] kibana / artifacts container image
(https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/86)
- [x] kibana / artifacts snapshot
(https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/88)
- [x] kibana / artifacts staging
(https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/88)
- [x] 8.13 / 8.14 / 7.17 verification (only a few jobs need to work
here)

Originals:
- kibana / artifacts trigger
[kibana-artifacts-trigger.yml](https://buildkite.com/elastic/kibana-artifacts-trigger)
- kibana / artifacts container image
[kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-container-image)
- kibana / artifacts snapshot
[kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-snapshot)
- kibana / artifacts staging
[kibana-artifacts.yml](https://buildkite.com/elastic/kibana-artifacts-staging)

Backports:
 - https://github.com/elastic/kibana/pull/182781
 - https://github.com/elastic/kibana/pull/182780
 
The backports don't need to have the pipeline resource definition files,
however, we forked 8.14 off from main, where we already had the
resources. I'll remove all the unnecessary resource defs from the legacy
branches, once we finalize the state (simply to save a little
inconvenience on future backports.)

.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml

@@ -0,0 +1,44 @@
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-container-image
+  description: Kibana container image artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-container-image
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts container image
+      description: Kibana container image artifact builds
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-serverless-test-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts_container_image.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml

@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-snapshot
+  description: Kibana snapshot artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-snapshot
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts snapshot
+      description: Kibana snapshot artifact builds
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main 8.14 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (main):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: main
+        Daily build (8.13):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.13'
+        Daily build (8.14):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.14'
+        Daily build (7.17):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml

@@ -0,0 +1,58 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-staging
+  description: Kibana staging artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-staging
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts staging
+      description: Kibana staging artifact builds
+    spec:
+      env:
+        RELEASE_BUILD: 'true'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: 7.17 8.13 8.14
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (8.13):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.13'
+        Daily build (8.14):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.14'
+        Daily build (7.17):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml

@@ -0,0 +1,55 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-trigger
+  description: Kibana artifact trigger
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-artifacts-trigger'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts trigger
+      description: Kibana artifact trigger
+    spec:
+      env:
+        RELEASE_BUILD: 'true'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: '8.13 8.14'
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts_trigger.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (8.13):
+          cronline: 0 */2 * * * America/New_York
+          message: Daily build
+          branch: '8.13'
+        Daily build (8.14):
+          cronline: 0 */2 * * * America/New_York
+          message: Daily build
+          branch: '8.14'

.buildkite/pipeline-resource-definitions/locations.yml

@@ -9,6 +9,10 @@ spec:
   targets:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-apis-capacity-testing-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

.buildkite/pipelines/artifacts.yml

@@ -2,7 +2,10 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/build.sh
     label: Build Kibana Artifacts
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: c2-standard-16
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -14,7 +17,13 @@ steps:
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -24,7 +33,13 @@ steps:
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -34,7 +49,13 @@ steps:
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -44,7 +65,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -54,7 +80,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -64,7 +95,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -77,7 +113,12 @@ steps:
       - exit_status: 255
       - exit_status: -1
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 60
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
@@ -99,5 +140,10 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/publish.sh
     label: 'Publish Kibana Artifacts'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30

.buildkite/pipelines/artifacts_container_image.yml

@@ -2,5 +2,8 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
     label: Build serverless container images
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
     timeout_in_minutes: 60

.buildkite/pipelines/artifacts_trigger.yml

@@ -2,5 +2,8 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/trigger.sh
     label: Trigger artifacts build
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10