[BK] Migrate buildkite batch 5 (unsupported ftr / flaky test runner) (#180403)

## Summary Validation: - [x] RREs checked locally - [x] Pipelines staged - [x] Unsupported FTRs: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/30 (expecting similar errors with the same parameterization of [this](https://buildkite.com/elastic/kibana-on-merge-unsupported-ftrs/builds/14597#018ec7ff-2ab8-4cd2-a533-00aec6287e88)) - [x] Flaky test runner: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/35 - [x] Considerations for - [x] 7.17: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/46 (PR: https://github.com/elastic/kibana/pull/180575) - [x] 8.13: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/43 (PR: https://github.com/elastic/kibana/pull/180602) Backporting should be done manually, as the auto-backports will easily fail. Part of: https://github.com/elastic/kibana-operations/issues/79 Migrates: - kibana / on merge unsupported ftrs [kibana-on-merge-unsupported-ftrs.yml](https://buildkite.com/elastic/kibana-on-merge-unsupported-ftrs) - kibana / flaky-test-suite-runner [kibana-flaky.yml](https://buildkite.com/elastic/kibana-flaky-test-suite-runner)
2025-04-24 17:59:23 -04:00 · 2024-04-17 11:13:15 +02:00 · 2024-04-17 11:13:15 +02:00 · 507986e9db
commit 507986e9db
parent 0840c64d67
7 changed files with 173 additions and 17 deletions
--- a/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
+++ b/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
@ -0,0 +1,40 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-flaky-test-suite-runner
+  description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-flaky-test-suite-runner'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / flaky-test-suite-runner
+      description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+    spec:
+      allow_rebuilds: false
+      default_branch: refs/pull/INSERT_PR_NUMBER/head
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/flaky_tests/pipeline.sh
+      skip_intermediate_builds: false
+      provider_settings:
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
--- a/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
+++ b/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
@ -0,0 +1,46 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-on-merge-unsupported-ftrs
+  description: Runs unsupported ftr tests for each commit of Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-on-merge-unsupported-ftrs'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / on merge unsupported ftrs
+      description: Runs unsupported ftr tests for each commit of Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
--- a/.buildkite/pipeline-resource-definitions/locations.yml
+++ b/.buildkite/pipeline-resource-definitions/locations.yml
@ -1,3 +1,4 @@
+# * This file is auto-updated by running fix-location-collection.ts * #
 apiVersion: backstage.io/v1alpha1
 kind: Location
 metadata:
@ -10,7 +11,9 @@ spec:
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml'
    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml'
--- a/.buildkite/pipelines/flaky_tests/pipeline.sh
+++ b/.buildkite/pipelines/flaky_tests/pipeline.sh
@ -5,4 +5,4 @@ set -euo pipefail
 UUID="$(cat /proc/sys/kernel/random/uuid)"
 export UUID

-ts-node .buildkite/pipelines/flaky_tests/pipeline.ts | buildkite-agent pipeline upload
+ts-node .buildkite/pipelines/flaky_tests/pipeline.ts
--- a/.buildkite/pipelines/flaky_tests/pipeline.ts
+++ b/.buildkite/pipelines/flaky_tests/pipeline.ts
@ -31,6 +31,34 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;

+// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
+/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
+const getAgentRule = (queueName: string = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 function getTestSuitesFromJson(json: string) {
  const fail = (errorMsg: string) => {
    console.error('+++ Invalid test config provided');
@ -49,8 +77,10 @@ function getTestSuitesFromJson(json: string) {
    fail(`JSON test config must be an array`);
  }

-  /** @type {Array<{ type: 'group', key: string; count: number } | { type: 'ftrConfig', ftrConfig: string; count: number }>} */
-  const testSuites = [];
+  const testSuites: Array<
+    | { type: 'group'; key: string; count: number }
+    | { type: 'ftrConfig'; ftrConfig: string; count: number }
+  > = [];
  for (const item of parsed) {
    if (typeof item !== 'object' || item === null) {
      fail(`testSuites must be objects`);
@ -73,6 +103,7 @@ function getTestSuitesFromJson(json: string) {
      }

      testSuites.push({
+        type: 'ftrConfig',
        ftrConfig,
        count,
      });
@ -84,6 +115,7 @@ function getTestSuitesFromJson(json: string) {
      fail(`testSuite.key must be a string`);
    }
    testSuites.push({
+      type: 'group',
      key,
      count,
    });
@ -117,7 +149,7 @@ const pipeline = {
 steps.push({
  command: '.buildkite/scripts/steps/build_kibana.sh',
  label: 'Build Kibana Distribution and Plugins',
-  agents: { queue: 'c2-8' },
+  agents: getAgentRule('c2-8'),
  key: 'build',
  if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@ -127,7 +159,7 @@ for (const testSuite of testSuites) {
    continue;
  }

-  if (testSuite.ftrConfig) {
+  if (testSuite.type === 'ftrConfig') {
    steps.push({
      command: `.buildkite/scripts/steps/test/ftr_configs.sh`,
      env: {
@ -138,9 +170,7 @@ for (const testSuite of testSuites) {
      concurrency,
      concurrency_group: process.env.UUID,
      concurrency_method: 'eager',
-      agents: {
-        queue: 'n2-4-spot-2',
-      },
+      agents: getAgentRule('n2-4-spot'),
      depends_on: 'build',
      timeout_in_minutes: 150,
      cancel_on_build_failing: true,
@ -164,7 +194,7 @@ for (const testSuite of testSuites) {
      steps.push({
        command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
        label: group.name,
-        agents: { queue: agentQueue },
+        agents: getAgentRule(agentQueue),
        depends_on: 'build',
        timeout_in_minutes: 150,
        parallelism: testSuite.count,
--- a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml
+++ b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml
@ -3,7 +3,10 @@ steps:
    label: Pre-Build
    timeout_in_minutes: 10
    agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
    retry:
      automatic:
        - exit_status: '*'
@ -12,7 +15,11 @@ steps:
  - command: .buildkite/scripts/steps/build_kibana.sh
    label: Build Kibana Distribution and Plugins
    agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
    key: build
    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
    timeout_in_minutes: 60
@ -24,7 +31,11 @@ steps:
  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
    label: 'APM Cypress Tests'
    agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
    depends_on: build
    timeout_in_minutes: 120
    retry:
@ -37,7 +48,11 @@ steps:
  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
    label: 'Profiling Cypress Tests'
    agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
    depends_on: build
    timeout_in_minutes: 120
    retry:
@ -50,7 +65,11 @@ steps:
  - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
    label: 'Synthetics @elastic/synthetics Tests'
    agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
    depends_on: build
    timeout_in_minutes: 120
    artifact_paths:
@ -65,7 +84,13 @@ steps:
  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
    label: 'Defend Workflows Cypress Tests'
    agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsdInterface: nvme
+      localSsds: 1
+      machineType: n2-standard-4
    depends_on: build
    timeout_in_minutes: 60
    parallelism: 20
@ -77,7 +102,13 @@ steps:
  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
    label: 'Defend Workflows Cypress Tests on Serverless'
    agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsdInterface: nvme
+      localSsds: 1
+      machineType: n2-standard-4
    depends_on: build
    timeout_in_minutes: 60
    parallelism: 14
--- a/.buildkite/pipelines/upload_pipeline.yml
+++ b/.buildkite/pipelines/upload_pipeline.yml
@ -2,4 +2,10 @@

 steps:
  - label: Upload tested pipeline
-    command: buildkite-agent pipeline upload ${TESTED_PIPELINE_PATH:-.buildkite/pipelines/pipeline_to_test.yml}
+    command: |
+      if [[ $TESTED_PIPELINE_PATH == *.yml ]]; then
+        echo "Uploading pipeline $TESTED_PIPELINE_PATH"
+        buildkite-agent pipeline upload $TESTED_PIPELINE_PATH
+      else
+        $TESTED_PIPELINE_PATH | buildkite-agent pipeline upload
+      fi