[BK][8.13] Migrate buildkite batch 5 (unsupported ftr / flaky test runner) (#180602)

## Summary

Manual backport of #180403 

Changes:
- scripts that generate pipeline steps should care for what's the
environment, and branch on agent targeting rules

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -5,4 +5,4 @@ set -euo pipefail
 UUID="$(cat /proc/sys/kernel/random/uuid)"
 export UUID
 
-ts-node .buildkite/pipelines/flaky_tests/pipeline.ts | buildkite-agent pipeline upload
+ts-node .buildkite/pipelines/flaky_tests/pipeline.ts

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -31,6 +31,34 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
+/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
+const getAgentRule = (queueName: string = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -49,8 +77,10 @@ function getTestSuitesFromJson(json: string) {
     fail(`JSON test config must be an array`);
   }
 
-  /** @type {Array<{ type: 'group', key: string; count: number } | { type: 'ftrConfig', ftrConfig: string; count: number }>} */
-  const testSuites = [];
+  const testSuites: Array<
+    | { type: 'group'; key: string; count: number }
+    | { type: 'ftrConfig'; ftrConfig: string; count: number }
+  > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
       fail(`testSuites must be objects`);
@@ -73,6 +103,7 @@ function getTestSuitesFromJson(json: string) {
       }
 
       testSuites.push({
+        type: 'ftrConfig',
         ftrConfig,
         count,
       });
@@ -84,6 +115,7 @@ function getTestSuitesFromJson(json: string) {
       fail(`testSuite.key must be a string`);
     }
     testSuites.push({
+      type: 'group',
       key,
       count,
     });
@@ -117,7 +149,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: { queue: 'c2-8' },
+  agents: getAgentRule('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -127,7 +159,7 @@ for (const testSuite of testSuites) {
     continue;
   }
 
-  if (testSuite.ftrConfig) {
+  if (testSuite.type === 'ftrConfig') {
     steps.push({
       command: `.buildkite/scripts/steps/test/ftr_configs.sh`,
       env: {
@@ -138,9 +170,7 @@ for (const testSuite of testSuites) {
       concurrency,
       concurrency_group: process.env.UUID,
       concurrency_method: 'eager',
-      agents: {
-        queue: 'n2-4-spot-2',
-      },
+      agents: getAgentRule('n2-4-spot'),
       depends_on: 'build',
       timeout_in_minutes: 150,
       cancel_on_build_failing: true,
@@ -164,7 +194,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: agentQueue },
+        agents: getAgentRule(agentQueue),
         depends_on: 'build',
         timeout_in_minutes: 150,
         parallelism: testSuite.count,

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -3,7 +3,10 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     retry:
       automatic:
         - exit_status: '*'
@@ -12,7 +15,11 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
@@ -24,7 +31,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -35,9 +46,13 @@ steps:
           limit: 1
 
   - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
-    label: 'Profling Cypress Tests'
+    label: 'Profiling Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -50,7 +65,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
     label: 'Synthetics @elastic/synthetics Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     artifact_paths:
@@ -65,11 +84,17 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows.sh
     label: 'Defend Workflows Cypress Tests'
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsdInterface: nvme
+      localSsds: 1
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 10
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
+        - exit_status: '-1'
+          limit: 1

.buildkite/pipelines/upload_pipeline.yml

@@ -2,4 +2,10 @@
 
 steps:
   - label: Upload tested pipeline
-    command: buildkite-agent pipeline upload ${TESTED_PIPELINE_PATH:-.buildkite/pipelines/pipeline_to_test.yml}
+    command: |
+      if [[ $TESTED_PIPELINE_PATH == *.yml ]]; then
+        echo "Uploading pipeline $TESTED_PIPELINE_PATH"
+        buildkite-agent pipeline upload $TESTED_PIPELINE_PATH
+      else
+        $TESTED_PIPELINE_PATH | buildkite-agent pipeline upload
+      fi