[BK][7.17] Migrate buildkite batch 5 (unsupported ftr / flaky test runner) (#180575)

## Summary

Manual backport of: https://github.com/elastic/kibana/pull/180403 with
some extra fixes:
- scripts that generate pipeline steps should care for what's the
environment, and branch on agent targeting rules
 - set google account for bazel to access cache

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -10,6 +10,34 @@ const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>}
   require('./groups.json').groups
 );
 
+// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
+/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
+const getAgentRule = (queueName = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -51,9 +79,7 @@ const pipeline = {
     {
       command: '.buildkite/pipelines/flaky_tests/runner.sh',
       label: 'Create pipeline',
-      agents: {
-        queue: 'kibana-default',
-      },
+      agents: getAgentRule('n2-4'),
     },
   ],
 };

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -2,4 +2,4 @@
 
 set -euo pipefail
 
-node .buildkite/pipelines/flaky_tests/pipeline.js | buildkite-agent pipeline upload
+node .buildkite/pipelines/flaky_tests/pipeline.js

.buildkite/pipelines/flaky_tests/runner.js

@@ -12,6 +12,32 @@ const concurrency = 25;
 const defaultCount = concurrency * 2;
 const initialJobs = 3;
 
+const getAgentRule = (queueName = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 function getTestSuitesFromMetadata() {
   const keys = execSync('buildkite-agent meta-data keys')
     .toString()
@@ -111,7 +137,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: { queue: 'c2-8' },
+  agents: getAgentRule('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -136,7 +162,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
           label: `Default CI Group ${CI_GROUP}`,
-          agents: { queue: 'n2-4' },
+          agents: getAgentRule('n2-4'),
           depends_on: 'build',
           parallelism: RUN_COUNT,
           concurrency: concurrency,
@@ -147,7 +173,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
           label: `OSS CI Group ${CI_GROUP}`,
-          agents: { queue: 'ci-group-4d' },
+          agents: getAgentRule('n2-4-spot'),
           depends_on: 'build',
           parallelism: RUN_COUNT,
           concurrency: concurrency,
@@ -161,7 +187,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
-        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
+        agents: getAgentRule(IS_XPACK ? 'n2-4' : 'n2-4-spot'),
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,
@@ -176,7 +202,7 @@ for (const testSuite of testSuites) {
           IS_XPACK ? 'xpack' : 'oss'
         }_accessibility.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
-        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
+        agents: getAgentRule(IS_XPACK ? 'n2-4' : 'n2-4-spot'),
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,

.buildkite/pipelines/upload_pipeline.yml

@@ -2,4 +2,10 @@
 
 steps:
   - label: Upload tested pipeline
-    command: buildkite-agent pipeline upload ${TESTED_PIPELINE_PATH:-.buildkite/pipelines/pipeline_to_test.yml}
+    command: |
+      if [[ $TESTED_PIPELINE_PATH == *.yml ]]; then
+        echo "Uploading pipeline $TESTED_PIPELINE_PATH"
+        buildkite-agent pipeline upload $TESTED_PIPELINE_PATH
+      else
+        $TESTED_PIPELINE_PATH | buildkite-agent pipeline upload
+      fi

.buildkite/scripts/common/setup_bazel.sh

@@ -29,7 +29,7 @@ if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
 
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET
-  build --google_default_credentials
+  build --google_credentials=$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
 EOF
 fi