[CI] Convert the remaining steps in primary pipelines to spot instances (#138168)

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -161,7 +161,7 @@ export async function pickTestGroupRunOrder() {
 
   const JEST_MAX_MINUTES = process.env.JEST_MAX_MINUTES
     ? parseFloat(process.env.JEST_MAX_MINUTES)
-    : 50;
+    : 40;
   if (Number.isNaN(JEST_MAX_MINUTES)) {
     throw new Error(`invalid JEST_MAX_MINUTES: ${process.env.JEST_MAX_MINUTES}`);
   }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -22,10 +22,14 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-16
+      queue: n2-16-spot
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'

.buildkite/pipelines/on_merge.yml

@@ -31,7 +31,7 @@ steps:
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-16
+      queue: n2-16-spot
     key: build
     timeout_in_minutes: 60
     retry:
@@ -43,10 +43,12 @@ steps:
     label: Build TS Refs and Check Public API Docs
     key: public-api-docs
     agents:
-      queue: c2-4
+      queue: n2-4-spot
     timeout_in_minutes: 80
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -90,9 +92,13 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: c2-16
+      queue: n2-16-spot
     key: linting_with_types
     timeout_in_minutes: 90
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
@@ -107,8 +113,12 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: c2-8
+      queue: n2-16-spot
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/pipelines/pull_request/base.yml

@@ -10,10 +10,14 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-16
+      queue: n2-16-spot
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/build_api_docs.sh
     label: 'Build API Docs'
@@ -66,9 +70,13 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: c2-16
+      queue: n2-16-spot
     key: linting_with_types
     timeout_in_minutes: 90
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
@@ -83,5 +91,9 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: c2-8
+      queue: n2-16-spot
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/scripts/steps/check_types.sh

@@ -8,4 +8,4 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Check Types
 checks-reporter-with-killswitch "Check Types" \
-  node scripts/type_check
+  node scripts/type_check --concurrency 8

src/dev/typescript/run_type_check_cli.ts

@@ -55,7 +55,13 @@ export async function runTypeCheckCli() {
         }
       }
 
-      const concurrency = Math.min(4, Math.round((Os.cpus() || []).length / 2) || 1) || 1;
+      const concurrencyArg =
+        typeof flags.concurrency === 'string' && parseInt(flags.concurrency, 10);
+      const concurrency =
+        concurrencyArg && concurrencyArg > 0
+          ? concurrencyArg
+          : Math.min(4, Math.round((Os.cpus() || []).length / 2) || 1) || 1;
+
       log.info('running type check in', projects.length, 'projects');
 
       const tscArgs = [
@@ -115,12 +121,13 @@ export async function runTypeCheckCli() {
           node scripts/type_check --project packages/kbn-pm/tsconfig.json
       `,
       flags: {
-        string: ['project'],
+        string: ['project', 'concurrency'],
         boolean: ['skip-lib-check'],
         help: `
-          --project [path]    Path to a tsconfig.json file determines the project to check
-          --skip-lib-check    Skip type checking of all declaration files (*.d.ts). Default is false
-          --help              Show this message
+          --concurrency <number>  Number of projects to check in parallel. Defaults to 50% of available CPUs, up to 4.
+          --project [path]        Path to a tsconfig.json file determines the project to check
+          --skip-lib-check        Skip type checking of all declaration files (*.d.ts). Default is false
+          --help                  Show this message
         `,
       },
     }