[ci] Reduce disk size for steps (#202015)

First pass on reducing disk size. This sets disk size for frequently run
steps not running builds or integration tests.

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -485,7 +485,10 @@ export async function pickTestGroupRunOrder() {
             parallelism: unit.count,
             timeout_in_minutes: 120,
             key: 'jest',
-            agents: expandAgentQueue('n2-4-spot'),
+            agents: {
+              ...expandAgentQueue('n2-4-spot'),
+              diskSizeGb: 75,
+            },
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },

.buildkite/pipelines/on_merge.yml

@@ -10,6 +10,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     retry:
       automatic:
         - exit_status: '*'
@@ -39,6 +40,7 @@ steps:
       provider: gcp
       machineType: n2-highcpu-8
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -53,6 +55,7 @@ steps:
       provider: gcp
       machineType: n2-standard-2
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -67,6 +70,7 @@ steps:
       provider: gcp
       machineType: n2-standard-16
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -81,6 +85,7 @@ steps:
       provider: gcp
       machineType: n2-standard-32
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -97,6 +102,7 @@ steps:
       diskType: 'hyperdisk-balanced'
       preemptible: true
       spotZones: us-central1-a,us-central1-b,us-central1-c
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -111,6 +117,7 @@ steps:
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -128,6 +135,7 @@ steps:
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 80
     retry:
       automatic:
@@ -143,6 +151,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -159,6 +168,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'

.buildkite/pipelines/pull_request/base.yml

@@ -4,6 +4,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
 
   - wait
 
@@ -26,6 +27,7 @@ steps:
     agents:
       machineType: n2-highcpu-8
       preemptible: true
+      diskSizeGb: 75
     key: quick_checks
     timeout_in_minutes: 60
     retry:
@@ -39,6 +41,7 @@ steps:
     agents:
       machineType: n2-standard-2
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -50,6 +53,7 @@ steps:
     agents:
       machineType: n2-standard-16
       preemptible: true
+      diskSizeGb: 75
     key: linting
     timeout_in_minutes: 60
     retry:
@@ -62,6 +66,7 @@ steps:
     agents:
       machineType: n2-standard-32
       preemptible: true
+      diskSizeGb: 75
     key: linting_with_types
     timeout_in_minutes: 60
     retry:
@@ -74,6 +79,7 @@ steps:
     agents:
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     key: check_oas_snapshot
     timeout_in_minutes: 60
     retry:
@@ -88,6 +94,7 @@ steps:
       diskType: 'hyperdisk-balanced'
       preemptible: true
       spotZones: us-central1-a,us-central1-b,us-central1-c
+      diskSizeGb: 75
     key: check_types
     timeout_in_minutes: 60
     retry:
@@ -101,6 +108,7 @@ steps:
     label: Mark CI Stats as ready
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -114,6 +122,7 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@@ -129,6 +138,7 @@ steps:
     agents:
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     key: build_api_docs
     timeout_in_minutes: 90
     retry: