Security periodic pipeline images non-preemptible (#208799)

Pre-GA, we previously modified the critical path, weekly Quality Gate
pipelines over to non-preemptible images in order to avoid the
inconsistent "agent lost" Buildkite issue that might cause unnecessary
churn.

Now that we're GA, the periodic pipelines are becoming more critical
path as well, and in addition, we're seeing a concerning increase in the
number of "agent lost" issues we're encountering, as described here:
https://github.com/elastic/security-team/issues/11700

As such, this PR is hoping to address this issue for the periodic
pipelines as well.

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml

@@ -11,7 +11,6 @@ steps:
           provider: gcp
           enableNestedVirtualization: true
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 300
         parallelism: 5
         retry:
@@ -91,7 +90,6 @@ steps:
           provider: gcp
           enableNestedVirtualization: true
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -107,7 +105,6 @@ steps:
           provider: gcp
           enableNestedVirtualization: true
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -123,7 +120,6 @@ steps:
           provider: gcp
           enableNestedVirtualization: true
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -142,7 +138,6 @@ steps:
           provider: gcp
           enableNestedVirtualization: true
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 300
         parallelism: 3
         retry:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_detection_engine.yml

@@ -12,7 +12,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 8
@@ -27,7 +26,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
           # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 6
@@ -43,7 +41,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -58,7 +55,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -73,7 +69,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -88,7 +83,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -103,7 +97,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -118,7 +111,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -133,7 +125,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -148,7 +139,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -163,7 +153,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -178,7 +167,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -193,7 +181,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -208,7 +195,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -223,7 +209,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -238,7 +223,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -253,7 +237,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -268,7 +251,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -283,7 +265,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -298,7 +279,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -313,7 +293,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -328,7 +307,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -343,7 +321,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -358,7 +335,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -373,7 +349,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -388,7 +363,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -403,7 +377,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -418,7 +391,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -433,7 +405,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_entity_analytics.yml

@@ -9,7 +9,6 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
-      preemptible: true
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 2
@@ -25,7 +24,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -40,7 +38,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml

@@ -9,7 +9,6 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
-      preemptible: true
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 4
@@ -25,7 +24,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -40,7 +38,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -55,7 +52,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -70,7 +66,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_gen_ai.yml

@@ -9,7 +9,6 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
-      preemptible: true
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 1
@@ -26,7 +25,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -41,7 +39,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml

@@ -9,7 +9,6 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
-      preemptible: true
       # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 8
@@ -25,7 +24,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -40,7 +38,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_rule_management.yml

@@ -12,7 +12,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 8
@@ -27,7 +26,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 4
@@ -43,7 +41,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -58,7 +55,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -73,7 +69,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -88,7 +83,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -103,7 +97,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -118,7 +111,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -133,7 +125,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -148,7 +139,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -163,7 +153,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -178,7 +167,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -193,7 +181,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -208,7 +195,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -223,7 +209,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -238,7 +223,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -253,7 +237,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -268,7 +251,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -283,7 +265,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -298,7 +279,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -313,7 +293,6 @@ steps:
           imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
-          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic: