Split up bc and pr upgrade tests (#129756)

Splits up bc and pr upgrade tests as they tend to be a bottle neck in intake and pr builds nowadays.

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -56,20 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
 
 cat <<EOF | buildkite-agent pipeline upload
 steps:
-    - label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
+  - group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
+    steps:
-      timeout_in_minutes: 300
+      - label: "bc-upgrade-tests-part{{matrix.PART}}"
-      agents:
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
-        provider: gcp
+        timeout_in_minutes: 300
-        image: family/elasticsearch-ubuntu-2004
+        agents:
-        machineType: n1-standard-32
+          provider: gcp
-        buildDirectory: /dev/shm/bk
+          image: family/elasticsearch-ubuntu-2004
-        preemptible: true
+          machineType: n1-standard-32
-      retry:
+          buildDirectory: /dev/shm/bk
-        automatic:
+        matrix:
-          - exit_status: "-1"
+          setup:
-            limit: 3
+            PART: ["1", "2", "3", "4", "5", "6"]
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
 EOF

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -24,22 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
 
 echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
 
-cat <<EOF | buildkite-agent pipeline upload
+cat << EOF | buildkite-agent pipeline upload
 steps:
-    - label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
+    - group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
+      steps:
-      timeout_in_minutes: 300
+        - label: "pr-upgrade-part-{{matrix.PART}}"
-      agents:
+          command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
-        provider: gcp
+          timeout_in_minutes: 300
-        image: family/elasticsearch-ubuntu-2004
+          agents:
-        machineType: n1-standard-32
+            provider: gcp
-        buildDirectory: /dev/shm/bk
+            image: family/elasticsearch-ubuntu-2004
-        preemptible: true
+            machineType: n1-standard-32
-      retry:
+            buildDirectory: /dev/shm/bk
-        automatic:
+          matrix:
-          - exit_status: "-1"
+            setup:
-            limit: 3
+              PART: ["1", "2", "3", "4", "5", "6"]
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
 EOF

build.gradle

@@ -351,95 +351,45 @@ allprojects {
     }
   }
 
+  def splitForCI = { proj, partString ->
+    proj.tasks.register("check$partString") {
+      dependsOn 'check'
+      withReleaseBuild {
+        dependsOn 'assemble'
+      }
+    }
+
+    proj.tasks.addRule("Pattern: v<BWC_VERSION>#bwcTest$partString") { name ->
+      if(name.endsWith("#bwcTest$partString")) {
+        proj.project.getTasks().register(name) {
+          task -> task.dependsOn(proj.tasks.named { tskName -> tskName == (name - partString) })
+        }
+      }
+    }
+    
+    proj.tasks.register("bcUpgradeTest$partString") {
+      dependsOn tasks.matching { it.name == 'bcUpgradeTest' }
+      withReleaseBuild {
+        dependsOn 'assemble'
+      }
+    }
+  }
+
   plugins.withId('lifecycle-base') {
     if (project.path.startsWith(":x-pack:")) {
       if (project.path.contains("security") || project.path.contains(":ml")) {
-        tasks.register('checkPart4') {
+        splitForCI(project, "Part4")
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart4") { name ->
-          if(name.endsWith("#bwcTestPart4")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part4") })
-            }
-          }
-        }
       } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
-        tasks.register('checkPart3') {
+        splitForCI(project, "Part3")
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart3") { name ->
-          if(name.endsWith("#bwcTestPart3")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part3") })
-            }
-          }
-        }
       } else if (project.path.contains("multi-node")) {
-        tasks.register('checkPart5') {
+        splitForCI(project, "Part5")
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart5") { name ->
-          if(name.endsWith("#bwcTestPart5")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part5") })
-            }
-          }
-        }
       } else {
-        tasks.register('checkPart2') {
+        splitForCI(project, "Part2")
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart2") { name ->
-          if(name.endsWith("#bwcTestPart2")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part2") })
-            }
-          }
-        }
       }
     } else if(project.path.startsWith(":qa:")) {
-      tasks.register('checkPart6') {
+      splitForCI(project, "Part6")
-        dependsOn 'check'
-        withReleaseBuild {
-          dependsOn 'assemble'
-        }
-      }
-      tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart6") { name ->
-        if(name.endsWith("#bwcTestPart6")) {
-          project.getTasks().register(name) {
-            task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part6") })
-          }
-        }
-      }
     } else {
-      tasks.register('checkPart1') {
+      splitForCI(project, "Part1")
-        dependsOn 'check'
-        withReleaseBuild {
-          dependsOn 'assemble'
-        }
-      }
-      tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart1") { name ->
-        if(name.endsWith("#bwcTestPart1")) {
-          project.getTasks().register(name) {
-            task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part1") })
-          }
-        }
-      }
     }
     tasks.register('functionalTests') {
       dependsOn 'check'