Split build benchmark jobs for faster feedback (#84978)

Also handle gradle-profiler introduced spotless issue when running performance tests

.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part1.yml

@@ -1,9 +1,9 @@
 ---
 - job:
-    name: "elastic+elasticsearch+pull-request+build-benchmark"
+    name: "elastic+elasticsearch+pull-request+build-benchmark-part1"
-    display-name: "elastic / elasticsearch - pull request build benchmark"
+    display-name: "elastic / elasticsearch - pull request build benchmark part 1"
-    description: "Testing of Elasticsearch pull requests - build benchmark"
+    description: "Testing of Elasticsearch pull requests - build benchmark part 1"
-    workspace: "/dev/shm/elastic+elasticsearch+pull-request+build-bench"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+build-bench-1"
     scm:
       - git:
           refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
@@ -25,7 +25,7 @@
           allow-whitelist-orgs-as-admins: true
           trigger-phrase: '.*run\W+elasticsearch-ci/build-bench.*'
           github-hooks: true
-          status-context: elasticsearch-ci/build-benchmark
+          status-context: elasticsearch-ci/build-benchmark-part1
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
@@ -45,5 +45,5 @@
          #!/usr/local/bin/runbld --redirect-stderr
          $WORKSPACE/.ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
          $WORKSPACE/.ci/scripts/install-gradle-profiler.sh
-         $WORKSPACE/.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-tool-update.scenarios --project-dir . --output-dir profile-out
+         $WORKSPACE/.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-part1.scenarios --project-dir . --output-dir profile-out
          tar -czf build/${BUILD_NUMBER}.tar.bz2 profile-out

.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part2.yml

@@ -0,0 +1,49 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+build-benchmark-part2"
+    display-name: "elastic / elasticsearch - pull request build benchmark part 2"
+    description: "Testing of Elasticsearch pull requests - build benchmark part 2"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+build-bench-2"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    properties:
+      - inject:
+          properties-content: |
+            BUILD_PERFORMANCE_TEST=true
+            COMPOSE_HTTP_TIMEOUT=120
+            JOB_BRANCH=%BRANCH%
+            HOME=$JENKINS_HOME
+            GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/build-bench.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/build-benchmark-part2
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'build-benchmark'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+      - shell: |
+         #!/usr/local/bin/runbld --redirect-stderr
+         $WORKSPACE/.ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
+         $WORKSPACE/.ci/scripts/install-gradle-profiler.sh
+         $WORKSPACE/.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-part2.scenarios --project-dir . --output-dir profile-out
+         tar -czf build/${BUILD_NUMBER}.tar.bz2 profile-out

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -59,7 +59,6 @@ public class FormattingPrecommitPlugin implements Plugin<Project> {
                 }
 
                 java.target("src/**/*.java");
-
                 java.removeUnusedImports();
 
                 // We enforce a standard order for imports
@@ -72,6 +71,13 @@ public class FormattingPrecommitPlugin implements Plugin<Project> {
                 // order, apply this one last, otherwise non-empty blank lines can creep
                 // in.
                 java.trimTrailingWhitespace();
+
+                // When running build benchmarks we alter the source in some scenarios.
+                // The gradle-profiler unfortunately does not generate compliant formatted
+                // sources so we ignore that altered file when running build benchmarks
+                if(Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
+                    java.ignoreErrorForPath("src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java");
+                }
             });
 
             project.getTasks().named("precommit").configure(precommitTask -> precommitTask.dependsOn("spotlessJavaCheck"));

build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios

@@ -1,21 +1,5 @@
 # Can specify scenarios to use when none are specified on the command line
-default-scenarios = ["buildConfiguration_master", "buildConfiguration_branch", "single_project_master", "single_project_branch", "precommit_master", "precommit_branch"]
+default-scenarios = ["buildConfiguration_master", "buildConfiguration_branch", "single_project_master", "single_project_branch"]
-
-buildConfiguration_branch {
-    title = "configuration phase (@testGitCommit@)"
-    tasks = ["help"]
-    gradle-args = ["--no-scan", "--no-build-cache"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "@testGitCommit@"
-    }
-}
 
 buildConfiguration_master {
     title = "configuration phase (master)"
@@ -33,10 +17,9 @@ buildConfiguration_master {
     }
 }
 
-precommit_branch {
+buildConfiguration_branch {
-    title = "precommit (@testGitCommit@)"
+    title = "configuration phase (@testGitCommit@)"
-    cleanup-tasks = ["clean"]
+    tasks = ["help"]
-    tasks = ["precommit"]
     gradle-args = ["--no-scan", "--no-build-cache"]
     run-using = cli // value can be "cli" or "tooling-api"
     daemon = warm // value can be "warm", "cold", or "none"
@@ -50,40 +33,6 @@ precommit_branch {
     }
 }
 
-precommit_master {
-    title = "precommit (master)"
-    cleanup-tasks = ["clean"]
-    tasks = ["precommit"]
-    gradle-args = ["--no-scan", "--no-build-cache"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "master"
-    }
-}
-
-single_project_branch {
-    title = "single project (@testGitCommit@)"
-    tasks = [":server:precommit"]
-    gradle-args = ["--no-scan"]
-    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "@testGitCommit@"
-    }
-}
-
 single_project_master {
     title = "single project (master)"
     tasks = [":server:precommit"]
@@ -100,3 +49,20 @@ single_project_master {
         build = "master"
     }
 }
+
+single_project_branch {
+    title = "single project (@testGitCommit@)"
+    tasks = [":server:precommit"]
+    gradle-args = ["--no-scan"]
+    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
+    run-using = cli // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    warm-ups = 5
+    iterations = 10
+    system-properties {
+        "BUILD_PERFORMANCE_TEST" = "true"
+    }
+    git-checkout = {
+        build = "@testGitCommit@"
+    }
+}

build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios

@@ -0,0 +1,35 @@
+default-scenarios = ["precommit_master", "precommit_branch"]
+
+precommit_master {
+    title = "precommit (master)"
+    cleanup-tasks = ["clean"]
+    tasks = ["precommit"]
+    gradle-args = ["--no-scan", "--no-build-cache"]
+    run-using = cli // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    warm-ups = 5
+    iterations = 10
+    system-properties {
+        "BUILD_PERFORMANCE_TEST" = "true"
+    }
+    git-checkout = {
+        build = "master"
+    }
+}
+
+precommit_branch {
+    title = "precommit (@testGitCommit@)"
+    cleanup-tasks = ["clean"]
+    tasks = ["precommit"]
+    gradle-args = ["--no-scan", "--no-build-cache"]
+    run-using = cli // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    warm-ups = 5
+    iterations = 10
+    system-properties {
+        "BUILD_PERFORMANCE_TEST" = "true"
+    }
+    git-checkout = {
+        build = "@testGitCommit@"
+    }
+}