[CI] Prep TeamCity pipelines to run in parallel with Jenkins (#87589)

.ci/teamcity/setup_ci_stats.js

@@ -24,7 +24,7 @@ const ciStats = require('./ci_stats');
       branch: process.env.GIT_BRANCH.replace(/^(refs\/heads\/|origin\/)/, ''),
       commit: process.env.GIT_COMMIT,
       targetBranch: process.env.GITHUB_PR_TARGET_BRANCH || null,
-      mergeBase: null, // TODO
+      mergeBase: process.env.GITHUB_PR_MERGE_BASE || null,
     });
   } catch (ex) {
     console.error(ex);

.ci/teamcity/setup_env.sh

@@ -33,15 +33,17 @@ fi
 
 if is_pr; then
   tc_set_env ELASTIC_APM_ACTIVE false
-  tc_set_env CHECKS_REPORTER_ACTIVE true
+  tc_set_env CHECKS_REPORTER_ACTIVE "${CI_REPORTING_ENABLED-}"
 
   # These can be removed once we're not supporting Jenkins and TeamCity at the same time
   # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
   tc_set_env ghprbGhRepository "elastic/kibana" # TODO?
   tc_set_env ghprbActualCommit "$GITHUB_PR_TRIGGERED_SHA"
   tc_set_env BUILD_URL "$TEAMCITY_BUILD_URL"
+
+  set_git_merge_base
 else
-  tc_set_env ELASTIC_APM_ACTIVE true
+  tc_set_env ELASTIC_APM_ACTIVE "${CI_REPORTING_ENABLED-}"
   tc_set_env CHECKS_REPORTER_ACTIVE false
 fi
 

.ci/teamcity/util.sh

@@ -79,3 +79,10 @@ tc_retry() {
   }
   tc_end_block "Retryable Step - Attempt #1"
 }
+
+set_git_merge_base() {
+  if [[ "${GITHUB_PR_TARGET_BRANCH-}" ]]; then
+    git fetch origin "$GITHUB_PR_TARGET_BRANCH"
+    tc_set_env GITHUB_PR_MERGE_BASE "$(git merge-base HEAD FETCH_HEAD)"
+  fi
+}

.teamcity/src/Agents.kt

@@ -10,14 +10,16 @@ val StandardAgents = sizes.map { size -> size to GoogleCloudAgent {
   machineType = "n2-standard-$size"
   diskSizeGb = 75
   diskType = GoogleCloudAgentDiskType.SSD
+  maxInstances = 750
 } }.toMap()
 
 val BuildAgent = GoogleCloudAgent {
-    sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
-    agentPrefix = "kibana-c2-16-"
-    machineType = "c2-standard-16"
-    diskSizeGb = 250
-    diskType = GoogleCloudAgentDiskType.SSD
+  sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
+  agentPrefix = "kibana-c2-16-"
+  machineType = "c2-standard-16"
+  diskSizeGb = 250
+  diskType = GoogleCloudAgentDiskType.SSD
+  maxInstances = 200
 }
 
 val CloudProfile = GoogleCloudProfile {

.teamcity/src/Common.kt

@@ -0,0 +1,27 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.DslContext
+
+// If set to true, github check/commit status will be reported, failed-test-reporter will run, etc.
+const val ENABLE_REPORTING = false
+
+// If set to false, jobs with triggers (scheduled, on commit, etc) will be paused
+const val ENABLE_TRIGGERS = false
+
+fun getProjectBranch(): String {
+  return DslContext.projectName
+}
+
+fun getCorrespondingESBranch(): String {
+  return getProjectBranch().replace("_teamcity", "")
+}
+
+fun areTriggersEnabled(): Boolean {
+  return ENABLE_TRIGGERS;
+}
+
+fun isReportingEnabled(): Boolean {
+  return ENABLE_REPORTING;
+}
+
+fun makeSafeId(id: String): String {
+  return id.replace(Regex("[^a-zA-Z0-9_]"), "_")
+}

.teamcity/src/Extensions.kt

@@ -39,7 +39,9 @@ val testArtifactRules = """
 fun BuildType.addTestSettings() {
   artifactRules += "\n" + testArtifactRules
   steps {
-    failedTestReporter()
+    if(isReportingEnabled()) {
+       failedTestReporter()
+    }
   }
   features {
     junit()

.teamcity/src/builds/BaselineCi.kt

@@ -1,10 +1,12 @@
 package builds
 
 import addSlackNotifications
+import areTriggersEnabled
 import builds.default.DefaultBuild
 import builds.default.DefaultSavedObjectFieldMetrics
 import builds.oss.OssBuild
 import dependsOn
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
 import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
@@ -15,14 +17,14 @@ object BaselineCi : BuildType({
   name = "Baseline CI"
   description = "Runs builds, saved object field metrics for every commit"
   type = Type.COMPOSITE
-  paused = true
+  paused = !areTriggersEnabled()
 
   templates(KibanaTemplate)
 
   triggers {
     vcs {
-      branchFilter = "refs/heads/master_teamcity"
-//      perCheckinTriggering = true // TODO re-enable this later, it wreaks havoc when I merge upstream
+      branchFilter = "refs/heads/${getProjectBranch()}"
+      perCheckinTriggering = areTriggersEnabled()
     }
   }
 

.teamcity/src/builds/HourlyCi.kt

@@ -1,7 +1,9 @@
 package builds
 
 import addSlackNotifications
+import areTriggersEnabled
 import dependsOn
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
 import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
 import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.schedule
@@ -11,6 +13,7 @@ object HourlyCi : BuildType({
   name = "Hourly CI"
   description = "Runs everything in CI, hourly"
   type = Type.COMPOSITE
+  paused = !areTriggersEnabled()
 
   triggers {
     schedule {
@@ -18,7 +21,7 @@ object HourlyCi : BuildType({
         hours = "*"
         minutes = "0"
       }
-      branchFilter = "refs/heads/master_teamcity"
+      branchFilter = "refs/heads/${getProjectBranch()}"
       triggerBuild = always()
       withPendingChangesOnly = true
     }

.teamcity/src/builds/PullRequestCi.kt

@@ -1,15 +1,14 @@
 package builds
 
+import builds.default.DefaultSavedObjectFieldMetrics
 import dependsOn
-import jetbrains.buildServer.configs.kotlin.v2019_2.AbsoluteId
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests
 import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests
 import vcs.Kibana
 
 object PullRequestCi : BuildType({
-  id = AbsoluteId("Kibana_PullRequest_CI")
+  id("Pull_Request")
   name = "Pull Request CI"
   type = Type.COMPOSITE
 
@@ -36,7 +35,7 @@ object PullRequestCi : BuildType({
 
   params {
     param("elastic.pull_request.enabled", "true")
-    param("elastic.pull_request.target_branch", "master_teamcity")
+    param("elastic.pull_request.target_branch", getProjectBranch())
     param("elastic.pull_request.allow_org_users", "true")
     param("elastic.pull_request.allowed_repo_permissions", "admin,write")
     param("elastic.pull_request.allowed_list", prAllowedList.joinToString(","))
@@ -74,5 +73,8 @@ object PullRequestCi : BuildType({
     }
   }
 
-  dependsOn(FullCi)
+  dependsOn(
+    FullCi,
+    DefaultSavedObjectFieldMetrics
+  )
 })

.teamcity/src/projects/Kibana.kt

@@ -32,7 +32,7 @@ fun Kibana(config: KibanaConfiguration = KibanaConfiguration()) : Project {
       param("teamcity.ui.settings.readOnly", "true")
 
       // https://github.com/JetBrains/teamcity-webhooks
-      param("teamcity.internal.webhooks.enable", "false")
+      param("teamcity.internal.webhooks.enable", "true")
       param("teamcity.internal.webhooks.events", "BUILD_STARTED;BUILD_FINISHED;BUILD_INTERRUPTED;CHANGES_LOADED;BUILD_TYPE_ADDED_TO_QUEUE;BUILD_PROBLEMS_CHANGED")
       param("teamcity.internal.webhooks.url", "https://ci-stats.kibana.dev/_teamcity_webhook")
       param("teamcity.internal.webhooks.username", "automation")

.teamcity/src/templates/KibanaTemplate.kt

@@ -2,6 +2,8 @@ package templates
 
 import StandardAgents
 import co.elastic.teamcity.common.requireAgent
+import getProjectBranch
+import isReportingEnabled
 import vcs.Kibana
 import jetbrains.buildServer.configs.kotlin.v2019_2.BuildStep
 import jetbrains.buildServer.configs.kotlin.v2019_2.ParameterDisplay
@@ -33,7 +35,7 @@ object KibanaTemplate : Template({
         authType = token {
           token = "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b"
         }
-        filterTargetBranch = "refs/heads/master_teamcity"
+        filterTargetBranch = "refs/heads/${getProjectBranch()}"
         filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER
       }
     }
@@ -63,6 +65,8 @@ object KibanaTemplate : Template({
     param("env.GIT_COMMIT", "%build.vcs.number%")
     param("env.branch_specifier", "%vcsroot.branch%")
 
+    param("env.CI_REPORTING_ENABLED", isReportingEnabled().toString())
+
     password("env.KIBANA_CI_STATS_CONFIG", "", display = ParameterDisplay.HIDDEN)
     password("env.CI_STATS_TOKEN", "credentialsJSON:ea975068-ca68-4da5-8189-ce90f4286bc0", display = ParameterDisplay.HIDDEN)
     password("env.CI_STATS_HOST", "credentialsJSON:933ba93e-4b06-44c1-8724-8c536651f2b6", display = ParameterDisplay.HIDDEN)
@@ -72,10 +76,19 @@ object KibanaTemplate : Template({
     // password("env.CI_STATS_HOST", "%vault:kibana-issues:secret/kibana-issues/dev/kibana_ci_stats!/api_host%", display = ParameterDisplay.HIDDEN)
 
     // TODO remove this once we are able to pull it out of vault and put it closer to the things that require it
-    password("env.GITHUB_TOKEN", "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b", display = ParameterDisplay.HIDDEN)
-    password("env.KIBANA_CI_REPORTER_KEY", "", display = ParameterDisplay.HIDDEN)
-    password("env.KIBANA_CI_REPORTER_KEY_BASE64", "credentialsJSON:86878779-4cf7-4434-82af-5164a1b992fb", display = ParameterDisplay.HIDDEN)
-
+    if(isReportingEnabled()) {
+      password(
+        "env.GITHUB_TOKEN",
+        "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b",
+        display = ParameterDisplay.HIDDEN
+      )
+      password("env.KIBANA_CI_REPORTER_KEY", "", display = ParameterDisplay.HIDDEN)
+      password(
+        "env.KIBANA_CI_REPORTER_KEY_BASE64",
+        "credentialsJSON:86878779-4cf7-4434-82af-5164a1b992fb",
+        display = ParameterDisplay.HIDDEN
+      )
+    }
   }
 
   steps {

.teamcity/src/vcs/Elasticsearch.kt

@@ -1,11 +1,13 @@
 package vcs
 
+import getCorrespondingESBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
+import makeSafeId
 
 object Elasticsearch : GitVcsRoot({
-  id("elasticsearch_master")
+  id("elasticsearch_${makeSafeId(getCorrespondingESBranch())}")
 
-  name = "elasticsearch / master"
+  name = "elasticsearch / ${getCorrespondingESBranch()}"
   url = "https://github.com/elastic/elasticsearch.git"
-  branch = "refs/heads/master"
+  branch = "refs/heads/${getCorrespondingESBranch()}"
 })

.teamcity/src/vcs/Kibana.kt

@@ -1,11 +1,13 @@
 package vcs
 
+import getProjectBranch
 import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
+import makeSafeId
 
 object Kibana : GitVcsRoot({
-  id("kibana_master")
+  id("kibana_${makeSafeId(getProjectBranch())}")
 
-  name = "kibana / master"
+  name = "kibana / ${getProjectBranch()}"
   url = "https://github.com/elastic/kibana.git"
-  branch = "refs/heads/master_teamcity"
+  branch = "refs/heads/${getProjectBranch()}"
 })

.teamcity/tests/projects/KibanaTest.kt

@@ -2,6 +2,7 @@ package projects
 
 import jetbrains.buildServer.configs.kotlin.v2019_2.AbsoluteId
 import jetbrains.buildServer.configs.kotlin.v2019_2.DslContext
+import makeSafeId
 import org.junit.Assert.*
 import org.junit.Test