[CI] Add slack alerts to tracked branch jobs, change default channel, change formatting (#66580) (#66742)

# Conflicts:
#	Jenkinsfile

Jenkinsfile

@@ -39,8 +39,10 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
           }
         }
 
-        retryable.printFlakyFailures()
-        kibanaPipeline.sendMail()
+        if (params.NOTIFY_ON_FAILURE) {
+          slackNotifications.onFailure()
+          kibanaPipeline.sendMail()
+        }
       }
     }
   }

vars/slackNotifications.groovy

@@ -62,21 +62,32 @@ def getTestFailures() {
   def messages = []
   messages << "*Test Failures*"
 
-  def list = failures.collect { "• <${it.url}|${it.fullDisplayName}>" }.join("\n")
+  def list = failures.collect { "• <${it.url}|${it.fullDisplayName.split('.', 2)[-1]}>" }.join("\n")
   return "*Test Failures*\n${list}"
 }
 
-def sendFailedBuild(Map params = [:]) {
-  def displayName = "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
+def getDefaultDisplayName() {
+  return "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
+}
 
+def getDefaultContext() {
+  def duration = currentBuild.durationString.replace(' and counting', '')
+
+  return contextBlock([
+    "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}",
+    "<https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>",
+  ].join(' · '))
+}
+
+def sendFailedBuild(Map params = [:]) {
   def config = [
-    channel: '#kibana-operations',
-    title: ":broken_heart: *<${env.BUILD_URL}|${displayName}>*",
-    message: ":broken_heart: ${displayName}",
+    channel: '#kibana-operations-alerts',
+    title: ":broken_heart: *<${env.BUILD_URL}|${getDefaultDisplayName()}>*",
+    message: ":broken_heart: ${getDefaultDisplayName()}",
     color: 'danger',
     icon: ':jenkins:',
     username: 'Kibana Operations',
-    context: contextBlock("${displayName} · <https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>"),
+    context: getDefaultContext(),
   ] + params
 
   def blocks = [markdownBlock(config.title)]
@@ -94,18 +105,24 @@ def sendFailedBuild(Map params = [:]) {
   )
 }
 
+def onFailure(Map options = [:]) {
+  catchError {
+    def status = buildUtils.getBuildStatus()
+    if (status != "SUCCESS") {
+      catchErrors {
+        sendFailedBuild(options)
+      }
+    }
+  }
+}
+
 def onFailure(Map options = [:], Closure closure) {
   // try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
   catchError {
     closure()
   }
 
-  def status = buildUtils.getBuildStatus()
-  if (status != "SUCCESS" && status != "UNSTABLE") {
-    catchErrors {
-      sendFailedBuild(options)
-    }
-  }
+  onFailure(options)
 }
 
 return this