[CI] Slack alerts - Elasticsearch snapshot failures (#64724) (#65270)

.ci/es-snapshots/Jenkinsfile_verify_es

@@ -21,41 +21,46 @@ def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-d
 
 kibanaPipeline(timeoutMinutes: 135) {
   catchErrors {
-    retryable.enable(2)
-    withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
-      parallel([
-        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-        'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-        'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-          'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-          'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-          'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-          'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-          'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-          'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-          'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-          'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-          'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-          'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-          'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-        ]),
-        'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-          'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-          'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-          'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-          'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-          'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-          'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-          'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-          'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-          'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-        ]),
-      ])
-    }
+    slackNotifications.onFailure(
+      title: ":broken_heart: *<${env.BUILD_URL}|[${SNAPSHOT_VERSION}] ES Snapshot Verification Failure>*",
+      message: ":broken_heart: [${SNAPSHOT_VERSION}] ES Snapshot Verification Failure",
+    ) {
+      retryable.enable(2)
+      withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
+        parallel([
+          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+          ]),
+          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
+            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
+            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
+            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+          ]),
+        ])
+      }
 
-    promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
+      promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
+    }
   }
 
   kibanaPipeline.sendMail()

vars/slackNotifications.groovy

@@ -0,0 +1,111 @@
+def getFailedBuildBlocks() {
+  def messages = [
+    getFailedSteps(),
+    getTestFailures(),
+  ]
+
+  return messages
+    .findAll { !!it } // No blank strings
+    .collect { markdownBlock(it) }
+}
+
+def dividerBlock() {
+  return [ type: "divider" ]
+}
+
+def markdownBlock(message) {
+  return [
+    type: "section",
+    text: [
+      type: "mrkdwn",
+      text: message,
+    ],
+  ]
+}
+
+def contextBlock(message) {
+  return [
+    type: "context",
+    elements: [
+      [
+        type: 'mrkdwn',
+        text: message,
+      ]
+    ]
+  ]
+}
+
+def getFailedSteps() {
+  try {
+    def steps = jenkinsApi.getFailedSteps()?.findAll { step ->
+      step.displayName != 'Check out from version control'
+    }
+
+    if (steps?.size() > 0) {
+      def list = steps.collect { "• <${it.logs}|${it.displayName}>" }.join("\n")
+      return "*Failed Steps*\n${list}"
+    }
+  } catch (ex) {
+    buildUtils.printStacktrace(ex)
+    print "Error retrieving failed pipeline steps for PR comment, will skip this section"
+  }
+
+  return ""
+}
+
+def getTestFailures() {
+  def failures = testUtils.getFailures()
+  if (!failures) {
+    return ""
+  }
+
+  def messages = []
+  messages << "*Test Failures*"
+
+  def list = failures.collect { "• <${it.url}|${it.fullDisplayName}>" }.join("\n")
+  return "*Test Failures*\n${list}"
+}
+
+def sendFailedBuild(Map params = [:]) {
+  def displayName = "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
+
+  def config = [
+    channel: '#kibana-operations',
+    title: ":broken_heart: *<${env.BUILD_URL}|${displayName}>*",
+    message: ":broken_heart: ${displayName}",
+    color: 'danger',
+    icon: ':jenkins:',
+    username: 'Kibana Operations',
+    context: contextBlock("${displayName} · <https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>"),
+  ] + params
+
+  def blocks = [markdownBlock(config.title)]
+  getFailedBuildBlocks().each { blocks << it }
+  blocks << dividerBlock()
+  blocks << config.context
+
+  slackSend(
+    channel: config.channel,
+    username: config.username,
+    iconEmoji: config.icon,
+    color: config.color,
+    message: config.message,
+    blocks: blocks
+  )
+}
+
+def onFailure(Map options = [:], Closure closure) {
+  // try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
+  catchError {
+    closure()
+  }
+
+  def status = buildUtils.getBuildStatus()
+  if (status != "SUCCESS" && status != "UNSTABLE") {
+    catchErrors {
+      sendFailedBuild(options)
+    }
+  }
+}
+
+return this