[CI] Harden Slack notifications (#73361) (#73475)

.ci/pipeline-library/src/test/KibanaBasePipelineTest.groovy

@@ -78,6 +78,10 @@ class KibanaBasePipelineTest extends BasePipelineTest {
     return helper.callStack.find { it.methodName == name }
   }
 
+  def fnMocks(String name) {
+    helper.callStack.findAll { it.methodName == name }
+  }
+
   void mockFailureBuild() {
     props([
       buildUtils: [

.ci/pipeline-library/src/test/slackNotifications.groovy

@@ -59,4 +59,67 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
       args.blocks[2].text.text.toString()
     )
   }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with a backup message when first attempt fails'() {
+    mockFailureBuild()
+    def counter = 0
+    helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 })
+    slackNotifications.sendFailedBuild()
+
+    def args = fnMocks('slackSend')[1].args[0]
+
+    def expected = [
+      channel: '#kibana-operations-alerts',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':broken_heart: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+
+    assertEquals(
+      ":broken_heart: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*" +
+        "\n\nFirst attempt at sending this notification failed. Please check the build.",
+      args.blocks[0].text.text.toString()
+    )
+  }
+
+  @Test
+  void 'getTestFailures() should truncate list of failures to 10'() {
+    prop('testUtils', [
+      getFailures: {
+        return (1..12).collect {
+          return [
+            url: Mocks.TEST_FAILURE_URL,
+            fullDisplayName: "Failure #${it}",
+          ]
+        }
+      },
+    ])
+
+    def message = (String) slackNotifications.getTestFailures()
+
+    assertTrue("Message ends with truncated indicator", message.endsWith("...and 2 more"))
+    assertTrue("Message contains Failure #10", message.contains("Failure #10"))
+    assertTrue("Message does not contain Failure #11", !message.contains("Failure #11"))
+  }
+
+  @Test
+  void 'shortenMessage() should truncate a long message, but leave parts that fit'() {
+    assertEquals('Hello\nHello\n[...truncated...]', slackNotifications.shortenMessage('Hello\nHello\nthis is a long string', 29))
+  }
+
+  @Test
+  void 'shortenMessage() should not modify a short message'() {
+    assertEquals('Hello world', slackNotifications.shortenMessage('Hello world', 11))
+  }
+
+  @Test
+  void 'shortenMessage() should truncate an entire message with only one part'() {
+    assertEquals('[...truncated...]', slackNotifications.shortenMessage('Hello world this is a really long message', 40))
+  }
 }

vars/slackNotifications.groovy

@@ -13,12 +13,35 @@ def dividerBlock() {
   return [ type: "divider" ]
 }
 
+// If a message is longer than the limit, split it up by '\n' into parts, and return as many parts as will fit within the limit
+def shortenMessage(message, sizeLimit = 3000) {
+  if (message.size() <= sizeLimit) {
+    return message
+  }
+
+  def truncatedMessage = "[...truncated...]"
+
+  def parts = message.split("\n")
+  message = ""
+
+  for(def part in parts) {
+    if ((message.size() + part.size() + truncatedMessage.size() + 1) > sizeLimit) {
+      break;
+    }
+    message += part+"\n"
+  }
+
+  message += truncatedMessage
+
+  return message.size() <= sizeLimit ? message : truncatedMessage
+}
+
 def markdownBlock(message) {
   return [
     type: "section",
     text: [
       type: "mrkdwn",
-      text: message,
+      text: shortenMessage(message, 3000), // 3000 is max text length for `section`s only
     ],
   ]
 }
@@ -29,7 +52,7 @@ def contextBlock(message) {
     elements: [
       [
         type: 'mrkdwn',
-        text: message,
+        text: message, // Not sure what the size limit is here, I tried 10000s of characters and it still worked
       ]
     ]
   ]
@@ -62,7 +85,7 @@ def getTestFailures() {
   def messages = []
   messages << "*Test Failures*"
 
-  def list = failures.collect {
+  def list = failures.take(10).collect {
     def name = it
       .fullDisplayName
       .split(/\./, 2)[-1]
@@ -73,7 +96,9 @@ def getTestFailures() {
 
     return "• <${it.url}|${name}>"
   }.join("\n")
-  return "*Test Failures*\n${list}"
+
+  def moreText = failures.size() > 10 ? "\n• ...and ${failures.size()-10} more" : ""
+  return "*Test Failures*\n${list}${moreText}"
 }
 
 def getDefaultDisplayName() {
@@ -98,6 +123,10 @@ def getStatusIcon() {
   return ':broken_heart:'
 }
 
+def getBackupMessage(config) {
+  return "${getStatusIcon()} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build."
+}
+
 def sendFailedBuild(Map params = [:]) {
   def config = [
     channel: '#kibana-operations-alerts',
@@ -117,7 +146,7 @@ def sendFailedBuild(Map params = [:]) {
   blocks << dividerBlock()
   blocks << config.context
 
-  slackSend(
+  def resp = slackSend(
     channel: config.channel,
     username: config.username,
     iconEmoji: config.icon,
@@ -125,6 +154,17 @@ def sendFailedBuild(Map params = [:]) {
     message: message,
     blocks: blocks
   )
+
+  if (!resp) {
+    slackSend(
+      channel: config.channel,
+      username: config.username,
+      iconEmoji: config.icon,
+      color: config.color,
+      message: message,
+      blocks: [markdownBlock(getBackupMessage(config))]
+    )
+  }
 }
 
 def onFailure(Map options = [:]) {