[CI] Slack alerts - Elasticsearch snapshot failures (#64724) (#65273)

This commit is contained in:
Brian Seeders 2020-05-05 11:28:24 -04:00 committed by GitHub
parent be1575036f
commit 14252b6d88
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 150 additions and 35 deletions

View file

@ -23,42 +23,46 @@ timeout(time: 120, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
catchError {
retryable.enable(2)
withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
parallel([
// TODO we just need to run integration tests from intake?
'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
]),
'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
]),
])
slackNotifications.onFailure(
title: ":broken_heart: *<${env.BUILD_URL}|[${SNAPSHOT_VERSION}] ES Snapshot Verification Failure>*",
message: ":broken_heart: [${SNAPSHOT_VERSION}] ES Snapshot Verification Failure",
) {
retryable.enable(2)
withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
parallel([
// TODO we just need to run integration tests from intake?
'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
]),
'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
]),
])
}
promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
}
promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
}
kibanaPipeline.sendMail()

View file

@ -0,0 +1,111 @@
def getFailedBuildBlocks() {
def messages = [
getFailedSteps(),
getTestFailures(),
]
return messages
.findAll { !!it } // No blank strings
.collect { markdownBlock(it) }
}
def dividerBlock() {
return [ type: "divider" ]
}
def markdownBlock(message) {
return [
type: "section",
text: [
type: "mrkdwn",
text: message,
],
]
}
def contextBlock(message) {
return [
type: "context",
elements: [
[
type: 'mrkdwn',
text: message,
]
]
]
}
def getFailedSteps() {
try {
def steps = jenkinsApi.getFailedSteps()?.findAll { step ->
step.displayName != 'Check out from version control'
}
if (steps?.size() > 0) {
def list = steps.collect { "• <${it.logs}|${it.displayName}>" }.join("\n")
return "*Failed Steps*\n${list}"
}
} catch (ex) {
buildUtils.printStacktrace(ex)
print "Error retrieving failed pipeline steps for PR comment, will skip this section"
}
return ""
}
def getTestFailures() {
def failures = testUtils.getFailures()
if (!failures) {
return ""
}
def messages = []
messages << "*Test Failures*"
def list = failures.collect { "• <${it.url}|${it.fullDisplayName}>" }.join("\n")
return "*Test Failures*\n${list}"
}
def sendFailedBuild(Map params = [:]) {
def displayName = "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
def config = [
channel: '#kibana-operations',
title: ":broken_heart: *<${env.BUILD_URL}|${displayName}>*",
message: ":broken_heart: ${displayName}",
color: 'danger',
icon: ':jenkins:',
username: 'Kibana Operations',
context: contextBlock("${displayName} · <https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>"),
] + params
def blocks = [markdownBlock(config.title)]
getFailedBuildBlocks().each { blocks << it }
blocks << dividerBlock()
blocks << config.context
slackSend(
channel: config.channel,
username: config.username,
iconEmoji: config.icon,
color: config.color,
message: config.message,
blocks: blocks
)
}
def onFailure(Map options = [:], Closure closure) {
// try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
catchError {
closure()
}
def status = buildUtils.getBuildStatus()
if (status != "SUCCESS" && status != "UNSTABLE") {
catchErrors {
sendFailedBuild(options)
}
}
}
return this