github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[CI] In-progress Slack notifications (#74012) (#74034)

Browse source
This commit is contained in:
Brian Seeders 2020-08-01 17:36:14 -04:00 • committed by GitHub
parent 59640715dc
commit 5801b5c014
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 183 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

64
.ci/pipeline-library/src/test/slackNotifications.groovy
View file

@ -9,6 +9,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
super.setUp()
helper.registerAllowedMethod('slackSend', [Map.class], null)
prop('buildState', loadScript("vars/buildState.groovy"))
slackNotifications = loadScript('vars/slackNotifications.groovy')
}
@ -25,13 +26,49 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
}
@Test
void 'sendFailedBuild() should call slackSend() with message'() {
void 'sendFailedBuild() should call slackSend() with an in-progress message'() {
mockFailureBuild()
slackNotifications.sendFailedBuild()
def args = fnMock('slackSend').args[0]
def expected = [
channel: '#kibana-operations-alerts',
username: 'Kibana Operations',
iconEmoji: ':jenkins:',
color: 'danger',
message: ':hourglass_flowing_sand: elastic / kibana # master #1',
]
expected.each {
assertEquals(it.value.toString(), args[it.key].toString())
}
assertEquals(
":hourglass_flowing_sand: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*",
args.blocks[0].text.text.toString()
)
assertEquals(
"*Failed Steps*\n• <http://jenkins.localhost:8080|Execute test task>",
args.blocks[1].text.text.toString()
)
assertEquals(
"*Test Failures*\n• <https://localhost/|x-pack/test/functional/apps/fake/test·ts.Fake test &lt;Component&gt; should &amp; pass &amp;>",
args.blocks[2].text.text.toString()
)
}
@Test
void 'sendFailedBuild() should call slackSend() with message'() {
mockFailureBuild()
slackNotifications.sendFailedBuild(isFinal: true)
def args = fnMock('slackSend').args[0]
def expected = [
channel: '#kibana-operations-alerts',
username: 'Kibana Operations',
@ -65,7 +102,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
mockFailureBuild()
def counter = 0
helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 })
slackNotifications.sendFailedBuild()
slackNotifications.sendFailedBuild(isFinal: true)
def args = fnMocks('slackSend')[1].args[0]
@ -88,6 +125,29 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
)
}
@Test
void 'sendFailedBuild() should call slackSend() with a channel id and timestamp on second call'() {
mockFailureBuild()
helper.registerAllowedMethod('slackSend', [Map.class], { [ channelId: 'CHANNEL_ID', ts: 'TIMESTAMP' ] })
slackNotifications.sendFailedBuild(isFinal: false)
slackNotifications.sendFailedBuild(isFinal: true)
def args = fnMocks('slackSend')[1].args[0]
def expected = [
channel: 'CHANNEL_ID',
timestamp: 'TIMESTAMP',
username: 'Kibana Operations',
iconEmoji: ':jenkins:',
color: 'danger',
message: ':broken_heart: elastic / kibana # master #1',
]
expected.each {
assertEquals(it.value.toString(), args[it.key].toString())
}
}
@Test
void 'getTestFailures() should truncate list of failures to 10'() {
prop('testUtils', [

95
Jenkinsfile vendored
View file

@ -4,59 +4,60 @@ library 'kibana-pipeline-library'
kibanaLibrary.load()
kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) {
githubPr.withDefaultPrComments {
ciStats.trackBuild {
catchError {
retryable.enable()
parallel([
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
// 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
]),
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'),
'xpack-securitySolutionCypress': { processNumber ->
whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/', 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/', 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx']) {
kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
}
},
slackNotifications.onFailure(disabled: !params.NOTIFY_ON_FAILURE) {
githubPr.withDefaultPrComments {
ciStats.trackBuild {
catchError {
retryable.enable()
parallel([
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
// 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
]),
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'),
'xpack-securitySolutionCypress': { processNumber ->
whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/', 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/', 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx']) {
kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
}
},
// 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
]),
])
// 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
]),
])
}
}
}
}
if (params.NOTIFY_ON_FAILURE) {
slackNotifications.onFailure()
kibanaPipeline.sendMail()
}
}

15
vars/githubPr.groovy
View file

@ -15,7 +15,7 @@
*/
def withDefaultPrComments(closure) {
catchErrors {
// sendCommentOnError() needs to know if comments are enabled, so lets track it with a global
// kibanaPipeline.notifyOnError() needs to know if comments are enabled, so lets track it with a global
// isPr() just ensures this functionality is skipped for non-PR builds
buildState.set('PR_COMMENTS_ENABLED', isPr())
catchErrors {
@ -59,19 +59,6 @@ def sendComment(isFinal = false) {
}
}
def sendCommentOnError(Closure closure) {
try {
closure()
} catch (ex) {
// If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure
currentBuild.result = 'FAILURE'
catchErrors {
sendComment(false)
}
throw ex
}
}
// Checks whether or not this currently executing build was triggered via a PR in the elastic/kibana repo
def isPr() {
return !!(env.ghprbPullId && env.ghprbPullLink && env.ghprbPullLink =~ /\/elastic\/kibana\//)

27
vars/kibanaPipeline.groovy
View file

@ -16,6 +16,25 @@ def withPostBuildReporting(Closure closure) {
}
}
def notifyOnError(Closure closure) {
try {
closure()
} catch (ex) {
// If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure
currentBuild.result = 'FAILURE'
catchErrors {
githubPr.sendComment(false)
}
catchErrors {
// an empty map is a valid config, but is falsey, so let's use .has()
if (buildState.has('SLACK_NOTIFICATION_CONFIG')) {
slackNotifications.sendFailedBuild(buildState.get('SLACK_NOTIFICATION_CONFIG'))
}
}
throw ex
}
}
def functionalTestProcess(String name, Closure closure) {
return { processNumber ->
def kibanaPort = "61${processNumber}1"
@ -35,7 +54,7 @@ def functionalTestProcess(String name, Closure closure) {
"JOB=${name}",
"KBN_NP_PLUGINS_BUILT=true",
]) {
githubPr.sendCommentOnError {
notifyOnError {
closure()
}
}
@ -165,7 +184,7 @@ def bash(script, label) {
}
def doSetup() {
githubPr.sendCommentOnError {
notifyOnError {
retryWithDelay(2, 15) {
try {
runbld("./test/scripts/jenkins_setup.sh", "Setup Build Environment and Dependencies")
@ -182,13 +201,13 @@ def doSetup() {
}
def buildOss() {
githubPr.sendCommentOnError {
notifyOnError {
runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana")
}
}
def buildXpack() {
githubPr.sendCommentOnError {
notifyOnError {
runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana")
}
}

58
vars/slackNotifications.groovy
View file

@ -105,16 +105,26 @@ def getDefaultDisplayName() {
return "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
}
def getDefaultContext() {
def duration = currentBuild.durationString.replace(' and counting', '')
def getDefaultContext(config = [:]) {
def progressMessage = ""
if (config && !config.isFinal) {
progressMessage = "In-progress"
} else {
def duration = currentBuild.durationString.replace(' and counting', '')
progressMessage = "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}"
}
return contextBlock([
"${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}",
progressMessage,
"<https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>",
].join(' · '))
}
def getStatusIcon() {
def getStatusIcon(config = [:]) {
if (config && !config.isFinal) {
return ':hourglass_flowing_sand:'
}
def status = buildUtils.getBuildStatus()
if (status == 'UNSTABLE') {
return ':yellow_heart:'
@ -124,7 +134,7 @@ def getStatusIcon() {
}
def getBackupMessage(config) {
return "${getStatusIcon()} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build."
return "${getStatusIcon(config)} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build."
}
def sendFailedBuild(Map params = [:]) {
@ -135,19 +145,32 @@ def sendFailedBuild(Map params = [:]) {
color: 'danger',
icon: ':jenkins:',
username: 'Kibana Operations',
context: getDefaultContext(),
isFinal: false,
] + params
def title = "${getStatusIcon()} ${config.title}"
def message = "${getStatusIcon()} ${config.message}"
config.context = config.context ?: getDefaultContext(config)
def title = "${getStatusIcon(config)} ${config.title}"
def message = "${getStatusIcon(config)} ${config.message}"
def blocks = [markdownBlock(title)]
getFailedBuildBlocks().each { blocks << it }
blocks << dividerBlock()
blocks << config.context
def channel = config.channel
def timestamp = null
def previousResp = buildState.get('SLACK_NOTIFICATION_RESPONSE')
if (previousResp) {
// When using `timestamp` to update a previous message, you have to use the channel ID from the previous response
channel = previousResp.channelId
timestamp = previousResp.ts
}
def resp = slackSend(
channel: config.channel,
channel: channel,
timestamp: timestamp,
username: config.username,
iconEmoji: config.icon,
color: config.color,
@ -156,7 +179,7 @@ def sendFailedBuild(Map params = [:]) {
)
if (!resp) {
slackSend(
resp = slackSend(
channel: config.channel,
username: config.username,
iconEmoji: config.icon,
@ -165,6 +188,10 @@ def sendFailedBuild(Map params = [:]) {
blocks: [markdownBlock(getBackupMessage(config))]
)
}
if (resp) {
buildState.set('SLACK_NOTIFICATION_RESPONSE', resp)
}
}
def onFailure(Map options = [:]) {
@ -172,6 +199,7 @@ def onFailure(Map options = [:]) {
def status = buildUtils.getBuildStatus()
if (status != "SUCCESS") {
catchErrors {
options.isFinal = true
sendFailedBuild(options)
}
}
@ -179,6 +207,16 @@ def onFailure(Map options = [:]) {
}
def onFailure(Map options = [:], Closure closure) {
if (options.disabled) {
catchError {
closure()
}
return
}
buildState.set('SLACK_NOTIFICATION_CONFIG', options)
// try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
catchError {
closure()

2
vars/workers.groovy
View file

@ -126,7 +126,7 @@ def intake(jobName, String script) {
return {
ci(name: jobName, size: 's-highmem', ramDisk: true) {
withEnv(["JOB=${jobName}"]) {
githubPr.sendCommentOnError {
kibanaPipeline.notifyOnError {
runbld(script, "Execute ${jobName}")
}
}