github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[7.x] Flaky test runner improvements/fixes (#49367) (#54591)

Browse source 
* Allocate multiple flaky test runner agents as executions grow

* WIP support for deleting kibana install dir during CI

* Add setup script for testing scripts

* Add REMOVE_KIBANA_INSTALL_DIR=1 to flaky test runner

* Change flaky test runner worker processes from 8 to 12
This commit is contained in:
Brian Seeders 2020-01-13 12:14:42 -05:00 • committed by GitHub
parent 53fd409a38
commit 5ffb19b7d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 54 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

38
.ci/Jenkinsfile_flaky
View file

@ -11,26 +11,28 @@ def IS_XPACK = JOB_PARTS[0] == 'xpack'
def JOB = JOB_PARTS[1]
def NEED_BUILD = JOB != 'serverMocha'
def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : ''
def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger()
def AGENT_COUNT = getAgentCount(EXECUTIONS)
def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)
def workerFailures = []
currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
currentBuild.description = "${params.CI_GROUP}<br />Executions: ${params.NUMBER_EXECUTIONS}"
// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents
// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions
def agentCount = 1
currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"
stage("Kibana Pipeline") {
timeout(time: 180, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
def agents = [:]
for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) {
for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
def agentNumberInside = agentNumber
def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
agents["agent-${agentNumber}"] = {
catchError {
print "Agent ${agentNumberInside} - ${agentExecutions} executions"
kibanaPipeline.withWorkers('flaky-test-runner', {
if (NEED_BUILD) {
if (!IS_XPACK) {
@ -42,7 +44,7 @@ stage("Kibana Pipeline") {
kibanaPipeline.buildXpack()
}
}
}, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))()
}, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
}
}
}
@ -92,9 +94,9 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
}
}
def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) {
def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) {
def workerMap = [:]
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers)
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses)
for(def i = 1; i <= numberOfWorkers; i++) {
def workerExecutions = floor(numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0))
@ -102,7 +104,10 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor
workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
for(def j = 0; j < workerExecutions; j++) {
print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) {
withEnv([
"JOB=agent-${agentNumber}-worker-${workerNumber}-${j}",
"REMOVE_KIBANA_INSTALL_DIR=1",
]) {
catchError {
try {
worker(workerNumber)
@ -119,6 +124,11 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor
return workerMap
}
def getAgentCount(executions) {
// Increase agent count every 24 worker processess, up to 3 agents maximum
return Math.min(3, 1 + floor(executions/24))
}
def trunc(str, length) {
if (str.size() >= length) {
return str.take(length) + "..."
@ -126,3 +136,11 @@ def trunc(str, length) {
return str;
}
// All of the real rounding/truncating methods are sandboxed
def floor(num) {
return num
.toString()
.split('\\.')[0]
.toInteger()
}

8
test/scripts/jenkins_ci_group.sh
View file

@ -1,12 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
export TEST_BROWSER_HEADLESS=1
source test/scripts/jenkins_test_setup.sh
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
yarn run grunt functionalTests:ensureAllTestsInCiGroup;

6
test/scripts/jenkins_firefox_smoke.sh
View file

@ -1,10 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node scripts/build --debug --oss;

20
test/scripts/jenkins_test_setup.sh Normal file
View file

@ -0,0 +1,20 @@
set -e
function post_work() {
set +e
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node "$KIBANA_DIR/scripts/report_failed_tests"
fi
if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then
rm -rf "$REMOVE_KIBANA_INSTALL_DIR"
fi
}
trap 'post_work' EXIT
export TEST_BROWSER_HEADLESS=1
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi

9
test/scripts/jenkins_visual_regression.sh
View file

@ -1,11 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh
source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
@ -22,8 +17,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi
export TEST_BROWSER_HEADLESS=1
checks-reporter-with-killswitch "Kibana visual regression tests" \
yarn run percy exec -t 500 \
node scripts/functional_tests \

8
test/scripts/jenkins_xpack_ci_group.sh
View file

@ -1,12 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
export TEST_BROWSER_HEADLESS=1
source test/scripts/jenkins_test_setup.sh
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
echo " -> Ensuring all functional tests are in a ciGroup"

8
test/scripts/jenkins_xpack_firefox_smoke.sh
View file

@ -1,10 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node scripts/build --debug --no-oss;
@ -21,8 +17,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi
export TEST_BROWSER_HEADLESS=1
cd "$XPACK_DIR"
checks-reporter-with-killswitch "X-Pack firefox smoke test" \

9
test/scripts/jenkins_xpack_visual_regression.sh
View file

@ -1,11 +1,6 @@
#!/usr/bin/env bash
set -e
if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh
source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
@ -23,8 +18,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi
export TEST_BROWSER_HEADLESS=1
cd "$XPACK_DIR"
checks-reporter-with-killswitch "X-Pack visual regression tests" \