Jenkins pipeline with parallel cigroups (#45285)

* Pipeline

* WIP some work for parallelization with ciGroups

* Fix xpack kibana install dir, and add some debugging

* Attempt to quick fix a few tests

* Revert "Revert "Revert "[ci] compress jobs for CI stability" (#44584)""

This reverts commit 078ac2897f.

* Recombine test groups, and try runbld again

* Mostly cleanup, and fix failed_tests reporting to hopefully work for both pipeline and non-pipeline

* Fix typo in shell script

* Remove some debug code

* Add support for changing es transport.port during testing via TEST_ES_TRANSPORT_PORT

* Fix test that uses hard-coded es transport port and add it back in to parallel groups

* Disable checks reporter again for now

* Set env var for TEST_ES_TRANSPORT_PORT in pipeline

* Update Jenkinsfile for shorter testrunner labels

* Fix another hard-coded transport port

* Fix a new test with hard-coded URLs

* Jenkinsfile cleanup and fix one of the groups

* Fix double slash

* Testing vault credentials on jenkins server

* Add a non-existent credential

* Revert "Add a non-existent credential"

This reverts commit 0dc234c465a5483b1a994cb510a182fef766e9cc.

* Try github-checks-reporter again

* github-checks-reporter should only run for elastic/kibana, forks won't work

* Clean up some debug code

* Changing names around to try to make BlueOcean UI a little better

* Add more stages

* Make some changes to stage structure to mirror a nested example from CloudBees

* Handle TODOs, and some cleanup in Jenkinsfile

* Pass GIT_BRANCH when started without GHPRB, fix branch check

* Fix mailer problem and add code that ensures all tests are in cigroups back in

* Test adding worker/job name to junit report paths

* Remove some duplication from ci_setup scripts

* Fix unit test that uses junit path

* Don't reinstall node every time setup_env is run

* Fix yarn install logic

* Fix another unit test that uses junit output dir

* Download latest ES snapshot after kibana builds

* Make sure junit reports are always processed

* Add two failing tests for testing purposes

* Add support to Jenkinsfile for kibana build e-mails

* Remove some debug code for email sending

* Change JOB env handling in junit paths and move it to a sub-directory

* Revert "Add two failing tests for testing purposes"

This reverts commit 5715203e26922a93483feb0ebb8bb3fdcc3daf8c.

* Fix junit report path in test

* Don't send kibana emails on build abort

* Address PR feedback, formatting and use built-in url formatting library

* Fix path formatting for functional test

* Add email sending back in to Jenkinsfile

* Fix another unit test with path problem

(cherry picked from commit 27d23c4184)

# Conflicts:
#	src/dev/ci_setup/setup.sh
#	src/dev/failed_tests/cli.js

.ci/jobs.yml

@@ -1,21 +1,34 @@
 JOB:
-  - intake
-  - firefoxSmoke
+  - kibana-intake
+  - x-pack-intake
+  - kibana-firefoxSmoke
   - kibana-ciGroup1
   - kibana-ciGroup2
   - kibana-ciGroup3
   - kibana-ciGroup4
   - kibana-ciGroup5
   - kibana-ciGroup6
-  # - kibana-visualRegression
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
+  - x-pack-firefoxSmoke
   - x-pack-ciGroup1
   - x-pack-ciGroup2
   - x-pack-ciGroup3
   - x-pack-ciGroup4
   - x-pack-ciGroup5
-  # - x-pack-visualRegression
+  - x-pack-ciGroup6
+  - x-pack-ciGroup7
+  - x-pack-ciGroup8
+  - x-pack-ciGroup9
+  - x-pack-ciGroup10
+  - x-pack-visualRegression
 
 # `~` is yaml for `null`
 exclude: ~

.ci/run.sh

@@ -11,7 +11,7 @@ source src/dev/ci_setup/setup.sh
 source src/dev/ci_setup/checkout_sibling_es.sh
 
 case "$JOB" in
-intake)
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
 kibana-ciGroup*)
@@ -21,9 +21,12 @@ kibana-ciGroup*)
 kibana-visualRegression*)
   ./test/scripts/jenkins_visual_regression.sh
   ;;
-firefoxSmoke*)
+kibana-firefoxSmoke*)
   ./test/scripts/jenkins_firefox_smoke.sh
   ;;
+x-pack-intake)
+  ./test/scripts/jenkins_xpack.sh
+  ;;
 x-pack-ciGroup*)
   export CI_GROUP="${JOB##x-pack-ciGroup}"
   ./test/scripts/jenkins_xpack_ci_group.sh
@@ -31,6 +34,9 @@ x-pack-ciGroup*)
 x-pack-visualRegression*)
   ./test/scripts/jenkins_xpack_visual_regression.sh
   ;;
+x-pack-firefoxSmoke*)
+  ./test/scripts/jenkins_xpack_firefox_smoke.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

Jenkinsfile

@@ -0,0 +1,268 @@
+#!/bin/groovy
+
+properties([
+  durabilityHint('PERFORMANCE_OPTIMIZED'),
+])
+
+stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        catchError {
+          parallel([
+            'kibana-intake-agent': legacyJobRunner('kibana-intake'),
+            'x-pack-intake-agent': legacyJobRunner('x-pack-intake'),
+            'kibana-oss-agent': withWorkers('kibana-oss-tests', { buildOss() }, [
+              'oss-ciGroup1': getOssCiGroupWorker(1),
+              'oss-ciGroup2': getOssCiGroupWorker(2),
+              'oss-ciGroup3': getOssCiGroupWorker(3),
+              'oss-ciGroup4': getOssCiGroupWorker(4),
+              'oss-ciGroup5': getOssCiGroupWorker(5),
+              'oss-ciGroup6': getOssCiGroupWorker(6),
+              'oss-ciGroup7': getOssCiGroupWorker(7),
+              'oss-ciGroup8': getOssCiGroupWorker(8),
+              'oss-ciGroup9': getOssCiGroupWorker(9),
+              'oss-ciGroup10': getOssCiGroupWorker(10),
+              'oss-ciGroup11': getOssCiGroupWorker(11),
+              'oss-ciGroup12': getOssCiGroupWorker(12),
+              'oss-visualRegression': getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
+              'oss-firefoxSmoke': getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }),
+            ]),
+            'kibana-xpack-agent': withWorkers('kibana-xpack-tests', { buildXpack() }, [
+              'xpack-ciGroup1': getXpackCiGroupWorker(1),
+              'xpack-ciGroup2': getXpackCiGroupWorker(2),
+              'xpack-ciGroup3': getXpackCiGroupWorker(3),
+              'xpack-ciGroup4': getXpackCiGroupWorker(4),
+              'xpack-ciGroup5': getXpackCiGroupWorker(5),
+              'xpack-ciGroup6': getXpackCiGroupWorker(6),
+              'xpack-ciGroup7': getXpackCiGroupWorker(7),
+              'xpack-ciGroup8': getXpackCiGroupWorker(8),
+              'xpack-ciGroup9': getXpackCiGroupWorker(9),
+              'xpack-ciGroup10': getXpackCiGroupWorker(10),
+              'xpack-firefoxSmoke': getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }),
+              'xpack-visualRegression': getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
+            ]),
+          ])
+        }
+        node('flyweight') {
+          sendMail()
+        }
+      }
+    }
+  }
+}
+
+def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
+  return {
+    jobRunner('tests-xl') {
+      try {
+        doSetup()
+        preWorkerClosure()
+
+        def nextWorker = 1
+        def worker = { workerClosure ->
+          def workerNumber = nextWorker
+          nextWorker++
+
+          return {
+            workerClosure(workerNumber)
+          }
+        }
+
+        def workers = [:]
+        workerClosures.each { workerName, workerClosure ->
+          workers[workerName] = worker(workerClosure)
+        }
+
+        parallel(workers)
+      } finally {
+        catchError {
+          uploadAllGcsArtifacts(name)
+        }
+
+        catchError {
+          publishJunit()
+        }
+      }
+    }
+  }
+}
+
+def getPostBuildWorker(name, closure) {
+  return { workerNumber ->
+    def kibanaPort = "61${workerNumber}1"
+    def esPort = "61${workerNumber}2"
+    def esTransportPort = "61${workerNumber}3"
+
+    withEnv([
+      "CI_WORKER_NUMBER=${workerNumber}",
+      "TEST_KIBANA_HOST=localhost",
+      "TEST_KIBANA_PORT=${kibanaPort}",
+      "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}",
+      "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}",
+      "TEST_ES_TRANSPORT_PORT=${esTransportPort}",
+      "IS_PIPELINE_JOB=1",
+    ]) {
+      closure()
+    }
+  }
+}
+
+def getOssCiGroupWorker(ciGroup) {
+  return getPostBuildWorker("ciGroup" + ciGroup, {
+    withEnv([
+      "CI_GROUP=${ciGroup}",
+      "JOB=kibana-ciGroup${ciGroup}",
+    ]) {
+      runbld "./test/scripts/jenkins_ci_group.sh"
+    }
+  })
+}
+
+def getXpackCiGroupWorker(ciGroup) {
+  return getPostBuildWorker("xpack-ciGroup" + ciGroup, {
+    withEnv([
+      "CI_GROUP=${ciGroup}",
+      "JOB=xpack-kibana-ciGroup${ciGroup}",
+    ]) {
+      runbld "./test/scripts/jenkins_xpack_ci_group.sh"
+    }
+  })
+}
+
+def legacyJobRunner(name) {
+  return {
+    parallel([
+      "${name}": {
+        withEnv([
+          "JOB=${name}",
+        ]) {
+          jobRunner('linux && immutable') {
+            try {
+              runbld '.ci/run.sh'
+            } finally {
+              catchError {
+                uploadAllGcsArtifacts(name)
+              }
+              catchError {
+                publishJunit()
+              }
+            }
+          }
+        }
+      }
+    ])
+  }
+}
+
+def jobRunner(label, closure) {
+  node(label) {
+    def scmVars = checkout scm
+
+    withEnv([
+      "CI=true",
+      "HOME=${env.JENKINS_HOME}",
+      "PR_SOURCE_BRANCH=${env.ghprbSourceBranch}",
+      "PR_TARGET_BRANCH=${env.ghprbTargetBranch}",
+      "PR_AUTHOR=${env.ghprbPullAuthorLogin}",
+      "TEST_BROWSER_HEADLESS=1",
+      "GIT_BRANCH=${scmVars.GIT_BRANCH}",
+    ]) {
+      withCredentials([
+        string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),
+        string(credentialsId: 'vault-role-id', variable: 'VAULT_ROLE_ID'),
+        string(credentialsId: 'vault-secret-id', variable: 'VAULT_SECRET_ID'),
+      ]) {
+        // scm is configured to check out to the ./kibana directory
+        dir('kibana') {
+          closure()
+        }
+      }
+    }
+  }
+}
+
+// TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build?
+
+def uploadGcsArtifact(workerName, pattern) {
+  def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" // TODO
+  // def storageLocation = "gs://kibana-pipeline-testing/jobs/pipeline-test/${BUILD_NUMBER}/${workerName}"
+
+  googleStorageUpload(
+    credentialsId: 'kibana-ci-gcs-plugin',
+    bucket: storageLocation,
+    pattern: pattern,
+    sharedPublicly: true,
+    showInline: true,
+  )
+}
+
+def uploadAllGcsArtifacts(workerName) {
+  def ARTIFACT_PATTERNS = [
+    'target/kibana-*',
+    'target/junit/**/*',
+    'test/**/screenshots/**/*.png',
+    'test/functional/failure_debug/html/*.html',
+    'x-pack/test/**/screenshots/**/*.png',
+    'x-pack/test/functional/failure_debug/html/*.html',
+    'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
+  ]
+
+  ARTIFACT_PATTERNS.each { pattern ->
+    uploadGcsArtifact(workerName, pattern)
+  }
+}
+
+def publishJunit() {
+  junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true)
+}
+
+def sendMail() {
+  sendInfraMail()
+  sendKibanaMail()
+}
+
+def sendInfraMail() {
+  catchError {
+    step([
+      $class: 'Mailer',
+      notifyEveryUnstableBuild: true,
+      recipients: 'infra-root+build@elastic.co',
+      sendToIndividuals: false
+    ])
+  }
+}
+
+def sendKibanaMail() {
+  catchError {
+    if(params.NOTIFY_ON_FAILURE && currentBuild.result != 'SUCCESS' && currentBuild.result != 'ABORTED') {
+      emailext(
+        // to: 'build-kibana@elastic.co',
+        to: 'brian.seeders@elastic.co', // TODO switch this out after testing
+        subject: "${env.PROJECT_NAME} - Build # ${env.BUILD_NUMBER} - ${currentBuild.result}",
+        body: '${SCRIPT,template="groovy-html.template"}',
+        mimeType: 'text/html',
+      )
+    }
+  }
+}
+
+def runbld(script) {
+  sh '#!/usr/local/bin/runbld\n' + script
+}
+
+def bash(script) {
+  sh "#!/bin/bash -x\n${script}"
+}
+
+def doSetup() {
+  runbld "./test/scripts/jenkins_setup.sh"
+}
+
+def buildOss() {
+  runbld "./test/scripts/jenkins_build_kibana.sh"
+}
+
+def buildXpack() {
+  runbld "./test/scripts/jenkins_xpack_build_kibana.sh"
+}

packages/kbn-test/src/es/es_test_cluster.js

@@ -54,6 +54,7 @@ export function createEsTestCluster(options = {}) {
     basePath,
     esArgs,
   };
+  const transportPort = esTestConfig.getTransportPort();
 
   const cluster = new Cluster({ log, ssl });
 
@@ -88,6 +89,7 @@ export function createEsTestCluster(options = {}) {
           `cluster.name=${clusterName}`,
           `http.port=${port}`,
           'discovery.type=single-node',
+          `transport.port=${transportPort}`,
           ...esArgs,
         ],
         esEnvVars,

packages/kbn-test/src/es/es_test_config.js

@@ -38,6 +38,10 @@ export const esTestConfig = new (class EsTestConfig {
     return process.env.TEST_ES_FROM || 'snapshot';
   }
 
+  getTransportPort() {
+    return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
+  }
+
   getUrlParts() {
     // Allow setting one complete TEST_ES_URL for Es like https://elastic:changeme@myCloudInstance:9200
     if (process.env.TEST_ES_URL) {

src/dev/ci_setup/get_percy_env.js

@@ -23,11 +23,7 @@ const pkg = require('../../../package.json');
 const { stdout: commit } = execa.sync('git', ['rev-parse', 'HEAD']);
 const shortCommit = commit.slice(0, 8);
 
-if (!process.env.JOB_NAME) {
-  throw new Error('getPercyEnv: [JOB_NAME] environment variable required');
-}
-
-const isPr = process.env.JOB_NAME.includes('elastic+kibana+pull-request');
+const isPr = !!process.env.ghprbPullId;
 if (isPr && !(process.env.PR_TARGET_BRANCH && process.env.PR_SOURCE_BRANCH)) {
   throw new Error(
     'getPercyEnv: Unable to determine percy environment in prs without [PR_TARGET_BRANCH] and [PR_SOURCE_BRANCH] environment variables'

src/dev/ci_setup/setup.sh

@@ -2,42 +2,7 @@
 
 set -e
 
-dir="$(pwd)"
-cacheDir="${CACHE_DIR:-"$HOME/.kibana"}"
-
-RED='\033[0;31m'
-C_RESET='\033[0m' # Reset color
-
-# force manually created 7.5 snapshots until live ones are available
-export KBN_ES_SNAPSHOT_URL="https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-7.5.0-SNAPSHOT-{os}-x86_64.{ext}"
-
-###
-### Since the Jenkins logging output collector doesn't look like a TTY
-### Node/Chalk and other color libs disable their color output. But Jenkins
-### can handle color fine, so this forces https://github.com/chalk/supports-color
-### to enable color support in Chalk and other related modules.
-###
-export FORCE_COLOR=1
-
-###
-### check that we seem to be in a kibana project
-###
-if [ -f "$dir/package.json" ] && [ -f "$dir/.node-version" ]; then
-  echo "Setting up node.js and yarn in $dir"
-else
-  echo "${RED}src/dev/ci_setup/setup.sh must be run within a kibana repo${C_RESET}"
-  exit 1
-fi
-
-
-export KIBANA_DIR="$dir"
-export XPACK_DIR="$KIBANA_DIR/x-pack"
-
-parentDir="$(cd "$KIBANA_DIR/.."; pwd)"
-export PARENT_DIR="$parentDir"
-
-kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")"
-export KIBANA_PKG_BRANCH="$kbnBranch"
+source src/dev/ci_setup/setup_env.sh true
 
 echo " -- KIBANA_DIR='$KIBANA_DIR'"
 echo " -- XPACK_DIR='$XPACK_DIR'"
@@ -45,76 +10,6 @@ echo " -- PARENT_DIR='$PARENT_DIR'"
 echo " -- KIBANA_PKG_BRANCH='$KIBANA_PKG_BRANCH'"
 echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'"
 
-###
-### download node
-###
-UNAME=$(uname)
-OS="linux"
-if [[ "$UNAME" = *"MINGW64_NT"* ]]; then
-  OS="win"
-fi
-echo " -- Running on OS: $OS"
-
-nodeVersion="$(cat "$dir/.node-version")"
-nodeDir="$cacheDir/node/$nodeVersion"
-
-if [[ "$OS" == "win" ]]; then
-  nodeBin="$HOME/node"
-  nodeUrl="https://nodejs.org/dist/v$nodeVersion/node-v$nodeVersion-win-x64.zip"
-else
-  nodeBin="$nodeDir/bin"
-  nodeUrl="https://nodejs.org/dist/v$nodeVersion/node-v$nodeVersion-linux-x64.tar.gz"
-fi
-
-echo " -- node: version=v${nodeVersion} dir=$nodeDir"
-
-echo " -- setting up node.js"
-if [ -x "$nodeBin/node" ] && [ "$("$nodeBin/node" --version)" == "v$nodeVersion" ]; then
-  echo " -- reusing node.js install"
-else
-  if [ -d "$nodeDir" ]; then
-    echo " -- clearing previous node.js install"
-    rm -rf "$nodeDir"
-  fi
-
-  echo " -- downloading node.js from $nodeUrl"
-  mkdir -p "$nodeDir"
-  if [[ "$OS" == "win" ]]; then
-    nodePkg="$nodeDir/${nodeUrl##*/}"
-    curl --silent -o "$nodePkg" "$nodeUrl"
-    unzip -qo "$nodePkg" -d "$nodeDir"
-    mv "${nodePkg%.*}" "$nodeBin"
-  else
-    curl --silent "$nodeUrl" | tar -xz -C "$nodeDir" --strip-components=1
-  fi
-fi
-
-###
-### "install" node into this shell
-###
-export PATH="$nodeBin:$PATH"
-
-###
-### downloading yarn
-###
-yarnVersion="$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")"
-npm install -g "yarn@^${yarnVersion}"
-
-###
-### setup yarn offline cache
-###
-yarn config set yarn-offline-mirror "$cacheDir/yarn-offline-cache"
-
-###
-### "install" yarn into this shell
-###
-yarnGlobalDir="$(yarn global bin)"
-export PATH="$PATH:$yarnGlobalDir"
-
-# use a proxy to fetch chromedriver/geckodriver asset
-export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-
 ###
 ### install dependencies
 ###
@@ -148,26 +43,17 @@ if [ "$GIT_CHANGES" ]; then
 fi
 
 ###
-### github-checks-reporter kill switch. Remove to disable
+### rebuild kbn-pm distributable to ensure it's not out of date
 ###
-export CHECKS_REPORTER_ACTIVE=true
+echo " -- building renovate config"
+node scripts/build_renovate_config
 
-### only run on pr jobs
-if [[ "$JOB_NAME" != "elastic+kibana+pull-request"* ]] ; then
-  export CHECKS_REPORTER_ACTIVE=false
+###
+### verify no git modifications
+###
+GIT_CHANGES="$(git ls-files --modified)"
+if [ "$GIT_CHANGES" ]; then
+  echo -e "\n${RED}ERROR: 'node scripts/build_renovate_config' caused changes to the following files:${C_RESET}\n"
+  echo -e "$GIT_CHANGES\n"
+  exit 1
 fi
-
-###
-### Implements github-checks-reporter kill switch when scripts are called from the command line
-### $@ - all arguments
-###
-function checks-reporter-with-killswitch() {
-  if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
-    yarn run github-checks-reporter "$@"
-  else
-    arguments=("$@");
-    "${arguments[@]:1}";
-  fi
-}
-
-export -f checks-reporter-with-killswitch

src/dev/ci_setup/setup_env.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -e
+
+installNode=$1
+
+dir="$(pwd)"
+cacheDir="${CACHE_DIR:-"$HOME/.kibana"}"
+
+RED='\033[0;31m'
+C_RESET='\033[0m' # Reset color
+
+###
+### Since the Jenkins logging output collector doesn't look like a TTY
+### Node/Chalk and other color libs disable their color output. But Jenkins
+### can handle color fine, so this forces https://github.com/chalk/supports-color
+### to enable color support in Chalk and other related modules.
+###
+export FORCE_COLOR=1
+
+###
+### check that we seem to be in a kibana project
+###
+if [ -f "$dir/package.json" ] && [ -f "$dir/.node-version" ]; then
+  echo "Setting up node.js and yarn in $dir"
+else
+  echo "${RED}src/dev/ci_setup/setup.sh must be run within a kibana repo${C_RESET}"
+  exit 1
+fi
+
+
+export KIBANA_DIR="$dir"
+export XPACK_DIR="$KIBANA_DIR/x-pack"
+
+parentDir="$(cd "$KIBANA_DIR/.."; pwd)"
+export PARENT_DIR="$parentDir"
+
+kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")"
+export KIBANA_PKG_BRANCH="$kbnBranch"
+
+###
+### download node
+###
+UNAME=$(uname)
+OS="linux"
+if [[ "$UNAME" = *"MINGW64_NT"* ]]; then
+  OS="win"
+fi
+echo " -- Running on OS: $OS"
+
+nodeVersion="$(cat "$dir/.node-version")"
+nodeDir="$cacheDir/node/$nodeVersion"
+
+if [[ "$OS" == "win" ]]; then
+  nodeBin="$HOME/node"
+  nodeUrl="https://nodejs.org/dist/v$nodeVersion/node-v$nodeVersion-win-x64.zip"
+else
+  nodeBin="$nodeDir/bin"
+  nodeUrl="https://nodejs.org/dist/v$nodeVersion/node-v$nodeVersion-linux-x64.tar.gz"
+fi
+
+if [[ "$installNode" == "true" ]]; then
+  echo " -- node: version=v${nodeVersion} dir=$nodeDir"
+
+  echo " -- setting up node.js"
+  if [ -x "$nodeBin/node" ] && [ "$("$nodeBin/node" --version)" == "v$nodeVersion" ]; then
+    echo " -- reusing node.js install"
+  else
+    if [ -d "$nodeDir" ]; then
+      echo " -- clearing previous node.js install"
+      rm -rf "$nodeDir"
+    fi
+
+    echo " -- downloading node.js from $nodeUrl"
+    mkdir -p "$nodeDir"
+    if [[ "$OS" == "win" ]]; then
+      nodePkg="$nodeDir/${nodeUrl##*/}"
+      curl --silent -o "$nodePkg" "$nodeUrl"
+      unzip -qo "$nodePkg" -d "$nodeDir"
+      mv "${nodePkg%.*}" "$nodeBin"
+    else
+      curl --silent "$nodeUrl" | tar -xz -C "$nodeDir" --strip-components=1
+    fi
+  fi
+fi
+
+###
+### "install" node into this shell
+###
+export PATH="$nodeBin:$PATH"
+
+if [[ "$installNode" == "true" || ! $(which yarn) ]]; then
+  ###
+  ### downloading yarn
+  ###
+  yarnVersion="$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")"
+  npm install -g "yarn@^${yarnVersion}"
+fi
+
+###
+### setup yarn offline cache
+###
+yarn config set yarn-offline-mirror "$cacheDir/yarn-offline-cache"
+
+###
+### "install" yarn into this shell
+###
+yarnGlobalDir="$(yarn global bin)"
+export PATH="$PATH:$yarnGlobalDir"
+
+# use a proxy to fetch chromedriver/geckodriver asset
+export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+
+
+export CHECKS_REPORTER_ACTIVE=false
+
+### only run on pr jobs for elastic/kibana, checks-reporter doesn't work for other repos
+if [[ "$ghprbPullId" && "$ghprbGhRepository" == 'elastic/kibana' ]] ; then
+  export CHECKS_REPORTER_ACTIVE=true
+fi
+
+###
+### Implements github-checks-reporter kill switch when scripts are called from the command line
+### $@ - all arguments
+###
+function checks-reporter-with-killswitch() {
+  if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
+    yarn run github-checks-reporter "$@"
+  else
+    arguments=("$@");
+    "${arguments[@]:1}";
+  fi
+}
+
+export -f checks-reporter-with-killswitch
+
+source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh"

src/dev/failed_tests/cli.js

@@ -22,16 +22,22 @@ const { resolve } = require('path');
 // force cwd
 process.chdir(resolve(__dirname, '../../..'));
 
-if (!process.env.JOB_NAME) {
-  console.log('Unable to determine job name');
+// JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
+
+const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
+const branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
+
+const isPr = !!process.env.ghprbPullId;
+
+if (!branch) {
+  console.log('Unable to determine originating branch from job name or other environment variables');
   process.exit(1);
 }
 
-// JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
-const [org, proj, branch] = process.env.JOB_NAME.split(/\+|\//);
-const masterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
-if (!(org === 'elastic' && proj === 'kibana' && masterOrVersion)) {
-  console.log(`Failure issues only created on master/version branch jobs [JOB_NAME=${process.env.JOB_NAME}] [${org}/${proj}/${branch}]`);
+const isMasterOrVersion = branch.match(/^(origin\/){0,1}master$/) || branch.match(/^(origin\/){0,1}\d+\.(x|\d+)$/);
+
+if (!isMasterOrVersion || isPr) {
+  console.log('Failure issues only created on master/version branch jobs');
   process.exit(0);
 }
 

src/dev/jest/integration_tests/junit_reporter.test.js

@@ -29,7 +29,7 @@ const MINUTE = 1000 * 60;
 const ROOT_DIR = resolve(__dirname, '../../../../');
 const FIXTURE_DIR = resolve(__dirname, '__fixtures__');
 const TARGET_DIR = resolve(FIXTURE_DIR, 'target');
-const XML_PATH = resolve(TARGET_DIR, 'junit/TEST-Jest Tests.xml');
+const XML_PATH = resolve(TARGET_DIR, 'junit', process.env.JOB || '.', 'TEST-Jest Tests.xml');
 
 afterAll(async () => {
   await del(TARGET_DIR);

src/dev/jest/junit_reporter.js

@@ -33,10 +33,7 @@ const ROOT_DIR = dirname(require.resolve('../../../package.json'));
  */
 export default class JestJUnitReporter {
   constructor(globalConfig, options = {}) {
-    const {
-      reportName = 'Jest Tests',
-      rootDirectory = ROOT_DIR
-    } = options;
+    const { reportName = 'Jest Tests', rootDirectory = ROOT_DIR } = options;
 
     this._reportName = reportName;
     this._rootDirectory = resolve(rootDirectory);
@@ -62,8 +59,8 @@ export default class JestJUnitReporter {
       { skipNullAttributes: true }
     );
 
-    const msToIso = ms => ms ? new Date(ms).toISOString().slice(0, -5) : undefined;
-    const msToSec = ms => ms ? (ms / 1000).toFixed(3) : undefined;
+    const msToIso = ms => (ms ? new Date(ms).toISOString().slice(0, -5) : undefined);
+    const msToSec = ms => (ms ? (ms / 1000).toFixed(3) : undefined);
 
     root.att({
       name: 'jest',
@@ -83,7 +80,7 @@ export default class JestJUnitReporter {
         tests: suite.testResults.length,
         failures: suite.numFailingTests,
         skipped: suite.numPendingTests,
-        file: suite.testFilePath
+        file: suite.testFilePath,
       });
 
       // nested in there are the tests in that file
@@ -93,10 +90,10 @@ export default class JestJUnitReporter {
         const testEl = suiteEl.ele('testcase', {
           classname,
           name: [...test.ancestorTitles, test.title].join(' '),
-          time: msToSec(test.duration)
+          time: msToSec(test.duration),
         });
 
-        test.failureMessages.forEach((message) => {
+        test.failureMessages.forEach(message => {
           testEl.ele('failure').dat(escapeCdata(message));
         });
 
@@ -106,12 +103,18 @@ export default class JestJUnitReporter {
       });
     });
 
-    const reportPath = resolve(rootDirectory, `target/junit/TEST-${reportName}.xml`);
+    const reportPath = resolve(
+      rootDirectory,
+      'target/junit',
+      process.env.JOB || '.',
+      `TEST-${reportName}.xml`
+    );
+
     const reportXML = root.end({
       pretty: true,
       indent: '  ',
       newline: '\n',
-      spacebeforeslash: ''
+      spacebeforeslash: '',
     });
 
     mkdirp.sync(dirname(reportPath));

src/dev/mocha/__tests__/junit_report_generation.js

@@ -50,7 +50,10 @@ describe('dev/mocha/junit report generation', () => {
     mocha.addFile(resolve(PROJECT_DIR, 'test.js'));
     await new Promise(resolve => mocha.run(resolve));
     const report = await fcb(cb =>
-      parseString(readFileSync(resolve(PROJECT_DIR, 'target/junit/TEST-test.xml')), cb)
+      parseString(
+        readFileSync(resolve(PROJECT_DIR, 'target/junit', process.env.JOB || '.', 'TEST-test.xml')),
+        cb
+      )
     );
 
     // test case results are wrapped in <testsuites></testsuites>

src/dev/mocha/junit_report_generation.js

@@ -136,7 +136,13 @@ export function setupJUnitReportGeneration(runner, options = {}) {
       }
     });
 
-    const reportPath = resolve(rootDirectory, `target/junit/TEST-${reportName}.xml`);
+    const reportPath = resolve(
+      rootDirectory,
+      'target/junit',
+      process.env.JOB || '.',
+      `TEST-${reportName}.xml`
+    );
+
     const reportXML = builder.end({
       pretty: true,
       indent: '  ',

src/dev/precommit_hook/casing_check_config.js

@@ -43,6 +43,8 @@ export const IGNORE_FILE_GLOBS = [
   'x-pack/docs/**/*',
   'src/legacy/ui/public/assets/fonts/**/*',
   'packages/kbn-utility-types/test-d/**/*',
+  'Jenkinsfile',
+  'Dockerfile*',
 
   // Files in this directory must match a pre-determined name in some cases.
   'x-pack/legacy/plugins/canvas/.storybook/*',

tasks/config/karma.js

@@ -59,11 +59,7 @@ module.exports = function (grunt) {
       customLaunchers: {
         Chrome_Headless: {
           base: 'Chrome',
-          flags: [
-            '--headless',
-            '--disable-gpu',
-            '--remote-debugging-port=9222',
-          ],
+          flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222'],
         },
       },
 
@@ -71,16 +67,13 @@ module.exports = function (grunt) {
       reporters: process.env.CI ? ['dots', 'junit'] : ['progress'],
 
       junitReporter: {
-        outputFile: resolve(ROOT, 'target/junit/TEST-karma.xml'),
+        outputFile: resolve(ROOT, 'target/junit', process.env.JOB || '.', 'TEST-karma.xml'),
         useBrowserName: false,
-        nameFormatter: (browser, result) => [
-          ...result.suite,
-          result.description
-        ].join(' '),
+        nameFormatter: (browser, result) => [...result.suite, result.description].join(' '),
         classNameFormatter: (browser, result) => {
           const rootSuite = result.suite[0] || result.description;
           return `Browser Unit Tests.${rootSuite.replace(/\./g, '·')}`;
-        }
+        },
       },
 
       // list of files / patterns to load in the browser
@@ -98,16 +91,16 @@ module.exports = function (grunt) {
         '/tests/': 'http://localhost:5610/tests/',
         '/bundles/': 'http://localhost:5610/bundles/',
         '/built_assets/dlls/': 'http://localhost:5610/built_assets/dlls/',
-        '/test_bundle/': 'http://localhost:5610/test_bundle/'
+        '/test_bundle/': 'http://localhost:5610/test_bundle/',
       },
 
       client: {
         mocha: {
           reporter: 'html', // change Karma's debug.html to the mocha web reporter
           timeout: 10000,
-          slow: 5000
-        }
-      }
+          slow: 5000,
+        },
+      },
     },
 
     dev: { singleRun: false },
@@ -116,11 +109,8 @@ module.exports = function (grunt) {
       singleRun: true,
       reporters: ['coverage'],
       coverageReporter: {
-        reporters: [
-          { type: 'html', dir: 'coverage' },
-          { type: 'text-summary' },
-        ]
-      }
+        reporters: [{ type: 'html', dir: 'coverage' }, { type: 'text-summary' }],
+      },
     },
   };
 
@@ -186,8 +176,8 @@ module.exports = function (grunt) {
 
           'http://localhost:5610/built_assets/dlls/vendors.style.dll.css',
           'http://localhost:5610/bundles/tests.style.css',
-        ]
-      }
+        ],
+      },
     };
   });
 

tasks/config/run.js

@@ -21,7 +21,7 @@ import { resolve } from 'path';
 import { getFunctionalTestGroupRunConfigs } from '../function_test_groups';
 
 const { version } = require('../../package.json');
-const KIBANA_INSTALL_DIR = `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`;
+const KIBANA_INSTALL_DIR = process.env.KIBANA_INSTALL_DIR || `./build/oss/kibana-${version}-SNAPSHOT-${process.platform}-x86_64`;
 
 module.exports = function (grunt) {
 

test/functional/apps/home/_navigation.js

@@ -73,14 +73,18 @@ export default function ({ getService, getPageObjects }) {
     });
 
     it('encodes portions of the URL as necessary', async () => {
-      await browser.get('http://localhost:5620/app/kibana#/home', false);
+      const host = process.env.TEST_KIBANA_HOST || 'localhost';
+      const port = process.env.TEST_KIBANA_PORT || '5620';
+      const basePath = `http://${host}:${port}`;
+
+      await browser.get(`${basePath}/app/kibana#/home`, false);
       await retry.waitFor('navigation to home app', async () => (
-        (await browser.getCurrentUrl()) === 'http://localhost:5620/app/kibana#/home?_g=()'
+        (await browser.getCurrentUrl()) === `${basePath}/app/kibana#/home?_g=()`
       ));
 
-      await browser.get('http://localhost:5620/app/kibana#/home?_g=()&a=b/c', false);
+      await browser.get(`${basePath}/app/kibana#/home?_g=()&a=b/c`, false);
       await retry.waitFor('hash to be properly encoded', async () => (
-        (await browser.getCurrentUrl()) === 'http://localhost:5620/app/kibana#/home?_g=()&a=b%2Fc'
+        (await browser.getCurrentUrl()) === `${basePath}/app/kibana#/home?_g=()&a=b%2Fc`
       ));
     });
   });

test/functional/apps/management/index.js

@@ -33,7 +33,7 @@ export default function ({ getService, loadTestFile }) {
     });
 
     describe('', function () {
-      this.tags('ciGroup1');
+      this.tags('ciGroup7');
 
       loadTestFile(require.resolve('./_create_index_pattern_wizard'));
       loadTestFile(require.resolve('./_index_pattern_create_delete'));
@@ -45,7 +45,7 @@ export default function ({ getService, loadTestFile }) {
     });
 
     describe('', function () {
-      this.tags('ciGroup2');
+      this.tags('ciGroup8');
 
       loadTestFile(require.resolve('./_index_pattern_filter'));
       loadTestFile(require.resolve('./_scripted_fields_filter'));

test/functional/apps/visualize/index.ts

@@ -40,7 +40,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function() {
-      this.tags('ciGroup3');
+      this.tags('ciGroup9');
 
       loadTestFile(require.resolve('./_embedding_chart'));
       loadTestFile(require.resolve('./_chart_types'));
@@ -50,7 +50,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function() {
-      this.tags('ciGroup4');
+      this.tags('ciGroup10');
 
       loadTestFile(require.resolve('./_inspector'));
       loadTestFile(require.resolve('./_experimental_vis'));
@@ -62,7 +62,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function() {
-      this.tags('ciGroup5');
+      this.tags('ciGroup11');
 
       loadTestFile(require.resolve('./_line_chart'));
       loadTestFile(require.resolve('./_pie_chart'));
@@ -76,7 +76,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function() {
-      this.tags('ciGroup6');
+      this.tags('ciGroup12');
 
       loadTestFile(require.resolve('./_tag_cloud'));
       loadTestFile(require.resolve('./_tile_map'));

test/mocha_decorations.d.ts

@@ -26,12 +26,12 @@ type Tags =
   | 'ciGroup4'
   | 'ciGroup5'
   | 'ciGroup6'
-  | 'ciGroup1'
-  | 'ciGroup2'
-  | 'ciGroup3'
-  | 'ciGroup4'
-  | 'ciGroup5'
-  | 'ciGroup6';
+  | 'ciGroup7'
+  | 'ciGroup8'
+  | 'ciGroup9'
+  | 'ciGroup10'
+  | 'ciGroup11'
+  | 'ciGroup12';
 
 // We need to use the namespace here to match the Mocha definition
 // eslint-disable-next-line @typescript-eslint/no-namespace

test/plugin_functional/test_suites/core_plugins/applications.js

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+import url from 'url';
 import expect from '@kbn/expect';
 
 export default function ({ getService, getPageObjects }) {
@@ -32,6 +32,14 @@ export default function ({ getService, getPageObjects }) {
   const loadingScreenShown = () =>
     testSubjects.existOrFail('kbnLoadingMessage');
 
+  const getKibanaUrl = (pathname, search) => url.format({
+    protocol: 'http:',
+    hostname: process.env.TEST_KIBANA_HOST || 'localhost',
+    port: process.env.TEST_KIBANA_PORT || '5620',
+    pathname,
+    search,
+  });
+
   describe('ui applications', function describeIndexTests() {
     before(async () => {
       await PageObjects.common.navigateToApp('foo');
@@ -44,20 +52,20 @@ export default function ({ getService, getPageObjects }) {
     it('navigates to its own pages', async () => {
       // Go to page A
       await testSubjects.click('fooNavPageA');
-      expect(await browser.getCurrentUrl()).to.eql(`http://localhost:5620/app/foo/page-a`);
+      expect(await browser.getCurrentUrl()).to.eql(getKibanaUrl('/app/foo/page-a'));
       await loadingScreenNotShown();
       await testSubjects.existOrFail('fooAppPageA');
 
       // Go to home page
       await testSubjects.click('fooNavHome');
-      expect(await browser.getCurrentUrl()).to.eql(`http://localhost:5620/app/foo/`);
+      expect(await browser.getCurrentUrl()).to.eql(getKibanaUrl('/app/foo/'));
       await loadingScreenNotShown();
       await testSubjects.existOrFail('fooAppHome');
     });
 
     it('can use the back button to navigate within an app', async () => {
       await browser.goBack();
-      expect(await browser.getCurrentUrl()).to.eql(`http://localhost:5620/app/foo/page-a`);
+      expect(await browser.getCurrentUrl()).to.eql(getKibanaUrl('/app/foo/page-a'));
       await loadingScreenNotShown();
       await testSubjects.existOrFail('fooAppPageA');
     });
@@ -66,7 +74,7 @@ export default function ({ getService, getPageObjects }) {
       await testSubjects.click('fooNavBarPageB');
       await loadingScreenNotShown();
       await testSubjects.existOrFail('barAppPageB');
-      expect(await browser.getCurrentUrl()).to.eql(`http://localhost:5620/app/bar/page-b?query=here`);
+      expect(await browser.getCurrentUrl()).to.eql(getKibanaUrl('/app/bar/page-b', 'query=here'));
     });
 
     it('preserves query parameters across apps', async () => {
@@ -76,7 +84,7 @@ export default function ({ getService, getPageObjects }) {
 
     it('can use the back button to navigate back to previous app', async () => {
       await browser.goBack();
-      expect(await browser.getCurrentUrl()).to.eql(`http://localhost:5620/app/foo/page-a`);
+      expect(await browser.getCurrentUrl()).to.eql(getKibanaUrl('/app/foo/page-a'));
       await loadingScreenNotShown();
       await testSubjects.existOrFail('fooAppPageA');
     });

test/scripts/jenkins_build_kibana.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+yarn run grunt functionalTests:ensureAllTestsInCiGroup;
+node scripts/build --debug --oss
+
+node scripts/es snapshot --license=oss --download-only;

test/scripts/jenkins_ci_group.sh

@@ -3,12 +3,23 @@
 set -e
 trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
-yarn run grunt functionalTests:ensureAllTestsInCiGroup;
-
-node scripts/build --debug --oss;
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
 
 export TEST_BROWSER_HEADLESS=1
 
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  yarn run grunt functionalTests:ensureAllTestsInCiGroup;	
+  node scripts/build --debug --oss;
+else
+  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+  destDir=${installDir}-${CI_WORKER_NUMBER}
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
+
 checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
 
 if [ "$CI_GROUP" == "1" ]; then

test/scripts/jenkins_firefox_smoke.sh

@@ -3,11 +3,23 @@
 set -e
 trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
-node scripts/build --debug --oss;
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  node scripts/build --debug --oss;
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+else
+  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+  destDir=${installDir}-${CI_WORKER_NUMBER}
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
 
 export TEST_BROWSER_HEADLESS=1
 
@@ -17,5 +29,3 @@ checks-reporter-with-killswitch "Firefox smoke test" \
     --kibana-install-dir "$installDir" \
     --include-tag "smoke" \
     --config test/functional/config.firefox.js;
-
-source "$KIBANA_DIR/test/scripts/jenkins_xpack_firefox_smoke.sh"

test/scripts/jenkins_setup.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/load_env_keys.sh
+source src/dev/ci_setup/extract_bootstrap_cache.sh
+source src/dev/ci_setup/setup.sh
+source src/dev/ci_setup/checkout_sibling_es.sh

test/scripts/jenkins_unit.sh

@@ -6,5 +6,3 @@ trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 export TEST_BROWSER_HEADLESS=1
 
 "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev;
-
-source "$KIBANA_DIR/test/scripts/jenkins_xpack.sh"

test/scripts/jenkins_visual_regression.sh

@@ -3,13 +3,25 @@
 set -e
 trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
 source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
-node scripts/build --debug --oss;
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  node scripts/build --debug --oss;
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+else
+  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+  destDir=${installDir}-${CI_WORKER_NUMBER}
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
 
 export TEST_BROWSER_HEADLESS=1
 

test/scripts/jenkins_xpack_build_kibana.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+echo " -> building and extracting default Kibana distributable for use in functional tests"
+cd "$KIBANA_DIR"
+source src/dev/ci_setup/setup_env.sh
+
+echo " -> Ensuring all functional tests are in a ciGroup"
+cd "$XPACK_DIR"
+node scripts/functional_tests --assert-none-excluded \
+  --include-tag ciGroup1 \
+  --include-tag ciGroup2 \
+  --include-tag ciGroup3 \
+  --include-tag ciGroup4 \
+  --include-tag ciGroup5 \
+  --include-tag ciGroup6 \
+  --include-tag ciGroup7 \
+  --include-tag ciGroup8 \
+  --include-tag ciGroup9 \
+  --include-tag ciGroup10
+
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+node scripts/es snapshot --download-only;

test/scripts/jenkins_xpack_ci_group.sh

@@ -5,27 +5,46 @@ trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
 export TEST_BROWSER_HEADLESS=1
 
-echo " -> Ensuring all functional tests are in a ciGroup"
-cd "$XPACK_DIR"
-node scripts/functional_tests --assert-none-excluded \
-  --include-tag ciGroup1 \
-  --include-tag ciGroup2 \
-  --include-tag ciGroup3 \
-  --include-tag ciGroup4 \
-  --include-tag ciGroup5 \
-  --include-tag ciGroup6 \
-  --include-tag ciGroup1 \
-  --include-tag ciGroup2 \
-  --include-tag ciGroup3 \
-  --include-tag ciGroup4
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  echo " -> Ensuring all functional tests are in a ciGroup"
+  cd "$XPACK_DIR"
+  node scripts/functional_tests --assert-none-excluded \
+    --include-tag ciGroup1 \
+    --include-tag ciGroup2 \
+    --include-tag ciGroup3 \
+    --include-tag ciGroup4 \
+    --include-tag ciGroup5 \
+    --include-tag ciGroup6 \
+    --include-tag ciGroup7 \
+    --include-tag ciGroup8 \
+    --include-tag ciGroup9 \
+    --include-tag ciGroup10
+fi
 
-echo " -> building and extracting default Kibana distributable for use in functional tests"
 cd "$KIBANA_DIR"
-node scripts/build --debug --no-oss
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  echo " -> building and extracting default Kibana distributable for use in functional tests"
+  node scripts/build --debug --no-oss
+
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+  export KIBANA_INSTALL_DIR="$installDir"
+else
+  installDir="$PARENT_DIR/install/kibana"
+  destDir="${installDir}-${CI_WORKER_NUMBER}"
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
 
 echo " -> Running functional and api tests"
 cd "$XPACK_DIR"
@@ -33,7 +52,7 @@ cd "$XPACK_DIR"
 checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
   node scripts/functional_tests \
     --debug --bail \
-    --kibana-install-dir "$installDir" \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
     --include-tag "ciGroup$CI_GROUP"
 
 echo ""

test/scripts/jenkins_xpack_firefox_smoke.sh

@@ -3,11 +3,24 @@
 set -e
 trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
-node scripts/build --debug --no-oss;
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  node scripts/build --debug --no-oss;
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+  export KIBANA_INSTALL_DIR="$installDir"
+else
+  installDir="$PARENT_DIR/install/kibana"
+  destDir="${installDir}-${CI_WORKER_NUMBER}"
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
 
 export TEST_BROWSER_HEADLESS=1
 
@@ -16,6 +29,6 @@ cd "$XPACK_DIR"
 checks-reporter-with-killswitch "X-Pack firefox smoke test" \
   node scripts/functional_tests \
     --debug --bail \
-    --kibana-install-dir "$installDir" \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
     --include-tag "smoke" \
     --config test/functional/config.firefox.js;

test/scripts/jenkins_xpack_visual_regression.sh

@@ -3,13 +3,26 @@
 set -e
 trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
 
+if [[ "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
 source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
-node scripts/build --debug --no-oss;
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  node scripts/build --debug --no-oss;
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+  export KIBANA_INSTALL_DIR="$installDir"
+else
+  installDir="$PARENT_DIR/install/kibana"
+  destDir="${installDir}-${CI_WORKER_NUMBER}"
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
 
 export TEST_BROWSER_HEADLESS=1
 
@@ -19,5 +32,5 @@ checks-reporter-with-killswitch "X-Pack visual regression tests" \
   yarn run percy exec \
   node scripts/functional_tests \
     --debug --bail \
-    --kibana-install-dir "$installDir" \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
     --config test/visual_regression/config.js;

test/server_integration/http/ssl_redirect/index.js

@@ -22,8 +22,12 @@ export default function ({ getService }) {
 
   describe('kibana server with ssl', () => {
     it('redirects http requests at redirect port to https', async () => {
+      const host = process.env.TEST_KIBANA_HOST || 'localhost';
+      const port = process.env.TEST_KIBANA_PORT || '5620';
+      const url = `https://${host}:${port}/`;
+
       await supertest.get('/')
-        .expect('location', 'https://localhost:5620/')
+        .expect('location', url)
         .expect(302);
 
       await supertest.get('/')

x-pack/test/alerting_api_integration/security_and_spaces/tests/index.ts

@@ -19,7 +19,7 @@ export default function alertingApiIntegrationTests({
   const esArchiver = getService('esArchiver');
 
   describe('alerting api integration security and spaces enabled', function() {
-    this.tags('ciGroup3');
+    this.tags('ciGroup8');
 
     before(async () => {
       for (const space of Spaces) {

x-pack/test/alerting_api_integration/spaces_only/tests/index.ts

@@ -17,7 +17,7 @@ export default function alertingApiIntegrationTests({
   const esArchiver = getService('esArchiver');
 
   describe('alerting api integration spaces only', function() {
-    this.tags('ciGroup3');
+    this.tags('ciGroup8');
 
     before(async () => {
       for (const space of Object.values(Spaces)) {

x-pack/test/api_integration/apis/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('apis', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
 
     loadTestFile(require.resolve('./es'));
     loadTestFile(require.resolve('./security'));

x-pack/test/api_integration/apis/management/cross_cluster_replication/remote_clusters.helpers.js

@@ -16,13 +16,16 @@ export const registerHelpers = (supertest) => {
   const addCluster = (name = REMOTE_CLUSTER_NAME) => {
     remoteClustersCreated.push(name);
 
+    // TODO how could this get pulled out of the ES config instead?
+    const esTransportPort = process.env.TEST_ES_TRANSPORT_PORT ? process.env.TEST_ES_TRANSPORT_PORT.split('-')[0] : '9300';
+
     return supertest
       .post(`${REMOTE_CLUSTERS_API_BASE_PATH}`)
       .set('kbn-xsrf', 'xxx')
       .send({
         'name': name,
         'seeds': [
-          'localhost:9300'
+          `localhost:${esTransportPort}`
         ],
         'skipUnavailable': true,
       });

x-pack/test/api_integration/apis/management/remote_clusters/constants.js

@@ -5,4 +5,3 @@
  */
 
 export const API_BASE_PATH = '/api/remote_clusters';
-export const NODE_SEED = 'localhost:9300';

x-pack/test/api_integration/apis/management/remote_clusters/remote_clusters.js

@@ -5,12 +5,15 @@
  */
 
 import expect from '@kbn/expect';
-import { API_BASE_PATH, NODE_SEED } from './constants';
+import { API_BASE_PATH } from './constants';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
   const retry = getService('retry');
 
+  const esTransportPort = process.env.TEST_ES_TRANSPORT_PORT ? process.env.TEST_ES_TRANSPORT_PORT.split('-')[0] : '9300';
+  const NODE_SEED = `localhost:${esTransportPort}`;
+
   describe('Remote Clusters', function () {
     this.tags(['skipCloud']);
 

x-pack/test/api_integration/apis/security/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('security', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
 
     loadTestFile(require.resolve('./basic_login'));
     loadTestFile(require.resolve('./builtin_es_privileges'));

x-pack/test/api_integration/apis/spaces/index.ts

@@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('spaces', function() {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
 
     loadTestFile(require.resolve('./saved_objects'));
     loadTestFile(require.resolve('./space_attributes'));

x-pack/test/functional/apps/apm/index.ts

@@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('APM', function() {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./feature_controls'));
   });
 }

x-pack/test/functional/apps/dashboard_mode/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('dashboard mode', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup7');
 
     loadTestFile(require.resolve('./dashboard_view_mode'));
   });

x-pack/test/functional/apps/index_lifecycle_management/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext) => {
   describe('Index Lifecycle Management app', function() {
-    this.tags('ciGroup2');
+    this.tags('ciGroup7');
     loadTestFile(require.resolve('./home_page'));
   });
 };

x-pack/test/functional/apps/infra/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext) => {
   describe('InfraOps app', function() {
-    this.tags('ciGroup2');
+    this.tags('ciGroup7');
 
     loadTestFile(require.resolve('./home_page'));
     loadTestFile(require.resolve('./feature_controls'));

x-pack/test/functional/apps/license_management/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext) => {
   describe('License app', function() {
-    this.tags('ciGroup2');
+    this.tags('ciGroup7');
     loadTestFile(require.resolve('./home_page'));
   });
 };

x-pack/test/functional/apps/maps/index.js

@@ -28,7 +28,7 @@ export default function ({ loadTestFile, getService }) {
     });
 
     describe('', function () {
-      this.tags('ciGroup2');
+      this.tags('ciGroup7');
       loadTestFile(require.resolve('./documents_source'));
       loadTestFile(require.resolve('./saved_object_management'));
       loadTestFile(require.resolve('./sample_data'));
@@ -38,7 +38,7 @@ export default function ({ loadTestFile, getService }) {
     });
 
     describe('', function () {
-      this.tags('ciGroup5');
+      this.tags('ciGroup10');
       loadTestFile(require.resolve('./es_geo_grid_source'));
       loadTestFile(require.resolve('./joins'));
       loadTestFile(require.resolve('./add_layer_panel'));

x-pack/test/functional/apps/uptime/index.ts

@@ -18,7 +18,7 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => {
       await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'UTC' });
     });
     after(async () => await esArchiver.unload(ARCHIVE));
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./overview'));

x-pack/test/kerberos_api_integration/apis/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('apis Kerberos', function() {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./security'));
   });
 }

x-pack/test/oidc_api_integration/apis/authorization_code_flow/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('apis', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./oidc_auth'));
   });
 }

x-pack/test/oidc_api_integration/apis/implicit_flow/index.ts

@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 // eslint-disable-next-line import/no-default-export
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('apis', function() {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./oidc_auth'));
   });
 }

x-pack/test/pki_api_integration/apis/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('apis PKI', function() {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./security'));
   });
 }

x-pack/test/reporting/functional/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('reporting app', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/saml_api_integration/apis/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('apis SAML', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./security'));
   });
 }

x-pack/test/saved_object_api_integration/security_only/apis/index.ts

@@ -12,7 +12,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) {
   const supertest = getService('supertest');
 
   describe('saved objects security only enabled', function() {
-    this.tags('ciGroup4');
+    this.tags('ciGroup9');
 
     before(async () => {
       await createUsersAndRoles(es, supertest);

x-pack/test/spaces_api_integration/security_and_spaces/apis/index.ts

@@ -13,7 +13,7 @@ export default function({ loadTestFile, getService }: TestInvoker) {
   const supertest = getService('supertest');
 
   describe('spaces api with security', function() {
-    this.tags('ciGroup3');
+    this.tags('ciGroup8');
 
     before(async () => {
       await createUsersAndRoles(es, supertest);

x-pack/test/token_api_integration/auth/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('token-based auth', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup6');
     loadTestFile(require.resolve('./login'));
     loadTestFile(require.resolve('./logout'));
     loadTestFile(require.resolve('./header'));

x-pack/test/ui_capabilities/security_and_spaces/tests/index.ts

@@ -17,7 +17,7 @@ export default function uiCapabilitiesTests({ loadTestFile, getService }: FtrPro
   const featuresService: FeaturesService = getService('features');
 
   describe('ui capabilities', function() {
-    this.tags('ciGroup4');
+    this.tags('ciGroup9');
 
     before(async () => {
       const features = await featuresService.get();

x-pack/test/ui_capabilities/security_only/tests/index.ts

@@ -13,7 +13,7 @@ export default function uiCapabilitesTests({ loadTestFile, getService }: FtrProv
   const securityService: SecurityService = getService('security');
 
   describe('ui capabilities', function() {
-    this.tags('ciGroup4');
+    this.tags('ciGroup9');
 
     before(async () => {
       for (const user of UserScenarios) {

x-pack/test/ui_capabilities/spaces_only/tests/index.ts

@@ -14,7 +14,7 @@ export default function uiCapabilitesTests({ loadTestFile, getService }: FtrProv
   const featuresService: FeaturesService = getService('features');
 
   describe('ui capabilities', function() {
-    this.tags('ciGroup4');
+    this.tags('ciGroup9');
 
     before(async () => {
       const features = await featuresService.get();

x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('upgrade assistant', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup7');
 
     loadTestFile(require.resolve('./query_default_field'));
     loadTestFile(require.resolve('./reindexing'));

x-pack/test/visual_regression/tests/maps/index.js

@@ -26,7 +26,7 @@ export default function ({ loadTestFile, getService }) {
       await esArchiver.unload('maps/kibana');
     });
 
-    this.tags('ciGroup5');
+    this.tags('ciGroup10');
     loadTestFile(require.resolve('./vector_styling'));
   });
 }