Code coverage setup on CI (#49003)

* running code coverage in CI

* apply review feedback

* add custom function to upload merged coverage reports

* fix artifacts upload without coverage

* add file extension to fix validation

* Check code_coverage is set

* run oss tests via grunt task

* review fixes

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.ci/Jenkinsfile_coverage

@@ -0,0 +1,112 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load() // load from the Jenkins instance
+
+stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        catchError {
+          withEnv([
+            'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
+          ]) {
+            parallel([
+              'kibana-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.legacyJobRunner('kibana-intake')()
+                }
+              },
+              'x-pack-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.legacyJobRunner('x-pack-intake')()
+                }
+              },
+              'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+                'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
+                'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
+                'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
+                'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
+                'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
+                'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
+                'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
+                'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
+                'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
+                'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
+                'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
+                'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
+              ]),
+              'kibana-xpack-agent-1': kibanaPipeline.withWorkers('kibana-xpack-tests-1', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
+                'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
+              ]),
+              'kibana-xpack-agent-2': kibanaPipeline.withWorkers('kibana-xpack-tests-2', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
+                'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
+              ]),
+
+              'kibana-xpack-agent-3': kibanaPipeline.withWorkers('kibana-xpack-tests-3', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
+                'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
+                'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
+                'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
+                'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
+                'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
+              ]),
+            ])
+            kibanaPipeline.jobRunner('tests-l', false) {
+              kibanaPipeline.downloadCoverageArtifacts()
+              kibanaPipeline.bash(
+                '''
+                  # bootstrap from x-pack folder
+                  source src/dev/ci_setup/setup_env.sh
+                  cd x-pack
+                  yarn kbn bootstrap --prefer-offline
+                  cd ..
+                  # extract archives
+                  mkdir -p /tmp/extracted_coverage
+                  echo extracting intakes
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-oss-tests
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-xpack-tests
+                  for i in {1..3}; do
+                    tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests-${i}/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  done
+                  # replace path in json files to have valid html report
+                  pwd=$(pwd)
+                  du -sh /tmp/extracted_coverage/target/kibana-coverage/
+                  echo replacing path in json files
+                  for i in {1..9}; do
+                    sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json &
+                  done
+                  wait
+                  # merge oss & x-pack reports
+                  echo merging coverage reports
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary
+                  echo copy mocha reports
+                  mkdir -p target/kibana-coverage/mocha-combined
+                  cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined
+                ''',
+                "run `yarn kbn bootstrap && merge coverage`"
+              )
+              sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
+              sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
+              sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
+            }
+          }
+        }
+        kibanaPipeline.sendMail()
+      }
+    }
+  }
+}

scripts/functional_tests.js

@@ -17,12 +17,23 @@
  * under the License.
  */
 
+// eslint-disable-next-line no-restricted-syntax
+const alwaysImportedTests = [
+  require.resolve('../test/functional/config.js'),
+  require.resolve('../test/plugin_functional/config.js'),
+  require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
+];
+// eslint-disable-next-line no-restricted-syntax
+const onlyNotInCoverageTests = [
+  require.resolve('../test/api_integration/config.js'),
+  require.resolve('../test/interpreter_functional/config.ts'),
+  require.resolve('../test/examples/config.js'),
+];
+
 require('../src/setup_node_env');
 require('@kbn/test').runTestsCli([
-  require.resolve('../test/functional/config.js'),
-  require.resolve('../test/api_integration/config.js'),
-  require.resolve('../test/plugin_functional/config.js'),
-  require.resolve('../test/interpreter_functional/config.ts'),
-  require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
-  require.resolve('../test/examples/config.js'),
+  // eslint-disable-next-line no-restricted-syntax
+  ...alwaysImportedTests,
+  // eslint-disable-next-line no-restricted-syntax
+  ...(!!process.env.CODE_COVERAGE ? [] : onlyNotInCoverageTests),
 ]);

src/dev/jest/config.js

@@ -69,7 +69,7 @@ export default {
   ],
   setupFilesAfterEnv: ['<rootDir>/src/dev/jest/setup/mocks.js'],
   coverageDirectory: '<rootDir>/target/kibana-coverage/jest',
-  coverageReporters: ['html', 'text'],
+  coverageReporters: !!process.env.CODE_COVERAGE ? ['json'] : ['html', 'text'],
   moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
   modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
   testMatch: ['**/*.test.{js,ts,tsx}'],

src/legacy/server/config/schema.js

@@ -181,7 +181,7 @@ export default () =>
         .default('localhost'),
       watchPrebuild: Joi.boolean().default(false),
       watchProxyTimeout: Joi.number().default(10 * 60000),
-      useBundleCache: Joi.boolean().default(Joi.ref('$prod')),
+      useBundleCache: Joi.boolean().default(!!process.env.CODE_COVERAGE ? true : Joi.ref('$prod')),
       sourceMaps: Joi.when('$prod', {
         is: true,
         then: Joi.boolean().valid(false),

tasks/config/run.js

@@ -152,6 +152,7 @@ module.exports = function(grunt) {
       args: [
         'nyc',
         '--reporter=html',
+        '--reporter=json-summary',
         '--report-dir=./target/kibana-coverage/mocha',
         NODE,
         'scripts/mocha',

tasks/function_test_groups.js

@@ -29,6 +29,21 @@ const TEST_TAGS = safeLoad(JOBS_YAML)
   .JOB.filter(id => id.startsWith('kibana-ciGroup'))
   .map(id => id.replace(/^kibana-/, ''));
 
+const getDefaultArgs = tag => {
+  return [
+    'scripts/functional_tests',
+    '--include-tag',
+    tag,
+    '--config',
+    'test/functional/config.js',
+    '--config',
+    'test/ui_capabilities/newsfeed_err/config.ts',
+    // '--config', 'test/functional/config.firefox.js',
+    '--bail',
+    '--debug',
+  ];
+};
+
 export function getFunctionalTestGroupRunConfigs({ kibanaInstallDir } = {}) {
   return {
     // include a run task for each test group
@@ -38,18 +53,8 @@ export function getFunctionalTestGroupRunConfigs({ kibanaInstallDir } = {}) {
         [`functionalTests_${tag}`]: {
           cmd: process.execPath,
           args: [
-            'scripts/functional_tests',
-            '--include-tag',
-            tag,
-            '--config',
-            'test/functional/config.js',
-            '--config',
-            'test/ui_capabilities/newsfeed_err/config.ts',
-            // '--config', 'test/functional/config.firefox.js',
-            '--bail',
-            '--debug',
-            '--kibana-install-dir',
-            kibanaInstallDir,
+            ...getDefaultArgs(tag),
+            ...(!!process.env.CODE_COVERAGE ? [] : ['--kibana-install-dir', kibanaInstallDir]),
           ],
         },
       }),

test/functional/services/remote/remote.ts

@@ -100,7 +100,9 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
     .subscribe({
       next({ message, level }) {
         const msg = message.replace(/\\n/g, '\n');
-        log[level === 'SEVERE' ? 'error' : 'debug'](`browser[${level}] ${msg}`);
+        log[level === 'SEVERE' || level === 'error' ? 'error' : 'debug'](
+          `browser[${level}] ${msg}`
+        );
       },
     });
 

test/scripts/jenkins_build_kibana.sh

@@ -8,5 +8,8 @@ node scripts/es snapshot --license=oss --download-only;
 echo " -> Ensuring all functional tests are in a ciGroup"
 yarn run grunt functionalTests:ensureAllTestsInCiGroup;
 
-echo " -> building and extracting OSS Kibana distributable for use in functional tests"
-node scripts/build --debug --oss
+# Do not build kibana for code coverage run
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> building and extracting OSS Kibana distributable for use in functional tests"
+  node scripts/build --debug --oss
+fi

test/scripts/jenkins_ci_group.sh

@@ -2,22 +2,30 @@
 
 source test/scripts/jenkins_test_setup.sh
 
-if [[ -z "$IS_PIPELINE_JOB" ]] ; then
-  yarn run grunt functionalTests:ensureAllTestsInCiGroup;
-  node scripts/build --debug --oss;
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+    yarn run grunt functionalTests:ensureAllTestsInCiGroup;
+    node scripts/build --debug --oss;
+  else
+    installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+    destDir=${installDir}-${CI_WORKER_NUMBER}
+    cp -R "$installDir" "$destDir"
+
+    export KIBANA_INSTALL_DIR="$destDir"
+  fi
+
+  checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
+
+  if [ "$CI_GROUP" == "1" ]; then
+    source test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh
+    yarn run grunt run:pluginFunctionalTestsRelease --from=source;
+    yarn run grunt run:exampleFunctionalTestsRelease --from=source;
+    yarn run grunt run:interpreterFunctionalTestsRelease;
+  fi
 else
-  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
-  destDir=${installDir}-${CI_WORKER_NUMBER}
-  cp -R "$installDir" "$destDir"
+  echo " -> Running Functional tests with code coverage"
 
-  export KIBANA_INSTALL_DIR="$destDir"
-fi
-
-checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
-
-if [ "$CI_GROUP" == "1" ]; then
-  source test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh
-  yarn run grunt run:pluginFunctionalTestsRelease --from=source;
-  yarn run grunt run:exampleFunctionalTestsRelease --from=source;
-  yarn run grunt run:interpreterFunctionalTestsRelease;
+  export NODE_OPTIONS=--max_old_space_size=8192
+
+  yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
 fi

test/scripts/jenkins_unit.sh

@@ -4,4 +4,16 @@ set -e
 
 export TEST_BROWSER_HEADLESS=1
 
-"$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev;
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev;
+else
+  echo "NODE_ENV=$NODE_ENV"
+  echo " -> Running jest tests with coverage"
+  node scripts/jest --ci --verbose --coverage
+  echo ""
+  echo ""
+  echo " -> Running mocha tests with coverage"
+  yarn run grunt "test:mochaCoverage";
+  echo ""
+  echo ""
+fi

test/scripts/jenkins_xpack.sh

@@ -4,33 +4,48 @@ set -e
 
 export TEST_BROWSER_HEADLESS=1
 
-echo " -> Running mocha tests"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser
-echo ""
-echo ""
-
-echo " -> Running jest tests"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose
-echo ""
-echo ""
-
-echo " -> Running SIEM cyclic dependency test"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps
-echo ""
-echo ""
-
-# FAILING: https://github.com/elastic/kibana/issues/44250
-# echo " -> Running jest contracts tests"
-# cd "$XPACK_DIR"
-# SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose
-# echo ""
-# echo ""
-
-# echo " -> Running jest integration tests"
-# cd "$XPACK_DIR"
-# node scripts/jest_integration --ci --verbose
-# echo ""
-# echo ""
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> Running mocha tests"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser
+  echo ""
+  echo ""
+    
+  echo " -> Running jest tests"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose
+  echo ""
+  echo ""
+    
+  echo " -> Running SIEM cyclic dependency test"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps
+  echo ""
+  echo ""
+    
+  # FAILING: https://github.com/elastic/kibana/issues/44250
+  # echo " -> Running jest contracts tests"
+  # cd "$XPACK_DIR"
+  # SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose
+  # echo ""
+  # echo ""
+    
+  # echo " -> Running jest integration tests"
+  # cd "$XPACK_DIR"
+  # node scripts/jest_integration --ci --verbose
+  # echo ""
+  # echo ""
+else
+  echo " -> Running jest tests with coverage"
+  cd "$XPACK_DIR"
+  # build runtime for canvas
+  echo "NODE_ENV=$NODE_ENV"
+  node ./legacy/plugins/canvas/scripts/shareable_runtime
+  node scripts/jest --ci --verbose --coverage
+  # rename file in order to be unique one
+  test -f ../target/kibana-coverage/jest/coverage-final.json \
+    && mv ../target/kibana-coverage/jest/coverage-final.json \
+    ../target/kibana-coverage/jest/xpack-coverage-final.json
+  echo ""
+  echo ""
+fi

test/scripts/jenkins_xpack_build_kibana.sh

@@ -20,10 +20,13 @@ node scripts/functional_tests --assert-none-excluded \
   --include-tag ciGroup9 \
   --include-tag ciGroup10
 
-echo " -> building and extracting default Kibana distributable for use in functional tests"
-cd "$KIBANA_DIR"
-node scripts/build --debug --no-oss
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+# Do not build kibana for code coverage run
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> building and extracting default Kibana distributable for use in functional tests"
+  cd "$KIBANA_DIR"
+  node scripts/build --debug --no-oss
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+fi

test/scripts/jenkins_xpack_ci_group.sh

@@ -2,59 +2,60 @@
 
 source test/scripts/jenkins_test_setup.sh
 
-if [[ -z "$IS_PIPELINE_JOB" ]] ; then
-  echo " -> Ensuring all functional tests are in a ciGroup"
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+    echo " -> Ensuring all functional tests are in a ciGroup"
+    cd "$XPACK_DIR"
+    node scripts/functional_tests --assert-none-excluded \
+      --include-tag ciGroup1 \
+      --include-tag ciGroup2 \
+      --include-tag ciGroup3 \
+      --include-tag ciGroup4 \
+      --include-tag ciGroup5 \
+      --include-tag ciGroup6 \
+      --include-tag ciGroup7 \
+      --include-tag ciGroup8 \
+      --include-tag ciGroup9 \
+      --include-tag ciGroup10
+  fi
+
+  cd "$KIBANA_DIR"
+
+  if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+    echo " -> building and extracting default Kibana distributable for use in functional tests"
+    node scripts/build --debug --no-oss
+
+    linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+    installDir="$PARENT_DIR/install/kibana"
+
+    mkdir -p "$installDir"
+    tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+    export KIBANA_INSTALL_DIR="$installDir"
+  else
+    installDir="$PARENT_DIR/install/kibana"
+    destDir="${installDir}-${CI_WORKER_NUMBER}"
+    cp -R "$installDir" "$destDir"
+
+    export KIBANA_INSTALL_DIR="$destDir"
+  fi
+
+  echo " -> Running functional and api tests"
   cd "$XPACK_DIR"
-  node scripts/functional_tests --assert-none-excluded \
-    --include-tag ciGroup1 \
-    --include-tag ciGroup2 \
-    --include-tag ciGroup3 \
-    --include-tag ciGroup4 \
-    --include-tag ciGroup5 \
-    --include-tag ciGroup6 \
-    --include-tag ciGroup7 \
-    --include-tag ciGroup8 \
-    --include-tag ciGroup9 \
-    --include-tag ciGroup10
-fi
 
-cd "$KIBANA_DIR"
+  checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
+    node scripts/functional_tests \
+      --debug --bail \
+      --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+      --include-tag "ciGroup$CI_GROUP"
 
-if [[ -z "$IS_PIPELINE_JOB" ]] ; then
-  echo " -> building and extracting default Kibana distributable for use in functional tests"
-  node scripts/build --debug --no-oss
-
-  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-  installDir="$PARENT_DIR/install/kibana"
-
-  mkdir -p "$installDir"
-  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
-
-  export KIBANA_INSTALL_DIR="$installDir"
+  echo ""
+  echo ""
 else
-  installDir="$PARENT_DIR/install/kibana"
-  destDir="${installDir}-${CI_WORKER_NUMBER}"
-  cp -R "$installDir" "$destDir"
+  echo " -> Running X-Pack functional tests with code coverage"
+  cd "$XPACK_DIR"
 
-  export KIBANA_INSTALL_DIR="$destDir"
+  export NODE_OPTIONS=--max_old_space_size=8192
+
+  node scripts/functional_tests --debug --include-tag "ciGroup$CI_GROUP"
 fi
-
-echo " -> Running functional and api tests"
-cd "$XPACK_DIR"
-
-checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
-  node scripts/functional_tests \
-    --debug --bail \
-    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
-    --include-tag "ciGroup$CI_GROUP"
-
-echo ""
-echo ""
-
-# checks-reporter-with-killswitch "X-Pack Firefox Functional tests / Group ${CI_GROUP}" \
-#   node scripts/functional_tests --debug --bail \
-#   --kibana-install-dir "$installDir" \
-#   --include-tag "ciGroup$CI_GROUP" \
-#   --config "test/functional/config.firefox.js"
-# echo ""
-# echo ""

vars/kibanaPipeline.groovy

@@ -176,6 +176,18 @@ def uploadGcsArtifact(uploadPrefix, pattern) {
   )
 }
 
+def downloadCoverageArtifacts() {
+  def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/coverage/"
+  def targetLocation = "/tmp/downloaded_coverage"
+
+  sh "mkdir -p '${targetLocation}' && gsutil -m cp -r '${storageLocation}' '${targetLocation}'"
+}
+
+def uploadCoverageArtifacts(prefix, pattern) {
+  def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/coverage/${prefix}"
+  uploadGcsArtifact(uploadPrefix, pattern)
+}
+
 def withGcsArtifactUpload(workerName, closure) {
   def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}"
   def ARTIFACT_PATTERNS = [
@@ -201,6 +213,11 @@ def withGcsArtifactUpload(workerName, closure) {
       }
     }
   })
+
+  if (env.CODE_COVERAGE) {
+    sh 'tar -czf kibana-coverage.tar.gz target/kibana-coverage/**/*'
+    uploadGcsArtifact("kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/coverage/${workerName}", 'kibana-coverage.tar.gz')
+  }
 }
 
 def publishJunit() {

x-pack/dev-tools/jest/create_jest_config.js

@@ -30,7 +30,7 @@ export function createJestConfig({ kibanaDirectory, xPackKibanaDirectory }) {
       '^test_utils/find_test_subject': `${xPackKibanaDirectory}/test_utils/find_test_subject.ts`,
     },
     coverageDirectory: '<rootDir>/../target/kibana-coverage/jest',
-    coverageReporters: ['html'],
+    coverageReporters: !!process.env.CODE_COVERAGE ? ['json'] : ['html'],
     setupFiles: [
       `${kibanaDirectory}/src/dev/jest/setup/babel_polyfill.js`,
       `<rootDir>/dev-tools/jest/setup/polyfills.js`,

x-pack/scripts/functional_tests.js

@@ -4,38 +4,43 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-require('@kbn/plugin-helpers').babelRegister();
-require('@kbn/test').runTestsCli([
+const alwaysImportedTests = [require.resolve('../test/functional/config.js')];
+const onlyNotInCoverageTests = [
   require.resolve('../test/reporting/configs/chromium_api.js'),
   require.resolve('../test/reporting/configs/chromium_functional.js'),
-  require.resolve('../test/reporting/configs/generate_api'),
-  require.resolve('../test/functional/config.js'),
+  require.resolve('../test/reporting/configs/generate_api.js'),
   require.resolve('../test/api_integration/config_security_basic.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/alerting_api_integration/spaces_only/config.ts'),
   require.resolve('../test/alerting_api_integration/security_and_spaces/config.ts'),
   require.resolve('../test/plugin_api_integration/config.js'),
-  require.resolve('../test/plugin_functional/config'),
-  require.resolve('../test/kerberos_api_integration/config'),
-  require.resolve('../test/kerberos_api_integration/anonymous_access.config'),
-  require.resolve('../test/saml_api_integration/config'),
-  require.resolve('../test/token_api_integration/config'),
-  require.resolve('../test/oidc_api_integration/config'),
-  require.resolve('../test/oidc_api_integration/implicit_flow.config'),
-  require.resolve('../test/pki_api_integration/config'),
-  require.resolve('../test/spaces_api_integration/spaces_only/config'),
-  require.resolve('../test/spaces_api_integration/security_and_spaces/config_trial'),
-  require.resolve('../test/spaces_api_integration/security_and_spaces/config_basic'),
-  require.resolve('../test/saved_object_api_integration/security_and_spaces/config_trial'),
-  require.resolve('../test/saved_object_api_integration/security_and_spaces/config_basic'),
-  require.resolve('../test/saved_object_api_integration/security_only/config_trial'),
-  require.resolve('../test/saved_object_api_integration/security_only/config_basic'),
-  require.resolve('../test/saved_object_api_integration/spaces_only/config'),
-  require.resolve('../test/ui_capabilities/security_and_spaces/config'),
-  require.resolve('../test/ui_capabilities/security_only/config'),
-  require.resolve('../test/ui_capabilities/spaces_only/config'),
-  require.resolve('../test/upgrade_assistant_integration/config'),
-  require.resolve('../test/licensing_plugin/config'),
-  require.resolve('../test/licensing_plugin/config.public'),
-  require.resolve('../test/licensing_plugin/config.legacy'),
+  require.resolve('../test/plugin_functional/config.ts'),
+  require.resolve('../test/kerberos_api_integration/config.ts'),
+  require.resolve('../test/kerberos_api_integration/anonymous_access.config.ts'),
+  require.resolve('../test/saml_api_integration/config.ts'),
+  require.resolve('../test/token_api_integration/config.js'),
+  require.resolve('../test/oidc_api_integration/config.ts'),
+  require.resolve('../test/oidc_api_integration/implicit_flow.config.ts'),
+  require.resolve('../test/pki_api_integration/config.ts'),
+  require.resolve('../test/spaces_api_integration/spaces_only/config.ts'),
+  require.resolve('../test/spaces_api_integration/security_and_spaces/config_trial.ts'),
+  require.resolve('../test/spaces_api_integration/security_and_spaces/config_basic.ts'),
+  require.resolve('../test/saved_object_api_integration/security_and_spaces/config_trial.ts'),
+  require.resolve('../test/saved_object_api_integration/security_and_spaces/config_basic.ts'),
+  require.resolve('../test/saved_object_api_integration/security_only/config_trial.ts'),
+  require.resolve('../test/saved_object_api_integration/security_only/config_basic.ts'),
+  require.resolve('../test/saved_object_api_integration/spaces_only/config.ts'),
+  require.resolve('../test/ui_capabilities/security_and_spaces/config.ts'),
+  require.resolve('../test/ui_capabilities/security_only/config.ts'),
+  require.resolve('../test/ui_capabilities/spaces_only/config.ts'),
+  require.resolve('../test/upgrade_assistant_integration/config.js'),
+  require.resolve('../test/licensing_plugin/config.ts'),
+  require.resolve('../test/licensing_plugin/config.public.ts'),
+  require.resolve('../test/licensing_plugin/config.legacy.ts'),
+];
+
+require('@kbn/plugin-helpers').babelRegister();
+require('@kbn/test').runTestsCli([
+  ...alwaysImportedTests,
+  ...(!!process.env.CODE_COVERAGE ? [] : onlyNotInCoverageTests),
 ]);