[QA][Code Coverage] follow up pr for ftr auto config handling & fix merge (#131734)

* [QA][Code Coverage] fixup for auto config handling pr ## Summary Embed buildkite pipeline definition. Follow up pr to change cc per auto config handling. Also, resolves https://github.com/elastic/kibana/issues/132706 Increase worker count for `node scripts/build_kibana_platform_plugins` to 4 workers. Normalize file names within coverage files such that nyc correctly builds the combined summaries. _Ci runs this on myriad servers, so the paths are different, which "breaks" nyc's output_ Split the final merge of functional coverage into 2 passes due to [nyc issue](https://github.com/istanbuljs/nyc/issues/1263) Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-23 17:28:26 -04:00 · 2022-05-27 12:28:16 +01:00 · 2022-05-27 12:28:16 +01:00 · 5f950e617a
commit 5f950e617a
parent e310b20bac
16 changed files with 475 additions and 157 deletions
--- a/.buildkite/pipelines/code_coverage/daily.yml
+++ b/.buildkite/pipelines/code_coverage/daily.yml
@ -0,0 +1,30 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
+    agents:
+      queue: kibana-default
+    env:
+      FTR_CONFIGS_DEPS: ''
+      LIMIT_CONFIG_TYPE: 'unit,functional,integration'
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/code_coverage/ftr_configs.sh'
+
+  - command: .buildkite/scripts/steps/code_coverage/ingest.sh
+    label: 'Merge and Ingest'
+    agents:
+      queue: c2-16
+    depends_on:
+      - jest
+      - jest-integration
+      - ftr-configs
+    timeout_in_minutes: 30
+    key: ingest
--- a/.buildkite/scripts/steps/code_coverage/clean_coverage_paths.js
+++ b/.buildkite/scripts/steps/code_coverage/clean_coverage_paths.js
@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { readFileSync, writeFileSync } = require('fs');
+
+const file = process.argv[2];
+const search = process.argv[3];
+const replace = process.argv[4];
+writeFileSync(file, readFileSync(file).toString().replaceAll(search, replace));
--- a/.buildkite/scripts/steps/code_coverage/ftr_configs.sh
+++ b/.buildkite/scripts/steps/code_coverage/ftr_configs.sh
@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/code_coverage/merge.sh
+source .buildkite/scripts/steps/code_coverage/util.sh
+source .buildkite/scripts/steps/code_coverage/node_scripts.sh
+
+export CODE_COVERAGE=1 # Kibana is bootstrapped differently for code coverage
+echo "--- KIBANA_DIR: $KIBANA_DIR"
+.buildkite/scripts/bootstrap.sh
+buildPlatformPlugins
+is_test_execution_step
+
+export JOB_NUM=$BUILDKITE_PARALLEL_JOB
+export JOB=ftr-configs-${JOB_NUM}
+
+functionalTarget="$KIBANA_DIR/target/kibana-coverage/functional"
+FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}"
+
+# a FTR failure will result in the script returning an exit code of 10
+exitCode=0
+configs="${FTR_CONFIG:-}"
+
+if [[ "$configs" == "" ]]; then
+  echo "--- Downloading ftr test run order"
+  buildkite-agent artifact download ftr_run_order.json .
+  configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
+fi
+
+echo "---   Config(s) for this FTR Group:"
+echo "${configs[@]}"
+
+failedConfigs=""
+results=()
+
+while read -r config; do
+  if [[ ! "$config" ]]; then
+    continue
+  fi
+  echo "--- Begin config $config"
+
+  start=$(date +%s)
+
+  # prevent non-zero exit code from breaking the loop
+  set +e
+  runFTRInstrumented "$config"
+  lastCode=$?
+  set -e
+  dasherize() {
+    local withoutExtension=${1%.*}
+    dasherized=$(echo "$withoutExtension" | tr '\/' '\-')
+  }
+  dasherize "$config"
+
+  if [[ -d "$functionalTarget" ]]; then
+    echo "--- Server and / or Client side code coverage collected"
+    if [[ -f "target/kibana-coverage/functional/coverage-final.json" ]]; then
+      # We potentially have more than one file with the same name being created,
+      # so we make them unique here.
+      mv target/kibana-coverage/functional/coverage-final.json "target/kibana-coverage/functional/${dasherized}-server-coverage.json"
+    fi
+  fi
+
+  timeSec=$(($(date +%s) - start))
+  if [[ $timeSec -gt 60 ]]; then
+    min=$((timeSec / 60))
+    sec=$((timeSec - (min * 60)))
+    duration="${min}m ${sec}s"
+  else
+    duration="${timeSec}s"
+  fi
+
+  results+=("- $config
+    duration: ${duration}
+    result: ${lastCode}")
+
+  if [ $lastCode -ne 0 ]; then
+    exitCode=10
+    echo "FTR exited with code $lastCode"
+    echo "^^^ +++"
+
+    if [[ "$failedConfigs" ]]; then
+      failedConfigs="${failedConfigs}"$'\n'"$config"
+    else
+      failedConfigs="$config"
+    fi
+  fi
+
+  echo "--- Config complete: $config"
+done <<<"$configs"
+
+# Each browser unload event, creates a new coverage file.
+# So, we merge them here.
+if [[ -d "$functionalTarget" ]]; then
+  reportMergeFunctional
+  uniqueifyFunctional "$(date +%s)"
+else
+  echo "--- Code coverage not found in: $functionalTarget"
+fi
+
+# Nyc uses matching absolute paths for reporting / merging
+# So, set all coverage json files to a specific prefx.
+# The prefix will be changed to the kibana dir, in the final stage,
+# so nyc doesnt error.
+echo "--- Normalize file paths prefix"
+replacePaths "$KIBANA_DIR/target/kibana-coverage/functional" "$KIBANA_DIR" "CC_REPLACEMENT_ANCHOR"
+
+if [[ "$failedConfigs" ]]; then
+  buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
+fi
+
+echo "--- FTR configs complete, result(s):"
+printf "%s\n" "${results[@]}"
+echo ""
+
+# So the last step "knows" this config ran
+uploadRanFile "ftr_configs"
+
+# Force exit 0 to ensure the next build step starts.
+exit 0
--- a/.buildkite/scripts/steps/code_coverage/ingest.sh
+++ b/.buildkite/scripts/steps/code_coverage/ingest.sh
@ -3,6 +3,8 @@
 set -euo pipefail

 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/code_coverage/util.sh
+source .buildkite/scripts/steps/code_coverage/merge.sh

 export CODE_COVERAGE=1
 echo "--- Reading Kibana stats cluster creds from vault"
@ -11,6 +13,9 @@ export PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-iss
 export HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
 export TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")

+echo "--- Print KIBANA_DIR"
+echo "### KIBANA_DIR: $KIBANA_DIR"
+
 echo "--- Download previous git sha"
 .buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
 previousSha=$(cat downloaded_previous.txt)
@ -19,10 +24,12 @@ echo "--- Upload new git sha"
 .buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh

 .buildkite/scripts/bootstrap.sh
- 
-echo "--- Download coverage arctifacts"
+
+echo "--- Download coverage artifacts"
 buildkite-agent artifact download target/kibana-coverage/jest/* .
 buildkite-agent artifact download target/kibana-coverage/functional/* .
+buildkite-agent artifact download target/ran_files/* .
+ls -l target/ran_files/* || echo "### No ran-files found"

 echo "--- process HTML Links"
 .buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh
@ -30,26 +37,30 @@ echo "--- process HTML Links"
 echo "--- collect VCS Info"
 .buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh

-# replace path in json files and generate final reports
-echo "--- Replace path in json files"
-export COVERAGE_TEMP_DIR=$KIBANA_DIR/target/kibana-coverage
-sed -i "s|/opt/local-ssd/buildkite/builds/kb-[[:alnum:]\-]\{20,27\}/elastic/kibana-code-coverage-main/kibana|${KIBANA_DIR}|g" $COVERAGE_TEMP_DIR/**/*.json
-
-echo "--- Jest: merging coverage files and generating the final combined report"
+echo "--- Jest: Reset file paths prefix, merge coverage files, and generate the final combined report"
+# Jest: Reset file paths prefix to Kibana Dir of final worker
+replacePaths "$KIBANA_DIR/target/kibana-coverage/jest" "CC_REPLACEMENT_ANCHOR" "$KIBANA_DIR"
 yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js

-echo "--- Functional: merging json files and generating the final combined report"
-yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js
+echo "--- Functional: Reset file paths prefix, merge coverage files, and generate the final combined report"
+# Functional: Reset file paths prefix to Kibana Dir of final worker
+set +e
+sed -ie "s|CC_REPLACEMENT_ANCHOR|${KIBANA_DIR}|g" target/kibana-coverage/functional/*.json
+echo "--- Begin Split and Merge for Functional"
+splitCoverage target/kibana-coverage/functional
+splitMerge
+set -e

-# archive reports to upload as build artifacts
 echo "--- Archive and upload combined reports"
-tar -czf target/kibana-coverage/jest/kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined
-tar -czf target/kibana-coverage/functional/kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined
-buildkite-agent artifact upload 'target/kibana-coverage/jest/kibana-jest-coverage.tar.gz'
-buildkite-agent artifact upload 'target/kibana-coverage/functional/kibana-functional-coverage.tar.gz'
+collectAndUpload target/kibana-coverage/jest/kibana-jest-coverage.tar.gz \
+  target/kibana-coverage/jest-combined
+collectAndUpload target/kibana-coverage/functional/kibana-functional-coverage.tar.gz \
+  target/kibana-coverage/functional-combined

 echo "--- Upload coverage static site"
 .buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh

 echo "--- Ingest results to Kibana stats cluster"
-.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' ${BUILDKITE_BUILD_NUMBER} ${BUILDKITE_BUILD_URL} ${previousSha} 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
+.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' \
+  ${BUILDKITE_BUILD_NUMBER} ${BUILDKITE_BUILD_URL} ${previousSha} \
+  'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
--- a/.buildkite/scripts/steps/code_coverage/jest.sh
+++ b/.buildkite/scripts/steps/code_coverage/jest.sh
@ -3,18 +3,14 @@
 set -euo pipefail

 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/code_coverage/util.sh

 is_test_execution_step

 .buildkite/scripts/bootstrap.sh

 echo '--- Jest code coverage'
-
 .buildkite/scripts/steps/code_coverage/jest_parallel.sh jest.config.js

-tar -czf kibana-jest-thread-coverage.tar.gz target/kibana-coverage/jest
-
-echo "--- Merging code coverage for a thread"
-yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js --reporter json
-rm -rf target/kibana-coverage/jest/*
-mv target/kibana-coverage/jest-combined/coverage-final.json "target/kibana-coverage/jest/jest-merged-coverage-$(date +%s%3N).json"
+# So the last step "knows" this config ran
+uploadRanFile "jest"
--- a/.buildkite/scripts/steps/code_coverage/jest_integration.sh
+++ b/.buildkite/scripts/steps/code_coverage/jest_integration.sh
@ -3,11 +3,16 @@
 set -euo pipefail

 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/code_coverage/util.sh

 is_test_execution_step

 .buildkite/scripts/bootstrap.sh

+JOB=${BUILDKITE_PARALLEL_JOB:-0}
+
 echo '--- Jest Integration code coverage'
-node --max-old-space-size=14336 scripts/jest_integration  --ci --coverage --coverageReporters json || true
-mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json"
+.buildkite/scripts/steps/code_coverage/jest_parallel.sh jest.integration.config.js
+
+# So the last step "knows" this config ran
+uploadRanFile "jest_integration"
--- a/.buildkite/scripts/steps/code_coverage/jest_parallel.sh
+++ b/.buildkite/scripts/steps/code_coverage/jest_parallel.sh
@ -1,38 +1,89 @@
 #!/bin/bash

-set -uo pipefail
+set -euo pipefail

-JOB=${BUILDKITE_PARALLEL_JOB:-0}
-JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-1}
+source .buildkite/scripts/steps/code_coverage/util.sh
+
+export JOB=$BUILDKITE_PARALLEL_JOB

 # a jest failure will result in the script returning an exit code of 10
-
-i=0
 exitCode=0
+results=()
+
+if [[ "$1" == 'jest.config.js' ]]; then
+  TEST_TYPE="unit"
+else
+  TEST_TYPE="integration"
+fi
+
+export TEST_TYPE
+echo "--- downloading jest test run order"
+buildkite-agent artifact download jest_run_order.json .
+configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
+
+echo "--- KIBANA_DIR: $KIBANA_DIR"
+
+echo "---  Config(s) for this JEST Group:"
+echo "${configs[@]}"

 while read -r config; do
-  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
-    echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
-    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" \
-      --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest \
-      --passWithNoTests || true
-    if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then
-      echo "Rename coverage-final.json to avoid overwrite"
-      mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
-    else
-      echo "Cannot find coverage-final.json"
-    fi
-    lastCode=$?
+  echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"

-    if [ $lastCode -ne 0 ]; then
-      exitCode=10
-      echo "Jest exited with code $lastCode"
-      echo "^^^ +++"
-    fi
+  echo "--- Print config name"
+  echo "### config: $config"
+
+  start=$(date +%s)
+
+  # prevent non-zero exit code from breaking the loop
+  set +e
+  NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest \
+    --config="$config" --runInBand --ci --coverage \
+    --coverageReporters json --passWithNoTests \
+    --coverageDirectory target/kibana-coverage/jest
+  lastCode=$?
+  set -e
+
+  if [[ -f target/kibana-coverage/jest/coverage-final.json ]]; then
+    echo "--- Rename target/kibana-coverage/jest/coverage-final.json to avoid overwrite"
+    mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+  else
+    echo "Cannot find coverage-final.json"
  fi

-  ((i=i+1))
-# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
-done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
+  timeSec=$(($(date +%s) - start))
+  if [[ $timeSec -gt 60 ]]; then
+    min=$((timeSec / 60))
+    sec=$((timeSec - (min * 60)))
+    duration="${min}m ${sec}s"
+  else
+    duration="${timeSec}s"
+  fi

-exit $exitCode
+  results+=("- $config
+    duration: ${duration}
+    result: ${lastCode}")
+
+  if [ $lastCode -ne 0 ]; then
+    echo "Jest exited with code $lastCode"
+    echo "^^^ +++"
+  fi
+done <<<"$configs"
+
+echo "--- Normalize file paths prefix before final stage"
+# Nyc uses matching absolute paths for reporting / merging
+# So, set all coverage json files to a specific prefx.
+# The prefix will be changed to the kibana dir, in the final stage,
+# so nyc doesnt error.
+replacePaths "$KIBANA_DIR/target/kibana-coverage/jest" "$KIBANA_DIR" "CC_REPLACEMENT_ANCHOR"
+
+echo "--- Merging code coverage for a thread"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js --reporter json
+rm -rf target/kibana-coverage/jest/*
+mv target/kibana-coverage/jest-combined/coverage-final.json \
+  "target/kibana-coverage/jest/jest-$TEST_TYPE-merged-coverage-$(date +%s%3N).json"
+
+echo "--- Jest [$TEST_TYPE] configs complete"
+printf "%s\n" "${results[@]}"
+
+# Force exit 0 to ensure the next build step starts.
+exit 0
--- a/.buildkite/scripts/steps/code_coverage/merge.sh
+++ b/.buildkite/scripts/steps/code_coverage/merge.sh
@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export CODE_COVERAGE=1
+
+base=target/kibana-coverage
+target="$base/functional"
+first="$target/first"
+rest="$target/rest"
+
+filesCount() {
+  count=$(find "$1" -maxdepth 1 -type f | wc -l | xargs) # xargs trims whitespace
+}
+
+_head() {
+  firstFile=$(find "$1" -maxdepth 1 -type f | head -1)
+}
+
+splitCoverage() {
+  echo "--- Running splitCoverage:"
+  filesCount "$1"
+  echo "### total: $count"
+
+  mkdir -p $first
+  mkdir -p $rest
+  half=$((count / 2))
+  echo "### half: $half"
+
+  echo "### Move the first half into the 'first' dir"
+  # the index variable is irrelevant
+  for x in $(seq 1 $half); do
+    _head "$1"
+    mv "$firstFile" "$first"
+  done
+
+  echo "### Move the second half into the 'rest' dir"
+  while read -r x; do
+    mv "$x" "$rest" || printf "\n\t### Trouble moving %s to %s" "$x" "$rest"
+  done <<<"$(find "$target" -maxdepth 1 -type f -name '*.json')"
+}
+
+splitMerge() {
+  echo "### Merge the 1st half of the coverage files"
+  yarn nyc merge target/kibana-coverage/functional/first target/kibana-coverage/functional/first.json
+  echo "### Merge the 2nd half of the coverage files"
+  yarn nyc merge target/kibana-coverage/functional/rest target/kibana-coverage/functional/rest.json
+  echo "### Report-Merge the 2 halves into one"
+  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js
+}
+
+uniqueifyFunctional() {
+  local unique=${1:?"Must pass first positional arg for 'unique'"}
+
+  # Drop the json files that where report-merged.
+  rm -rf target/kibana-coverage/functional/*
+
+  # Move from report-merge target dir, to: target/kibana-coverage/functional
+  mv target/kibana-coverage/functional-combined/coverage-final.json \
+    "target/kibana-coverage/functional/$unique-coverage-final.json"
+}
--- a/.buildkite/scripts/steps/code_coverage/node_scripts.sh
+++ b/.buildkite/scripts/steps/code_coverage/node_scripts.sh
@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+buildPlatformPlugins() {
+  echo "--- Build Platform Plugins"
+
+  NODE_OPTIONS=--max_old_space_size=14336 \
+    node scripts/build_kibana_platform_plugins \
+    --no-examples --test-plugins --workers 4
+}
+
+runFTRInstrumented() {
+  local ftrConfig=$1
+  echo "--- $ runFTRInstrumented against $ftrConfig"
+
+  NODE_OPTIONS=--max_old_space_size=16384 \
+    ./node_modules/.bin/nyc \
+    --nycrc-path ./src/dev/code_coverage/nyc_config/nyc.server.config.js \
+    node scripts/functional_tests \
+    --config="$ftrConfig" \
+    --exclude-tag "skipCoverage"
+}
+
+reportMergeFunctional() {
+  echo "--- Merging code coverage for FTR Configs"
+
+  NODE_OPTIONS=--max_old_space_size=16384 yarn nyc report \
+    --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
+}
--- a/.buildkite/scripts/steps/code_coverage/oss_cigroup.sh
+++ b/.buildkite/scripts/steps/code_coverage/oss_cigroup.sh
@ -1,43 +0,0 @@
-
-   
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/build_kibana_plugins.sh
-
-is_test_execution_step
-
-export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
-export JOB=kibana-oss-ciGroup${CI_GROUP}
-
-export NODE_OPTIONS=--max_old_space_size=8192
-export CODE_COVERAGE=1
-
-echo "--- OSS CI Group $CI_GROUP"
-echo " -> Running Functional tests with code coverage"
-
-NODE_OPTIONS=--max_old_space_size=14336 \
-  ./node_modules/.bin/nyc \
-  --nycrc-path src/dev/code_coverage/nyc_config/nyc.server.config.js \
-  node scripts/functional_tests \
-  --include-tag "ciGroup$CI_GROUP" \
-  --exclude-tag "skipCoverage" || true
-
-if [[ -d "$KIBANA_DIR/target/kibana-coverage/server" ]]; then
-  echo "--- Server side code coverage collected"
-  mkdir -p target/kibana-coverage/functional
-  mv target/kibana-coverage/server/coverage-final.json "target/kibana-coverage/functional/oss-${CI_GROUP}-server-coverage.json"
-fi
-
-if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
-  echo "--- Merging code coverage for CI Group $CI_GROUP"
-  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
-  rm -rf target/kibana-coverage/functional/*
-  mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/oss-${CI_GROUP}-coverage.json"
-else
-  echo "--- Code coverage not found"
-fi
--- a/.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh
+++ b/.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh
@ -34,15 +34,18 @@ export BUFFER_SIZE
 echo "### debug BUFFER_SIZE: ${BUFFER_SIZE}"

 # Build team assignments file
-CI_STATS_DISABLED=true node scripts/generate_team_assignments.js --verbose --src '.github/CODEOWNERS' --dest $TEAM_ASSIGN_PATH
+echo "### Generate Team Assignments"
+CI_STATS_DISABLED=true node scripts/generate_team_assignments.js \
+  --verbose --src '.github/CODEOWNERS' --dest $TEAM_ASSIGN_PATH

 for x in functional jest; do
  echo "### Ingesting coverage for ${x}"
-  COVERAGE_SUMMARY_FILE=target/kibana-coverage/${x}-combined/coverage-summary.json
-  # running in background to speed up ingestion
-  CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH &
+  COVERAGE_SUMMARY_FILE="target/kibana-coverage/${x}-combined/coverage-summary.json"
+
+  CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} \
+    --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH &
 done
 wait

-echo "###  Ingesting Code Coverage - Complete"
-echo ""
+echo "---  Ingesting Code Coverage - Complete"
+echo ""
--- a/.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh
+++ b/.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh
@ -8,11 +8,11 @@ uploadPrefixWithTimeStamp="${uploadPrefix}${TIME_STAMP}/"
 cat src/dev/code_coverage/www/index.html

 for x in 'src/dev/code_coverage/www/index.html' 'src/dev/code_coverage/www/404.html'; do
-    gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefix}
+   gsutil -m -q cp -r -a public-read -z js,css,html ${x} ${uploadPrefix}
 done

-gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}
+gsutil -m -q cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}

 for x in 'target/kibana-coverage/functional-combined' 'target/kibana-coverage/jest-combined'; do
-    gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}
-done
+    gsutil -m -q cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}
+done
--- a/.buildkite/scripts/steps/code_coverage/util.sh
+++ b/.buildkite/scripts/steps/code_coverage/util.sh
@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# $1 file name, ex: "target/dir-listing-jest.txt"
+# $2 directory to be listed, ex: target/kibana-coverage/jest
+dirListing() {
+  local fileName=$1
+  local dir=$2
+
+  ls -l "$dir" >"$fileName"
+
+  printf "\n### %s \n\tlisted to: %s\n" "$dir" "$fileName"
+  buildkite-agent artifact upload "$fileName"
+
+  printf "\n### %s Uploaded\n" "$fileName"
+}
+
+replacePaths() {
+  local dirName=$1
+  local search=$2
+  local replace=$3
+
+  for x in $(find "$dirName" -maxdepth 1 -type f -name '*.json'); do
+    node .buildkite/scripts/steps/code_coverage/clean_coverage_paths.js \
+      "$x" \
+      "$search" \
+      "$replace"
+  done
+}
+
+header() {
+  local fileName=$1
+
+  echo "" >"$fileName"
+
+  echo "### File Name:" >>"$fileName"
+  printf "\t%s\n" "$fileName" >>"$fileName"
+}
+
+fileHeads() {
+  local fileName=$1
+  local dir=$2
+  local ext=${3:-'*.json'}
+
+  header "$fileName"
+
+  while read -r x; do
+    printf "\n### BEGIN %s\n\n" "$x" >>"$fileName"
+    head -2 "$x" >>"$fileName"
+    printf "\n### END %s\n\n" "$x" >>"$fileName"
+  done <<<"$(find "$dir" -maxdepth 1 -type f -name "$ext")"
+
+  buildkite-agent artifact upload "$fileName"
+
+  printf "\n### %s Uploaded\n" "$fileName"
+}
+
+collectAndUpload() {
+  local fileName=$1
+  local dir=$2
+
+  tar -czf "$fileName" "$dir"
+
+  buildkite-agent artifact upload "$fileName"
+
+  printf "\n### %s Uploaded\n" "$fileName"
+}
+
+# Jest, Jest Integration, and FTR Configs will use this to "tell"
+# the last stage they ran.
+uploadRanFile() {
+  local ran=$1
+
+  mkdir -p target/ran_files
+
+  local fileName="target/ran_files/$ran.txt"
+
+  echo "$ran" >"$fileName"
+
+  buildkite-agent artifact upload "$fileName"
+}
--- a/.buildkite/scripts/steps/code_coverage/xpack_cigroup.sh
+++ b/.buildkite/scripts/steps/code_coverage/xpack_cigroup.sh
@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/build_kibana_plugins.sh
-
-is_test_execution_step
-
-
-export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
-export JOB=kibana-default-ciGroup${CI_GROUP}
-
-export NODE_OPTIONS=--max_old_space_size=8192
-export CODE_COVERAGE=1
-
-echo "--- Default CI Group $CI_GROUP"
-
-echo " -> Running X-Pack functional tests with code coverage"
-
-cd "$XPACK_DIR"
-
-NODE_OPTIONS=--max_old_space_size=14336 \
-  ./../node_modules/.bin/nyc \
-  --nycrc-path ./../src/dev/code_coverage/nyc_config/nyc.server.config.js \
-  node scripts/functional_tests \
-  --include-tag "ciGroup$CI_GROUP" \
-  --exclude-tag "skipCoverage" || true
-
-cd "$KIBANA_DIR"
-
-if [[ -d "$KIBANA_DIR/target/kibana-coverage/server" ]]; then
-  echo "--- Server side code coverage collected"
-  mkdir -p target/kibana-coverage/functional
-  mv target/kibana-coverage/server/coverage-final.json "target/kibana-coverage/functional/xpack-${CI_GROUP}-server-coverage.json"
-fi
-
-if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
-  echo "--- Merging code coverage for CI Group $CI_GROUP"
-  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
-  rm -rf target/kibana-coverage/functional/*
-  mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/xpack-${CI_GROUP}-coverage.json"
-else
-  echo "--- Code coverage not found"
-fi
-
-cd "$KIBANA_DIR"
--- a/src/dev/code_coverage/nyc_config/nyc.functional.config.js
+++ b/src/dev/code_coverage/nyc_config/nyc.functional.config.js
@ -19,11 +19,14 @@ const extraExclude = [
  '**/*.d.ts',
  '**/index.{js,ts,tsx}',
 ];
-const path = require('path');
+// const path = require('path');

 module.exports = {
+  // 'temp-dir': process.env.COVERAGE_TEMP_DIR
+  //   ? path.resolve(process.env.COVERAGE_TEMP_DIR, 'functional')
+  //   : 'target/kibana-coverage/functional',
  'temp-dir': process.env.COVERAGE_TEMP_DIR
-    ? path.resolve(process.env.COVERAGE_TEMP_DIR, 'functional')
+    ? process.env.COVERAGE_TEMP_DIR
    : 'target/kibana-coverage/functional',
  'report-dir': 'target/kibana-coverage/functional-combined',
  reporter: ['html', 'json-summary'],
--- a/src/dev/code_coverage/nyc_config/nyc.server.config.js
+++ b/src/dev/code_coverage/nyc_config/nyc.server.config.js
@ -11,8 +11,8 @@ const path = require('path');
 module.exports = {
  extends: '@istanbuljs/nyc-config-typescript',
  'report-dir': process.env.KIBANA_DIR
-    ? path.resolve(process.env.KIBANA_DIR, 'target/kibana-coverage/server')
-    : 'target/kibana-coverage/server',
+    ? path.resolve(process.env.KIBANA_DIR, 'target/kibana-coverage/functional')
+    : 'target/kibana-coverage/functional',
  reporter: ['json'],
  all: true,
  include: [