[code coverage] create pipeline on Buildkite (#120364)

* [code coverage] buildkite pipeline

* [code coverage] move scripts to reporting folder

* [code coverage] increase retry count & request timeout, remove test logic

* use DISABLE_FAILED_TEST_REPORTER to ignore missing test results

* fix

* [ingestion] use legacy http driver

* print index page before upload

* Update packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

Co-authored-by: Brian Seeders <brian.seeders@elastic.co>

* remove step var

* change var to DISABLE_MISSING_TEST_REPORT_ERRORS

Co-authored-by: Brian Seeders <brian.seeders@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/scripts/lifecycle/post_command.sh

@@ -9,8 +9,9 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_
 if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   echo "--- Upload Artifacts"
   buildkite-agent artifact upload 'target/junit/**/*'
-  buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*'
+  buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
+  buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'

.buildkite/scripts/steps/code_coverage/ingest.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export CODE_COVERAGE=1
+echo "--- Reading Kibana stats cluster creds from vault"
+export USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)"
+export PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)"
+export HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
+export TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
+
+echo "--- Download previous git sha"
+.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
+previousSha=$(cat downloaded_previous.txt)
+
+echo "--- Upload new git sha"
+.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh
+
+.buildkite/scripts/bootstrap.sh
+ 
+echo "--- Download coverage arctifacts"
+buildkite-agent artifact download target/kibana-coverage/jest/* .
+buildkite-agent artifact download target/kibana-coverage/functional/* .
+
+echo "--- process HTML Links"
+.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh
+
+echo "--- collect VCS Info"
+.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh
+
+# replace path in json files and generate final reports
+echo "--- Replace path in json files"
+export COVERAGE_TEMP_DIR=$KIBANA_DIR/target/kibana-coverage
+sed -i "s|/opt/local-ssd/buildkite/builds/kb-[[:alnum:]\-]\{20,27\}/elastic/kibana-code-coverage-main/kibana|${KIBANA_DIR}|g" $COVERAGE_TEMP_DIR/**/*.json
+
+echo "--- Jest: merging coverage files and generating the final combined report"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js
+
+echo "--- Functional: merging json files and generating the final combined report"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js
+
+# archive reports to upload as build artifacts
+echo "--- Archive and upload combined reports"
+tar -czf target/kibana-coverage/jest/kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined
+tar -czf target/kibana-coverage/functional/kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined
+buildkite-agent artifact upload 'target/kibana-coverage/jest/kibana-jest-coverage.tar.gz'
+buildkite-agent artifact upload 'target/kibana-coverage/functional/kibana-functional-coverage.tar.gz'
+
+echo "--- Upload coverage static site"
+.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh
+
+echo "--- Ingest results to Kibana stats cluster"
+.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' ${BUILDKITE_BUILD_ID} ${BUILDKITE_BUILD_URL} ${previousSha} 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'

.buildkite/scripts/steps/code_coverage/jest.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+is_test_execution_step
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest code coverage'
+
+.buildkite/scripts/steps/code_coverage/jest_parallel.sh
+
+tar -czf kibana-jest-thread-coverage.tar.gz target/kibana-coverage/jest
+
+echo "--- Merging code coverage for a thread"
+yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js --reporter json
+rm -rf target/kibana-coverage/jest/*
+mv target/kibana-coverage/jest-combined/coverage-final.json "target/kibana-coverage/jest/jest-merged-coverage-$(date +%s%3N).json"

.buildkite/scripts/steps/code_coverage/jest_integration.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+is_test_execution_step
+
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Jest Integration code coverage'
+node --max-old-space-size=14336 scripts/jest_integration  --ci --coverage --coverageReporters json || true
+mv target/kibana-coverage/jest/coverage-final.json "target/kibana-coverage/jest/jest-integration-coverage.json"

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -uo pipefail
+
+JOB=$BUILDKITE_PARALLEL_JOB
+JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+
+# a jest failure will result in the script returning an exit code of 10
+
+i=0
+exitCode=0
+
+while read -r config; do
+  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
+    echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
+    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest || true
+    echo "Rename coverage-final.json to avoid overwrite"
+    mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+    lastCode=$?
+
+    if [ $lastCode -ne 0 ]; then
+      exitCode=10
+      echo "Jest exited with code $lastCode"
+      echo "^^^ +++"
+    fi
+  fi
+
+  ((i=i+1))
+# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
+done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
+
+exit $exitCode

.buildkite/scripts/steps/code_coverage/oss_cigroup.sh

@@ -0,0 +1,34 @@
+
+   
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana_plugins.sh
+
+is_test_execution_step
+
+export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
+export JOB=kibana-oss-ciGroup${CI_GROUP}
+
+export NODE_OPTIONS=--max_old_space_size=8192
+export CODE_COVERAGE=1
+
+echo "--- OSS CI Group $CI_GROUP"
+echo " -> Running Functional tests with code coverage"
+
+node scripts/functional_tests \
+  --include-tag "ciGroup$CI_GROUP" \
+  --exclude-tag "skipCoverage" || true
+
+if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
+  echo "--- Merging code coverage for CI Group $CI_GROUP"
+  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
+  rm -rf target/kibana-coverage/functional/*
+  mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/oss-${CI_GROUP}-coverage.json"
+else
+  echo "--- Code coverage not found"
+fi

.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "### Prok'd Index File: ..."
+cat src/dev/code_coverage/www/index.html
+
+predicate() {
+  x=$1
+  if [ -n "$x" ]; then
+    return
+  else
+    echo "### 1 or more variables that Code Coverage needs, are undefined"
+    exit 1
+  fi
+}
+CMD="git log --pretty=format"
+XS=("${GIT_BRANCH}" \
+  "$(${CMD}":%h" -1)" \
+  "$(${CMD}":%an" -1)" \
+  "$(${CMD}":%s" -1)")
+touch VCS_INFO.txt
+for X in "${!XS[@]}"; do
+  {
+    predicate "${XS[X]}"
+    echo "${XS[X]}" >> VCS_INFO.txt
+  }
+done
+echo "### VCS_INFO:"
+cat VCS_INFO.txt

.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+gsutil -m cp -r gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/previous.txt . || echo "### Previous Pointer NOT FOUND?"
+
+if [ -e ./previous.txt ]; then
+    mv previous.txt downloaded_previous.txt
+    echo "### downloaded_previous.txt"
+    cat downloaded_previous.txt
+fi

.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "### Ingesting Code Coverage"
+echo ""
+
+COVERAGE_JOB_NAME=$1
+export COVERAGE_JOB_NAME
+echo "### debug COVERAGE_JOB_NAME: ${COVERAGE_JOB_NAME}"
+
+BUILD_ID=$2
+export BUILD_ID
+
+CI_RUN_URL=$3
+export CI_RUN_URL
+echo "### debug CI_RUN_URL: ${CI_RUN_URL}"
+
+FETCHED_PREVIOUS=$4
+export FETCHED_PREVIOUS
+echo "### debug FETCHED_PREVIOUS: ${FETCHED_PREVIOUS}"
+
+ES_HOST="https://${USER_FROM_VAULT}:${PASS_FROM_VAULT}@${HOST_FROM_VAULT}"
+export ES_HOST
+
+STATIC_SITE_URL_BASE='https://kibana-coverage.elastic.dev'
+export STATIC_SITE_URL_BASE
+
+TEAM_ASSIGN_PATH=$5
+echo "### debug TEAM_ASSIGN_PATH: ${TEAM_ASSIGN_PATH}"
+
+BUFFER_SIZE=500
+export BUFFER_SIZE
+echo "### debug BUFFER_SIZE: ${BUFFER_SIZE}"
+
+# Build team assignments file
+CI_STATS_DISABLED=true node scripts/generate_team_assignments.js --verbose --src '.github/CODEOWNERS' --dest $TEAM_ASSIGN_PATH
+
+for x in functional jest; do
+  echo "### Ingesting coverage for ${x}"
+  COVERAGE_SUMMARY_FILE=target/kibana-coverage/${x}-combined/coverage-summary.json
+  # running in background to speed up ingestion
+  CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH &
+done
+wait
+
+echo "###  Ingesting Code Coverage - Complete"
+echo ""

.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cat << EOF > src/dev/code_coverage/www/index_partial_2.html
+        <a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/jest-combined/index.html">Latest Jest</a>
+        <a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/functional-combined/index.html">Latest FTR</a>
+      </nav>
+    </div>
+  </header>
+  <main role="main" class="inner cover">
+    <!--    <h1 class="cover-heading"> - Master Branch</h1>-->
+    <p class="lead">Use Kibana Stats to mine coverage data</p>
+    <p class="lead">
+      <a href="https://kibana-stats.elastic.dev/app/kibana#/dashboard/58b8db70-62f9-11ea-8312-7f2d69b79843?_g=(filters%3A!()%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-7d%2Cto%3Anow))" class="btn btn-lg btn-primary">Dashboard</a>
+    </p>
+  </main>
+  <footer class="mastfoot mt-auto">
+    <div class="inner">
+      <p>Please slack us at <a href="https://app.slack.com/client/T0CUZ52US/C0TR0FAET">#kibana-qa</a> if youve questions</p>
+    </div>
+  </footer>
+</div>
+</body>
+</html>
+EOF
+
+cat src/dev/code_coverage/www/index_partial.html > src/dev/code_coverage/www/index.html
+cat src/dev/code_coverage/www/index_partial_2.html >> src/dev/code_coverage/www/index.html

.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+collectPrevious() {
+    PREVIOUS=$(git log --pretty=format:%h -1)
+    echo "### PREVIOUS: ${PREVIOUS}"
+    echo $PREVIOUS > previous.txt
+}
+collectPrevious
+
+gsutil cp previous.txt gs://elastic-bekitzur-kibana-coverage-live/previous_pointer/

.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+uploadPrefix="gs://elastic-bekitzur-kibana-coverage-live/"
+uploadPrefixWithTimeStamp="${uploadPrefix}${TIME_STAMP}/"
+
+cat src/dev/code_coverage/www/index.html
+
+for x in 'src/dev/code_coverage/www/index.html' 'src/dev/code_coverage/www/404.html'; do
+    gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefix}
+done
+
+gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}
+
+for x in 'target/kibana-coverage/functional-combined' 'target/kibana-coverage/jest-combined'; do
+    gsutil -m cp -r -a public-read -z js,css,html ${x} ${uploadPrefixWithTimeStamp}
+done

.buildkite/scripts/steps/code_coverage/xpack_cigroup.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana_plugins.sh
+
+is_test_execution_step
+
+
+export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
+export JOB=kibana-default-ciGroup${CI_GROUP}
+
+export NODE_OPTIONS=--max_old_space_size=8192
+export CODE_COVERAGE=1
+
+echo "--- Default CI Group $CI_GROUP"
+
+echo " -> Running X-Pack functional tests with code coverage"
+
+cd "$XPACK_DIR"
+
+node scripts/functional_tests \
+  --include-tag "ciGroup$CI_GROUP" \
+  --exclude-tag "skipCoverage" || true
+
+cd "$KIBANA_DIR"
+
+if [[ -d "$KIBANA_DIR/target/kibana-coverage/functional" ]]; then
+  echo "--- Merging code coverage for CI Group $CI_GROUP"
+  yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js --reporter json
+  rm -rf target/kibana-coverage/functional/*
+  mv target/kibana-coverage/functional-combined/coverage-final.json "target/kibana-coverage/functional/xpack-${CI_GROUP}-coverage.json"
+else
+  echo "--- Code coverage not found"
+fi
+
+cd "$KIBANA_DIR"

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -25,6 +25,8 @@ import { getBuildkiteMetadata } from './buildkite_metadata';
 import { ExistingFailedTestIssues } from './existing_failed_test_issues';
 
 const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
+const DISABLE_MISSING_TEST_REPORT_ERRORS =
+  process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true';
 
 export function runFailedTestsReporterCli() {
   run(
@@ -88,6 +90,11 @@ export function runFailedTestsReporterCli() {
           absolute: true,
         });
 
+        if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) {
+          // it is fine for code coverage to not have test results
+          return;
+        }
+
         if (!reportPaths.length) {
           throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
         }

src/dev/code_coverage/ingest_coverage/ingest.js

@@ -6,33 +6,33 @@
  * Side Public License, v 1.
  */
 
-const { Client } = require('@elastic/elasticsearch');
+const { Client, HttpConnection } = require('@elastic/elasticsearch');
 import { RESEARCH_CI_JOB_NAME } from './constants';
 import { whichIndex } from './ingest_helpers';
 import { fromNullable } from './either';
 import { always, id, flatMap, ccMark, lazyF } from './utils';
 
 const node = process.env.ES_HOST || 'http://localhost:9200';
-const client = new Client({ node });
+const client = new Client({
+  node,
+  maxRetries: 5,
+  requestTimeout: 60000,
+  Connection: HttpConnection,
+});
 const isResearchJob = process.env.COVERAGE_JOB_NAME === RESEARCH_CI_JOB_NAME ? true : false;
 
 export const ingestList = (log) => async (xs) => {
-  fromNullable(process.env.NODE_ENV).fold(bulkIngest, justLog);
+  await bulkIngest();
 
   async function bulkIngest() {
     log.verbose(`\n${ccMark} Ingesting ${xs.length} docs at a time`);
 
     const body = parseIndexes(xs);
 
-    const { body: bulkResponse } = await client.bulk({ refresh: true, body });
+    const bulkResponse = await client.bulk({ refresh: true, body });
 
     handleErrors(body, bulkResponse)(log);
   }
-
-  function justLog() {
-    log.verbose(`\n${ccMark} Just logging first item from current (buffered) bulk list`);
-    log.verbose(`\n${ccMark} ${JSON.stringify(xs[0], null, 2)}`);
-  }
 };
 
 function handleErrors(body, bulkResponse) {