mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[code coverage] removing instrumentation & plugin for functional tests (#148748)
## Summary Quite awhile ago we decided to stop collecting code coverage for our functional (e2e, api integration) FTR tests. This is a cleanup PR to remove the code we no longer use. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Dzmitry Lemechko
GitHub
parent
a1923c5f6a
commit
bc2cb5dc61
11 changed files with 11 additions and 324 deletions
|
|
@ -17,7 +17,6 @@ steps:
|
|||
LIMIT_CONFIG_TYPE: 'unit,integration'
|
||||
JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest.sh'
|
||||
JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest_integration.sh'
|
||||
FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/code_coverage/ftr_configs.sh'
|
||||
|
||||
- command: .buildkite/scripts/steps/code_coverage/ingest.sh
|
||||
label: 'Merge and Ingest'
|
||||
|
|
|
|||
|
|
@ -1,124 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/steps/code_coverage/merge.sh
|
||||
source .buildkite/scripts/steps/code_coverage/util.sh
|
||||
source .buildkite/scripts/steps/code_coverage/node_scripts.sh
|
||||
|
||||
export CODE_COVERAGE=1 # Kibana is bootstrapped differently for code coverage
|
||||
echo "--- KIBANA_DIR: $KIBANA_DIR"
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
buildPlatformPlugins
|
||||
is_test_execution_step
|
||||
|
||||
FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
|
||||
if [ "$FTR_CONFIG_GROUP_KEY" == "" ]; then
|
||||
echo "Missing FTR_CONFIG_GROUP_KEY env var"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export JOB="$FTR_CONFIG_GROUP_KEY"
|
||||
|
||||
functionalTarget="$KIBANA_DIR/target/kibana-coverage/functional"
|
||||
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
|
||||
|
||||
configs="${FTR_CONFIG:-}"
|
||||
|
||||
if [[ "$configs" == "" ]]; then
|
||||
echo "--- Downloading ftr test run order"
|
||||
download_artifact ftr_run_order.json .
|
||||
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
|
||||
fi
|
||||
|
||||
echo "--- Config(s) for this FTR Group:"
|
||||
echo "${configs[@]}"
|
||||
|
||||
failedConfigs=""
|
||||
results=()
|
||||
|
||||
while read -r config; do
|
||||
if [[ ! "$config" ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "--- Begin config $config"
|
||||
|
||||
start=$(date +%s)
|
||||
|
||||
# prevent non-zero exit code from breaking the loop
|
||||
set +e
|
||||
runFTRInstrumented "$config"
|
||||
lastCode=$?
|
||||
set -e
|
||||
dasherize() {
|
||||
local withoutExtension=${1%.*}
|
||||
dasherized=$(echo "$withoutExtension" | tr '\/' '\-')
|
||||
}
|
||||
dasherize "$config"
|
||||
|
||||
if [[ -d "$functionalTarget" ]]; then
|
||||
echo "--- Server and / or Client side code coverage collected"
|
||||
if [[ -f "target/kibana-coverage/functional/coverage-final.json" ]]; then
|
||||
# We potentially have more than one file with the same name being created,
|
||||
# so we make them unique here.
|
||||
mv target/kibana-coverage/functional/coverage-final.json "target/kibana-coverage/functional/${dasherized}-server-coverage.json"
|
||||
fi
|
||||
fi
|
||||
|
||||
timeSec=$(($(date +%s) - start))
|
||||
if [[ $timeSec -gt 60 ]]; then
|
||||
min=$((timeSec / 60))
|
||||
sec=$((timeSec - (min * 60)))
|
||||
duration="${min}m ${sec}s"
|
||||
else
|
||||
duration="${timeSec}s"
|
||||
fi
|
||||
|
||||
results+=("- $config
|
||||
duration: ${duration}
|
||||
result: ${lastCode}")
|
||||
|
||||
if [ $lastCode -ne 0 ]; then
|
||||
echo "FTR exited with code $lastCode"
|
||||
echo "^^^ +++"
|
||||
|
||||
if [[ "$failedConfigs" ]]; then
|
||||
failedConfigs="${failedConfigs}"$'\n'"$config"
|
||||
else
|
||||
failedConfigs="$config"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "--- Config complete: $config"
|
||||
done <<<"$configs"
|
||||
|
||||
# Each browser unload event, creates a new coverage file.
|
||||
# So, we merge them here.
|
||||
if [[ -d "$functionalTarget" ]]; then
|
||||
reportMergeFunctional
|
||||
uniqueifyFunctional "$(date +%s)"
|
||||
else
|
||||
echo "--- Code coverage not found in: $functionalTarget"
|
||||
fi
|
||||
|
||||
# Nyc uses matching absolute paths for reporting / merging
|
||||
# So, set all coverage json files to a specific prefx.
|
||||
# The prefix will be changed to the kibana dir, in the final stage,
|
||||
# so nyc doesnt error.
|
||||
echo "--- Normalize file paths prefix"
|
||||
replacePaths "$KIBANA_DIR/target/kibana-coverage/functional" "$KIBANA_DIR" "CC_REPLACEMENT_ANCHOR"
|
||||
|
||||
if [[ "$failedConfigs" ]]; then
|
||||
buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
|
||||
fi
|
||||
|
||||
echo "--- FTR configs complete, result(s):"
|
||||
printf "%s\n" "${results[@]}"
|
||||
echo ""
|
||||
|
||||
# So the last step "knows" this config ran
|
||||
uploadRanFile "functional"
|
||||
|
||||
# Force exit 0 to ensure the next build step starts.
|
||||
exit 0
|
||||
|
|
@ -4,7 +4,6 @@ set -euo pipefail
|
|||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/steps/code_coverage/util.sh
|
||||
source .buildkite/scripts/steps/code_coverage/merge.sh
|
||||
|
||||
export CODE_COVERAGE=1
|
||||
echo "--- Reading Kibana stats cluster creds from vault"
|
||||
|
|
@ -77,13 +76,7 @@ mergeAll() {
|
|||
replacePaths "$KIBANA_DIR/target/kibana-coverage/jest" "CC_REPLACEMENT_ANCHOR" "$KIBANA_DIR"
|
||||
yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js
|
||||
elif [ "$x" == "functional" ]; then
|
||||
echo "---[$x] : Reset file paths prefix, merge coverage files, and generate the final combined report"
|
||||
set +e
|
||||
sed -ie "s|CC_REPLACEMENT_ANCHOR|${KIBANA_DIR}|g" target/kibana-coverage/functional/*.json
|
||||
echo "--- Begin Split and Merge for Functional"
|
||||
splitCoverage target/kibana-coverage/functional
|
||||
splitMerge
|
||||
set -e
|
||||
echo "--- Code coverage for functional tests is not collected"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
|
||||
export CODE_COVERAGE=1
|
||||
|
||||
base=target/kibana-coverage
|
||||
target="$base/functional"
|
||||
first="$target/first"
|
||||
rest="$target/rest"
|
||||
|
||||
filesCount() {
|
||||
count=$(find "$1" -maxdepth 1 -type f | wc -l | xargs) # xargs trims whitespace
|
||||
}
|
||||
|
||||
_head() {
|
||||
firstFile=$(find "$1" -maxdepth 1 -type f | head -1)
|
||||
}
|
||||
|
||||
splitCoverage() {
|
||||
echo "--- Running splitCoverage:"
|
||||
filesCount "$1"
|
||||
echo "### total: $count"
|
||||
|
||||
mkdir -p $first
|
||||
mkdir -p $rest
|
||||
half=$((count / 2))
|
||||
echo "### half: $half"
|
||||
|
||||
echo "### Move the first half into the 'first' dir"
|
||||
# the index variable is irrelevant
|
||||
for x in $(seq 1 $half); do
|
||||
_head "$1"
|
||||
mv "$firstFile" "$first"
|
||||
done
|
||||
|
||||
echo "### Move the second half into the 'rest' dir"
|
||||
while read -r x; do
|
||||
mv "$x" "$rest" || printf "\n\t### Trouble moving %s to %s" "$x" "$rest"
|
||||
done <<<"$(find "$target" -maxdepth 1 -type f -name '*.json')"
|
||||
}
|
||||
|
||||
splitMerge() {
|
||||
echo "### Merge the 1st half of the coverage files"
|
||||
yarn nyc merge target/kibana-coverage/functional/first target/kibana-coverage/functional/first.json
|
||||
echo "### Merge the 2nd half of the coverage files"
|
||||
yarn nyc merge target/kibana-coverage/functional/rest target/kibana-coverage/functional/rest.json
|
||||
echo "### Report-Merge the 2 halves into one"
|
||||
yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.functional.config.js
|
||||
}
|
||||
|
||||
uniqueifyFunctional() {
|
||||
local unique=${1:?"Must pass first positional arg for 'unique'"}
|
||||
|
||||
# Drop the json files that where report-merged.
|
||||
rm -rf target/kibana-coverage/functional/*
|
||||
|
||||
# Move from report-merge target dir, to: target/kibana-coverage/functional
|
||||
mv target/kibana-coverage/functional-combined/coverage-final.json \
|
||||
"target/kibana-coverage/functional/$unique-coverage-final.json"
|
||||
}
|
||||
|
|
@ -25,8 +25,6 @@ import { BundleMetricsPlugin } from './bundle_metrics_plugin';
|
|||
import { EmitStatsPlugin } from './emit_stats_plugin';
|
||||
import { PopulateBundleCachePlugin } from './populate_bundle_cache_plugin';
|
||||
|
||||
const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE;
|
||||
const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset');
|
||||
const BABEL_PRESET = [
|
||||
require.resolve('@kbn/babel-preset/webpack_preset'),
|
||||
{
|
||||
|
|
@ -232,7 +230,7 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
|
|||
options: {
|
||||
babelrc: false,
|
||||
envName: worker.dist ? 'production' : 'development',
|
||||
presets: IS_CODE_COVERAGE ? [ISTANBUL_PRESET_PATH, BABEL_PRESET] : [BABEL_PRESET],
|
||||
presets: [BABEL_PRESET],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -55,10 +55,6 @@ export default function () {
|
|||
`--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'otel_metrics')}`,
|
||||
`--newsfeed.service.urlRoot=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`,
|
||||
`--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json`,
|
||||
// code coverage reporting plugin
|
||||
...(!!process.env.CODE_COVERAGE
|
||||
? [`--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'coverage')}`]
|
||||
: []),
|
||||
`--logging.appenders.deprecation=${JSON.stringify({
|
||||
type: 'console',
|
||||
layout: {
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
{
|
||||
"id": "coverageFixtures",
|
||||
"owner": { "name": "Kibana Operations", "githubTeam": "kibana-operations" },
|
||||
"version": "kibana",
|
||||
"server": false,
|
||||
"ui": true
|
||||
}
|
||||
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { CodeCoverageReportingPlugin } from './plugin';
|
||||
|
||||
export function plugin() {
|
||||
return new CodeCoverageReportingPlugin();
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { Plugin } from '@kbn/core/server';
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
__coverage__: any;
|
||||
flushCoverageToLog: any;
|
||||
}
|
||||
}
|
||||
|
||||
export class CodeCoverageReportingPlugin implements Plugin {
|
||||
constructor() {}
|
||||
|
||||
public start() {}
|
||||
|
||||
public setup() {
|
||||
window.flushCoverageToLog = function () {
|
||||
if (window.__coverage__) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('coveragejson:' + btoa(JSON.stringify(window.__coverage__)));
|
||||
}
|
||||
};
|
||||
window.addEventListener('beforeunload', window.flushCoverageToLog);
|
||||
}
|
||||
}
|
||||
|
|
@ -6,11 +6,6 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Fs from 'fs';
|
||||
import { resolve } from 'path';
|
||||
|
||||
import { mergeMap } from 'rxjs/operators';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { initWebDriver, BrowserConfig } from './webdriver';
|
||||
import { Browsers } from './browsers';
|
||||
|
|
@ -20,10 +15,6 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
|
|||
const log = getService('log');
|
||||
const config = getService('config');
|
||||
const browserType: Browsers = config.get('browser.type');
|
||||
const collectCoverage: boolean = !!process.env.CODE_COVERAGE;
|
||||
const coveragePrefix = 'coveragejson:';
|
||||
const coverageDir = resolve(__dirname, '../../../../target/kibana-coverage/functional');
|
||||
let coverageCounter = 1;
|
||||
type BrowserStorage = 'sessionStorage' | 'localStorage';
|
||||
|
||||
const clearBrowserStorage = async (storageType: BrowserStorage) => {
|
||||
|
|
@ -36,24 +27,6 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
|
|||
}
|
||||
};
|
||||
|
||||
const writeCoverage = (coverageJson: string) => {
|
||||
// on CI we make hard link clone and run tests from kibana${process.env.CI_GROUP} root path
|
||||
const re = new RegExp(`kibana${process.env.CI_GROUP}`, 'g');
|
||||
const dir = process.env.CI ? coverageDir.replace(re, 'kibana') : coverageDir;
|
||||
|
||||
if (!Fs.existsSync(dir)) {
|
||||
Fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
const id = coverageCounter++;
|
||||
const timestamp = Date.now();
|
||||
const path = resolve(dir, `${id}.${timestamp}.coverage.json`);
|
||||
log.info('writing coverage to', path);
|
||||
|
||||
const jsonString = process.env.CI ? coverageJson.replace(re, 'kibana') : coverageJson;
|
||||
Fs.writeFileSync(path, JSON.stringify(JSON.parse(jsonString), null, 2));
|
||||
};
|
||||
|
||||
const browserConfig: BrowserConfig = {
|
||||
logPollingMs: config.get('browser.logPollingMs'),
|
||||
acceptInsecureCerts: config.get('browser.acceptInsecureCerts'),
|
||||
|
|
@ -62,11 +35,7 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
|
|||
const { driver, consoleLog$ } = await initWebDriver(log, browserType, lifecycle, browserConfig);
|
||||
const caps = await driver.getCapabilities();
|
||||
|
||||
log.info(
|
||||
`Remote initialized: ${caps.get('browserName')} ${caps.get(
|
||||
'browserVersion'
|
||||
)}, collectingCoverage=${collectCoverage}`
|
||||
);
|
||||
log.info(`Remote initialized: ${caps.get('browserName')} ${caps.get('browserVersion')}}`);
|
||||
|
||||
if ([Browsers.Chrome, Browsers.ChromiumEdge].includes(browserType)) {
|
||||
log.info(
|
||||
|
|
@ -74,29 +43,14 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
|
|||
);
|
||||
}
|
||||
|
||||
consoleLog$
|
||||
.pipe(
|
||||
mergeMap((logEntry) => {
|
||||
if (collectCoverage && logEntry.message.includes(coveragePrefix)) {
|
||||
const [, coverageJsonBase64] = logEntry.message.split(coveragePrefix);
|
||||
const coverageJson = Buffer.from(coverageJsonBase64, 'base64').toString('utf8');
|
||||
writeCoverage(coverageJson);
|
||||
|
||||
// filter out this message
|
||||
return [];
|
||||
}
|
||||
|
||||
return [logEntry];
|
||||
})
|
||||
)
|
||||
.subscribe({
|
||||
next({ message, level }) {
|
||||
const msg = message.replace(/\\n/g, '\n');
|
||||
log[level === 'SEVERE' || level === 'error' ? 'warning' : 'debug'](
|
||||
`browser[${level}] ${msg}`
|
||||
);
|
||||
},
|
||||
});
|
||||
consoleLog$.subscribe({
|
||||
next({ message, level }) {
|
||||
const msg = message.replace(/\\n/g, '\n');
|
||||
log[level === 'SEVERE' || level === 'error' ? 'warning' : 'debug'](
|
||||
`browser[${level}] ${msg}`
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
lifecycle.beforeTests.add(async () => {
|
||||
// hard coded default, can be overridden per suite using `browser.setWindowSize()`
|
||||
|
|
@ -121,17 +75,6 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
|
|||
});
|
||||
|
||||
lifecycle.cleanup.add(async () => {
|
||||
// Getting the last piece of code coverage before closing browser
|
||||
if (collectCoverage) {
|
||||
const coverageJson = await driver
|
||||
.executeScript('return window.__coverage__')
|
||||
.catch(() => undefined)
|
||||
.then((coverage) => (coverage ? JSON.stringify(coverage) : undefined));
|
||||
if (coverageJson) {
|
||||
writeCoverage(coverageJson);
|
||||
}
|
||||
}
|
||||
|
||||
await driver.quit();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -528,8 +528,6 @@
|
|||
"@kbn/core-usage-data-server-internal/*": ["packages/core/usage-data/core-usage-data-server-internal/*"],
|
||||
"@kbn/core-usage-data-server-mocks": ["packages/core/usage-data/core-usage-data-server-mocks"],
|
||||
"@kbn/core-usage-data-server-mocks/*": ["packages/core/usage-data/core-usage-data-server-mocks/*"],
|
||||
"@kbn/coverage-fixtures-plugin": ["test/common/fixtures/plugins/coverage"],
|
||||
"@kbn/coverage-fixtures-plugin/*": ["test/common/fixtures/plugins/coverage/*"],
|
||||
"@kbn/cross-cluster-replication-plugin": ["x-pack/plugins/cross_cluster_replication"],
|
||||
"@kbn/cross-cluster-replication-plugin/*": ["x-pack/plugins/cross_cluster_replication/*"],
|
||||
"@kbn/crypto": ["packages/kbn-crypto"],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue