kibana/test/plugin_functional/config.ts
Spencer 542b381fa5
[ftr] automatically determine config run order (#130983)
* [ftr] automatically determine config run order

* split lens config into two groups

* support ftr configs always running against CI

* Split detection_engine_api_integration rule exception list tests

* Add configs from previous commit

* [ftr] remove testMetadata and maintain a unique lifecycle instance per run

* Revert "[ftr] remove testMetadata and maintain a unique lifecycle instance per run"

This reverts commit d2b4fdb824.

* Split alerting_api_integration/security_and_spaces tests

* Add groups to yaml

* Revert "Revert "[ftr] remove testMetadata and maintain a unique lifecycle instance per run""

This reverts commit 56232eea68.

* stop ES more forcefully and fix timeout

* only cleanup lifecycle phases when the cleanup is totally complete

* only use kill when cleaning up an esTestInstance

* fix broken import

* fix runOptions.alwaysUseSource implementation

* fix config access

* fix x-pack/ccs config

* fix ml import file paths

* update kibana build id

* revert array.concat() change

* fix baseConfig usage

* fix pie chart data

* split up maps tests

* pull in all of group5 so that es archives are loaded correctly

* add to ftr configs.yml

* fix pie chart data without breaking legacy version

* fix more pie_chart stuff in new vis lib

* restore normal PR tasks

* bump kibana-buildkite-library

* remove ciGroup validation

* remove the script which is no longer called from checks.sh

* [CI] Auto-commit changed files from 'yarn kbn run build -i @kbn/pm'

* adapt flaky test runner scripts to handle ftrConfig paths

* fix types in alerting_api_integration

* improve flaky config parsing and use non-local var name for passing explicit configs to ftr_configs.sh

* Split xpack dashboard tests

* Add configs

* [flaky] remove key from ftr-config steps

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* restore cypress builds

* remove ciGroups from FTR config files

* fixup some docs

* add temporary script to hunt for FTR config files

* use config.base.js naming for clarity

* use script to power ftr_configs.yml

* remove usage of removed x-pack/scripts/functional_tests

* fix test names in dashboard snapshots

* bump kibana-buildkite-library

* Try retrying only failed configs

* be a little quieter about trying to get testStats from configs with testRunners defined

* Remove test code

* bump kibana-buildkite-library

* update es_snapshot and on_merge jobs too

* track duration and exit code for each config and print it at the end of the script

* store results in order, rather than by key, in case there are duplicates in $config

* bash is hard

* fix env source and use +e rather than disabling e for whole file

* bash sucks

* print config summary in jest jobs too

* define results in jest_parallel.sh

* simplify config summary print, format times a little better

* fix reference to unbound time variable, use better variable name

* skip the newline between each result

* finish with the nitpicking

* sync changes with ftr_configs.sh

* refuse to execute config files which aren't listed in the .buildkite/ftr_configs.yml

* fix config.edge.js base config import paths

* fix some readmes

* resolve paths from ftr_configs manifest

* fix readConfigFile tests

* just allow __fixtures__ configs

* list a few more cypress config files

* install the main branch of kibana-buildkite-library

* split up lens group1

* move ml data_visualizer tests to their own config

* fix import paths

* fix more imports

* install specific commit of buildkite-pipeline-library

* sort configs in ftr_configs.yml

* bump kibana-buildkite-library

* remove temporary script

* fix env var for limiting config types

* Update docs/developer/contributing/development-functional-tests.asciidoc

Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>

* produce a JUnit report for saved objects field count

* apply standard concurrency limits from flaky test runner

* support customizing FTR concurrency via the env

Co-authored-by: Brian Seeders <brian.seeders@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>
2022-05-04 17:05:58 -05:00

69 lines
2.7 KiB
TypeScript

/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
import path from 'path';
import fs from 'fs';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
// Find all folders in ./plugins since we treat all them as plugin folder
const allFiles = fs.readdirSync(path.resolve(__dirname, 'plugins'));
const plugins = allFiles.filter((file) =>
fs.statSync(path.resolve(__dirname, 'plugins', file)).isDirectory()
);
return {
rootTags: ['runOutsideOfCiGroups'],
testFiles: [
require.resolve('./test_suites/usage_collection'),
require.resolve('./test_suites/telemetry'),
require.resolve('./test_suites/core'),
require.resolve('./test_suites/custom_visualizations'),
require.resolve('./test_suites/panel_actions'),
require.resolve('./test_suites/core_plugins'),
require.resolve('./test_suites/management'),
require.resolve('./test_suites/application_links'),
require.resolve('./test_suites/data_plugin'),
require.resolve('./test_suites/saved_objects_management'),
require.resolve('./test_suites/saved_objects_hidden_type'),
],
services: {
...functionalConfig.get('services'),
},
pageObjects: functionalConfig.get('pageObjects'),
servers: functionalConfig.get('servers'),
esTestCluster: {
...functionalConfig.get('esTestCluster'),
serverArgs: ['xpack.security.enabled=false'],
},
apps: functionalConfig.get('apps'),
screenshots: functionalConfig.get('screenshots'),
junit: {
reportName: 'Plugin Functional Tests',
},
kbnTestServer: {
...functionalConfig.get('kbnTestServer'),
serverArgs: [
...functionalConfig.get('kbnTestServer.serverArgs'),
// Required to load new platform plugins via `--plugin-path` flag.
'--env.name=development',
'--corePluginDeprecations.oldProperty=hello',
'--corePluginDeprecations.secret=100',
'--corePluginDeprecations.noLongerUsed=still_using',
// for testing set buffer duration to 0 to immediately flush counters into saved objects.
'--usageCollection.usageCounters.bufferDuration=0',
...plugins.map(
(pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`
),
],
},
};
}