mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[ftr] automatically determine config run order (#130983)
* [ftr] automatically determine config run order * split lens config into two groups * support ftr configs always running against CI * Split detection_engine_api_integration rule exception list tests * Add configs from previous commit * [ftr] remove testMetadata and maintain a unique lifecycle instance per run * Revert "[ftr] remove testMetadata and maintain a unique lifecycle instance per run" This reverts commitd2b4fdb824. * Split alerting_api_integration/security_and_spaces tests * Add groups to yaml * Revert "Revert "[ftr] remove testMetadata and maintain a unique lifecycle instance per run"" This reverts commit56232eea68. * stop ES more forcefully and fix timeout * only cleanup lifecycle phases when the cleanup is totally complete * only use kill when cleaning up an esTestInstance * fix broken import * fix runOptions.alwaysUseSource implementation * fix config access * fix x-pack/ccs config * fix ml import file paths * update kibana build id * revert array.concat() change * fix baseConfig usage * fix pie chart data * split up maps tests * pull in all of group5 so that es archives are loaded correctly * add to ftr configs.yml * fix pie chart data without breaking legacy version * fix more pie_chart stuff in new vis lib * restore normal PR tasks * bump kibana-buildkite-library * remove ciGroup validation * remove the script which is no longer called from checks.sh * [CI] Auto-commit changed files from 'yarn kbn run build -i @kbn/pm' * adapt flaky test runner scripts to handle ftrConfig paths * fix types in alerting_api_integration * improve flaky config parsing and use non-local var name for passing explicit configs to ftr_configs.sh * Split xpack dashboard tests * Add configs * [flaky] remove key from ftr-config steps * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * restore cypress builds * remove ciGroups from FTR config files * fixup some docs * add temporary script to hunt for FTR config files * use config.base.js naming for clarity * use script to power ftr_configs.yml * remove usage of removed x-pack/scripts/functional_tests * fix test names in dashboard snapshots * bump kibana-buildkite-library * Try retrying only failed configs * be a little quieter about trying to get testStats from configs with testRunners defined * Remove test code * bump kibana-buildkite-library * update es_snapshot and on_merge jobs too * track duration and exit code for each config and print it at the end of the script * store results in order, rather than by key, in case there are duplicates in $config * bash is hard * fix env source and use +e rather than disabling e for whole file * bash sucks * print config summary in jest jobs too * define results in jest_parallel.sh * simplify config summary print, format times a little better * fix reference to unbound time variable, use better variable name * skip the newline between each result * finish with the nitpicking * sync changes with ftr_configs.sh * refuse to execute config files which aren't listed in the .buildkite/ftr_configs.yml * fix config.edge.js base config import paths * fix some readmes * resolve paths from ftr_configs manifest * fix readConfigFile tests * just allow __fixtures__ configs * list a few more cypress config files * install the main branch of kibana-buildkite-library * split up lens group1 * move ml data_visualizer tests to their own config * fix import paths * fix more imports * install specific commit of buildkite-pipeline-library * sort configs in ftr_configs.yml * bump kibana-buildkite-library * remove temporary script * fix env var for limiting config types * Update docs/developer/contributing/development-functional-tests.asciidoc Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co> * produce a JUnit report for saved objects field count * apply standard concurrency limits from flaky test runner * support customizing FTR concurrency via the env Co-authored-by: Brian Seeders <brian.seeders@elastic.co> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>
This commit is contained in:
Spencer
GitHub
parent
fb453aca45
commit
542b381fa5
850 changed files with 4367 additions and 2774 deletions
236
.buildkite/ftr_configs.yml
Normal file
236
.buildkite/ftr_configs.yml
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
disabled:
|
||||
# TODO: Enable once RBAC timeline search strategy test updated
|
||||
- x-pack/test/timeline/security_and_spaces/config_basic.ts
|
||||
|
||||
# Base config files, only necessary to inform config finding script
|
||||
- test/functional/config.base.js
|
||||
- x-pack/test/functional/config.base.js
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/config.base.ts
|
||||
- x-pack/test/functional_enterprise_search/base_config.ts
|
||||
- test/server_integration/config.base.js
|
||||
|
||||
# QA suites that are run out-of-band
|
||||
- x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
|
||||
- x-pack/test/upgrade/config.ts
|
||||
- test/functional/config.edge.js
|
||||
- x-pack/test/functional/config.edge.js
|
||||
|
||||
# Cypress configs, for now these are still run manually
|
||||
- x-pack/test/fleet_cypress/cli_config.ts
|
||||
- x-pack/test/fleet_cypress/config.ts
|
||||
- x-pack/test/fleet_cypress/visual_config.ts
|
||||
- x-pack/test/functional_enterprise_search/cypress.config.ts
|
||||
- x-pack/test/osquery_cypress/cli_config.ts
|
||||
- x-pack/test/osquery_cypress/config.ts
|
||||
- x-pack/test/osquery_cypress/visual_config.ts
|
||||
- x-pack/test/security_solution_cypress/cases_cli_config.ts
|
||||
- x-pack/test/security_solution_cypress/ccs_config.ts
|
||||
- x-pack/test/security_solution_cypress/cli_config.ts
|
||||
- x-pack/test/security_solution_cypress/config.firefox.ts
|
||||
- x-pack/test/security_solution_cypress/config.ts
|
||||
- x-pack/test/security_solution_cypress/response_ops_cli_config.ts
|
||||
- x-pack/test/security_solution_cypress/upgrade_config.ts
|
||||
- x-pack/test/security_solution_cypress/visual_config.ts
|
||||
- x-pack/test/functional_enterprise_search/with_host_configured.config.ts
|
||||
- x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
|
||||
- x-pack/plugins/apm/ftr_e2e/ftr_config.ts
|
||||
- x-pack/plugins/synthetics/e2e/config.ts
|
||||
|
||||
# Configs that exist but weren't running in CI when this file was introduced
|
||||
- test/visual_regression/config.ts
|
||||
- x-pack/test/visual_regression/config.ts
|
||||
- x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
|
||||
- x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts
|
||||
- x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts
|
||||
- x-pack/test/banners_functional/config.ts
|
||||
- x-pack/test/cloud_integration/config.ts
|
||||
- x-pack/test/performance/config.playwright.ts
|
||||
- x-pack/test/load/config.ts
|
||||
- x-pack/test/plugin_api_perf/config.js
|
||||
- x-pack/test/screenshot_creation/config.ts
|
||||
|
||||
enabled:
|
||||
- test/accessibility/config.ts
|
||||
- test/analytics/config.ts
|
||||
- test/api_integration/config.js
|
||||
- test/examples/config.js
|
||||
- test/functional/apps/bundles/config.ts
|
||||
- test/functional/apps/console/config.ts
|
||||
- test/functional/apps/context/config.ts
|
||||
- test/functional/apps/dashboard_elements/config.ts
|
||||
- test/functional/apps/dashboard/group1/config.ts
|
||||
- test/functional/apps/dashboard/group2/config.ts
|
||||
- test/functional/apps/dashboard/group3/config.ts
|
||||
- test/functional/apps/dashboard/group4/config.ts
|
||||
- test/functional/apps/dashboard/group5/config.ts
|
||||
- test/functional/apps/discover/config.ts
|
||||
- test/functional/apps/getting_started/config.ts
|
||||
- test/functional/apps/home/config.ts
|
||||
- test/functional/apps/management/config.ts
|
||||
- test/functional/apps/saved_objects_management/config.ts
|
||||
- test/functional/apps/status_page/config.ts
|
||||
- test/functional/apps/visualize/group1/config.ts
|
||||
- test/functional/apps/visualize/group2/config.ts
|
||||
- test/functional/apps/visualize/group3/config.ts
|
||||
- test/functional/apps/visualize/group4/config.ts
|
||||
- test/functional/apps/visualize/group5/config.ts
|
||||
- test/functional/apps/visualize/group6/config.ts
|
||||
- test/functional/apps/visualize/replaced_vislib_chart_types/config.ts
|
||||
- test/functional/config.ccs.ts
|
||||
- test/functional/config.firefox.js
|
||||
- test/interactive_setup_api_integration/enrollment_flow.config.ts
|
||||
- test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
|
||||
- test/interactive_setup_api_integration/manual_configuration_flow.config.ts
|
||||
- test/interactive_setup_functional/enrollment_token.config.ts
|
||||
- test/interactive_setup_functional/manual_configuration_without_security.config.ts
|
||||
- test/interactive_setup_functional/manual_configuration_without_tls.config.ts
|
||||
- test/interactive_setup_functional/manual_configuration.config.ts
|
||||
- test/interpreter_functional/config.ts
|
||||
- test/new_visualize_flow/config.ts
|
||||
- test/plugin_functional/config.ts
|
||||
- test/server_integration/http/platform/config.status.ts
|
||||
- test/server_integration/http/platform/config.ts
|
||||
- test/server_integration/http/ssl_redirect/config.js
|
||||
- test/server_integration/http/ssl_with_p12_intermediate/config.js
|
||||
- test/server_integration/http/ssl_with_p12/config.js
|
||||
- test/server_integration/http/ssl/config.js
|
||||
- test/ui_capabilities/newsfeed_err/config.ts
|
||||
- x-pack/test/accessibility/config.ts
|
||||
- x-pack/test/alerting_api_integration/basic/config.ts
|
||||
- x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
|
||||
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
|
||||
- x-pack/test/alerting_api_integration/spaces_only/config.ts
|
||||
- x-pack/test/api_integration_basic/config.ts
|
||||
- x-pack/test/api_integration/config_security_basic.ts
|
||||
- x-pack/test/api_integration/config_security_trial.ts
|
||||
- x-pack/test/api_integration/config.ts
|
||||
- x-pack/test/apm_api_integration/basic/config.ts
|
||||
- x-pack/test/apm_api_integration/rules/config.ts
|
||||
- x-pack/test/apm_api_integration/trial/config.ts
|
||||
- x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts
|
||||
- x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts
|
||||
- x-pack/test/cases_api_integration/spaces_only/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/basic/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group2/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group3/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts
|
||||
- x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
|
||||
- x-pack/test/encrypted_saved_objects_api_integration/config.ts
|
||||
- x-pack/test/endpoint_api_integration_no_ingest/config.ts
|
||||
- x-pack/test/examples/config.ts
|
||||
- x-pack/test/fleet_api_integration/config.ts
|
||||
- x-pack/test/fleet_functional/config.ts
|
||||
- x-pack/test/functional_basic/config.ts
|
||||
- x-pack/test/functional_cors/config.ts
|
||||
- x-pack/test/functional_embedded/config.ts
|
||||
- x-pack/test/functional_enterprise_search/without_host_configured.config.ts
|
||||
- x-pack/test/functional_execution_context/config.ts
|
||||
- x-pack/test/functional_synthetics/config.js
|
||||
- x-pack/test/functional_with_es_ssl/config.ts
|
||||
- x-pack/test/functional/apps/advanced_settings/config.ts
|
||||
- x-pack/test/functional/apps/api_keys/config.ts
|
||||
- x-pack/test/functional/apps/apm/config.ts
|
||||
- x-pack/test/functional/apps/canvas/config.ts
|
||||
- x-pack/test/functional/apps/cross_cluster_replication/config.ts
|
||||
- x-pack/test/functional/apps/dashboard/group1/config.ts
|
||||
- x-pack/test/functional/apps/dashboard/group2/config.ts
|
||||
- x-pack/test/functional/apps/data_views/config.ts
|
||||
- x-pack/test/functional/apps/dev_tools/config.ts
|
||||
- x-pack/test/functional/apps/discover/config.ts
|
||||
- x-pack/test/functional/apps/graph/config.ts
|
||||
- x-pack/test/functional/apps/grok_debugger/config.ts
|
||||
- x-pack/test/functional/apps/home/config.ts
|
||||
- x-pack/test/functional/apps/index_lifecycle_management/config.ts
|
||||
- x-pack/test/functional/apps/index_management/config.ts
|
||||
- x-pack/test/functional/apps/infra/config.ts
|
||||
- x-pack/test/functional/apps/ingest_pipelines/config.ts
|
||||
- x-pack/test/functional/apps/lens/group1/config.ts
|
||||
- x-pack/test/functional/apps/lens/group2/config.ts
|
||||
- x-pack/test/functional/apps/lens/group3/config.ts
|
||||
- x-pack/test/functional/apps/license_management/config.ts
|
||||
- x-pack/test/functional/apps/logstash/config.ts
|
||||
- x-pack/test/functional/apps/management/config.ts
|
||||
- x-pack/test/functional/apps/maps/group1/config.ts
|
||||
- x-pack/test/functional/apps/maps/group2/config.ts
|
||||
- x-pack/test/functional/apps/maps/group3/config.ts
|
||||
- x-pack/test/functional/apps/maps/group4/config.ts
|
||||
- x-pack/test/functional/apps/ml/data_visualizer/config.ts
|
||||
- x-pack/test/functional/apps/ml/group1/config.ts
|
||||
- x-pack/test/functional/apps/ml/group2/config.ts
|
||||
- x-pack/test/functional/apps/ml/group3/config.ts
|
||||
- x-pack/test/functional/apps/monitoring/config.ts
|
||||
- x-pack/test/functional/apps/remote_clusters/config.ts
|
||||
- x-pack/test/functional/apps/reporting_management/config.ts
|
||||
- x-pack/test/functional/apps/rollup_job/config.ts
|
||||
- x-pack/test/functional/apps/saved_objects_management/config.ts
|
||||
- x-pack/test/functional/apps/security/config.ts
|
||||
- x-pack/test/functional/apps/snapshot_restore/config.ts
|
||||
- x-pack/test/functional/apps/spaces/config.ts
|
||||
- x-pack/test/functional/apps/status_page/config.ts
|
||||
- x-pack/test/functional/apps/transform/config.ts
|
||||
- x-pack/test/functional/apps/upgrade_assistant/config.ts
|
||||
- x-pack/test/functional/apps/uptime/config.ts
|
||||
- x-pack/test/functional/apps/visualize/config.ts
|
||||
- x-pack/test/functional/apps/watcher/config.ts
|
||||
- x-pack/test/functional/config_security_basic.ts
|
||||
- x-pack/test/functional/config.ccs.ts
|
||||
- x-pack/test/functional/config.firefox.js
|
||||
- x-pack/test/licensing_plugin/config.public.ts
|
||||
- x-pack/test/licensing_plugin/config.ts
|
||||
- x-pack/test/lists_api_integration/security_and_spaces/config.ts
|
||||
- x-pack/test/observability_api_integration/basic/config.ts
|
||||
- x-pack/test/observability_api_integration/trial/config.ts
|
||||
- x-pack/test/observability_functional/with_rac_write.config.ts
|
||||
- x-pack/test/plugin_api_integration/config.ts
|
||||
- x-pack/test/plugin_functional/config.ts
|
||||
- x-pack/test/reporting_api_integration/reporting_and_security.config.ts
|
||||
- x-pack/test/reporting_api_integration/reporting_without_security.config.ts
|
||||
- x-pack/test/reporting_functional/reporting_and_deprecated_security.config.ts
|
||||
- x-pack/test/reporting_functional/reporting_and_security.config.ts
|
||||
- x-pack/test/reporting_functional/reporting_without_security.config.ts
|
||||
- x-pack/test/rule_registry/security_and_spaces/config_basic.ts
|
||||
- x-pack/test/rule_registry/security_and_spaces/config_trial.ts
|
||||
- x-pack/test/rule_registry/spaces_only/config_basic.ts
|
||||
- x-pack/test/rule_registry/spaces_only/config_trial.ts
|
||||
- x-pack/test/saved_object_api_integration/security_and_spaces/config_basic.ts
|
||||
- x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts
|
||||
- x-pack/test/saved_object_api_integration/spaces_only/config.ts
|
||||
- x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts
|
||||
- x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts
|
||||
- x-pack/test/saved_object_tagging/functional/config.ts
|
||||
- x-pack/test/saved_objects_field_count/config.ts
|
||||
- x-pack/test/search_sessions_integration/config.ts
|
||||
- x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts
|
||||
- x-pack/test/security_api_integration/anonymous.config.ts
|
||||
- x-pack/test/security_api_integration/audit.config.ts
|
||||
- x-pack/test/security_api_integration/http_bearer.config.ts
|
||||
- x-pack/test/security_api_integration/http_no_auth_providers.config.ts
|
||||
- x-pack/test/security_api_integration/kerberos_anonymous_access.config.ts
|
||||
- x-pack/test/security_api_integration/kerberos.config.ts
|
||||
- x-pack/test/security_api_integration/login_selector.config.ts
|
||||
- x-pack/test/security_api_integration/oidc_implicit_flow.config.ts
|
||||
- x-pack/test/security_api_integration/oidc.config.ts
|
||||
- x-pack/test/security_api_integration/pki.config.ts
|
||||
- x-pack/test/security_api_integration/saml.config.ts
|
||||
- x-pack/test/security_api_integration/session_idle.config.ts
|
||||
- x-pack/test/security_api_integration/session_invalidate.config.ts
|
||||
- x-pack/test/security_api_integration/session_lifespan.config.ts
|
||||
- x-pack/test/security_api_integration/token.config.ts
|
||||
- x-pack/test/security_functional/login_selector.config.ts
|
||||
- x-pack/test/security_functional/oidc.config.ts
|
||||
- x-pack/test/security_functional/saml.config.ts
|
||||
- x-pack/test/security_solution_endpoint_api_int/config.ts
|
||||
- x-pack/test/security_solution_endpoint/config.ts
|
||||
- x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
|
||||
- x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts
|
||||
- x-pack/test/spaces_api_integration/spaces_only/config.ts
|
||||
- x-pack/test/timeline/security_and_spaces/config_trial.ts
|
||||
- x-pack/test/ui_capabilities/security_and_spaces/config.ts
|
||||
- x-pack/test/ui_capabilities/spaces_only/config.ts
|
||||
- x-pack/test/upgrade_assistant_integration/config.js
|
||||
- x-pack/test/usage_collection/config.ts
|
||||
12
.buildkite/package-lock.json
generated
12
.buildkite/package-lock.json
generated
|
|
@ -8,7 +8,7 @@
|
|||
"name": "kibana-buildkite",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
|
||||
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
|
|
@ -355,8 +355,8 @@
|
|||
},
|
||||
"node_modules/kibana-buildkite-library": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "git+https://git@github.com/elastic/kibana-buildkite-library.git#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
|
||||
"integrity": "sha512-Ayiyy3rAE/jOWcR65vxiv4zacMlpxuRZ+WKvly6magfClWTWIUTcW1aiOH2/PYWP3faiCbIDHOyxLeGGajk5dQ==",
|
||||
"resolved": "git+https://git@github.com/elastic/kibana-buildkite-library.git#f5381bea52e0a71f50a6919cb6357ff3262cf2d6",
|
||||
"integrity": "sha512-L1JP2NvXR7mhKn9JBwROPgTEV4vDr5HWwZtkdvxtHjZ/MeOnJYFSDqB4JUY/gXTz6v3CO3eUm3GQ0BP/kewoqQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@octokit/rest": "^18.10.0",
|
||||
|
|
@ -801,9 +801,9 @@
|
|||
}
|
||||
},
|
||||
"kibana-buildkite-library": {
|
||||
"version": "git+https://git@github.com/elastic/kibana-buildkite-library.git#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
|
||||
"integrity": "sha512-Ayiyy3rAE/jOWcR65vxiv4zacMlpxuRZ+WKvly6magfClWTWIUTcW1aiOH2/PYWP3faiCbIDHOyxLeGGajk5dQ==",
|
||||
"from": "kibana-buildkite-library@git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
|
||||
"version": "git+https://git@github.com/elastic/kibana-buildkite-library.git#f5381bea52e0a71f50a6919cb6357ff3262cf2d6",
|
||||
"integrity": "sha512-L1JP2NvXR7mhKn9JBwROPgTEV4vDr5HWwZtkdvxtHjZ/MeOnJYFSDqB4JUY/gXTz6v3CO3eUm3GQ0BP/kewoqQ==",
|
||||
"from": "kibana-buildkite-library@git+https://git@github.com/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6",
|
||||
"requires": {
|
||||
"@octokit/rest": "^18.10.0",
|
||||
"axios": "^0.21.4",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
|
||||
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,50 +27,17 @@ steps:
|
|||
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
|
||||
timeout_in_minutes: 60
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
|
||||
label: 'Default CI Group'
|
||||
parallelism: 31
|
||||
agents:
|
||||
queue: n2-4
|
||||
depends_on: build
|
||||
timeout_in_minutes: 150
|
||||
key: default-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
|
||||
label: 'OSS CI Group'
|
||||
parallelism: 12
|
||||
agents:
|
||||
queue: ci-group-4d
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
key: oss-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
|
||||
label: 'Pick Jest Config Run Order'
|
||||
- command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
|
||||
label: 'Pick Test Group Run Order'
|
||||
agents:
|
||||
queue: kibana-default
|
||||
env:
|
||||
FILTER_JEST_CONFIG_TYPE: integration
|
||||
LIMIT_CONFIG_TYPE: integration,functional
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/api_integration.sh
|
||||
label: 'API Integration Tests'
|
||||
agents:
|
||||
queue: n2-2
|
||||
timeout_in_minutes: 120
|
||||
key: api-integration
|
||||
|
||||
- command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
|
||||
label: Trigger promotion
|
||||
timeout_in_minutes: 10
|
||||
|
|
|
|||
|
|
@ -1,46 +1,20 @@
|
|||
{
|
||||
"groups": [
|
||||
{
|
||||
"key": "oss/cigroup",
|
||||
"name": "OSS CI Group",
|
||||
"ciGroups": 12
|
||||
},
|
||||
{
|
||||
"key": "oss/firefox",
|
||||
"name": "OSS Firefox"
|
||||
},
|
||||
{
|
||||
"key": "oss/accessibility",
|
||||
"name": "OSS Accessibility"
|
||||
},
|
||||
{
|
||||
"key": "xpack/cypress/security_solution",
|
||||
"key": "cypress/security_solution",
|
||||
"name": "Security Solution - Cypress"
|
||||
},
|
||||
{
|
||||
"key": "xpack/cypress/osquery_cypress",
|
||||
"key": "cypress/osquery_cypress",
|
||||
"name": "Osquery - Cypress"
|
||||
},
|
||||
{
|
||||
"key": "xpack/cypress/fleet_cypress",
|
||||
"key": "cypress/fleet_cypress",
|
||||
"name": "Fleet - Cypress"
|
||||
},
|
||||
{
|
||||
"key": "xpack/cypress/apm_cypress",
|
||||
"key": "cypress/apm_cypress",
|
||||
"name": "APM - Cypress"
|
||||
},
|
||||
{
|
||||
"key": "xpack/cigroup",
|
||||
"name": "Default CI Group",
|
||||
"ciGroups": 30
|
||||
},
|
||||
{
|
||||
"key": "xpack/firefox",
|
||||
"name": "Default Firefox"
|
||||
},
|
||||
{
|
||||
"key": "xpack/accessibility",
|
||||
"name": "Default Accessibility"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,18 @@ const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>}
|
|||
require('./groups.json').groups
|
||||
);
|
||||
|
||||
const concurrency = 25;
|
||||
const initialJobs = 3;
|
||||
const concurrency = process.env.KIBANA_FLAKY_TEST_CONCURRENCY
|
||||
? parseInt(process.env.KIBANA_FLAKY_TEST_CONCURRENCY, 10)
|
||||
: 25;
|
||||
|
||||
if (Number.isNaN(concurrency)) {
|
||||
throw new Error(
|
||||
`invalid KIBANA_FLAKY_TEST_CONCURRENCY: ${process.env.KIBANA_FLAKY_TEST_CONCURRENCY}`
|
||||
);
|
||||
}
|
||||
|
||||
const BASE_JOBS = 1;
|
||||
const MAX_JOBS = 500;
|
||||
|
||||
function getTestSuitesFromJson(json) {
|
||||
const fail = (errorMsg) => {
|
||||
|
|
@ -41,20 +51,40 @@ function getTestSuitesFromJson(json) {
|
|||
fail(`JSON test config must be an array`);
|
||||
}
|
||||
|
||||
/** @type {Array<{ key: string, count: number }>} */
|
||||
/** @type {Array<{ type: 'group', key: string; count: number } | { type: 'ftrConfig', ftrConfig: string; count: number }>} */
|
||||
const testSuites = [];
|
||||
for (const item of parsed) {
|
||||
if (typeof item !== 'object' || item === null) {
|
||||
fail(`testSuites must be objects`);
|
||||
}
|
||||
const key = item.key;
|
||||
if (typeof key !== 'string') {
|
||||
fail(`testSuite.key must be a string`);
|
||||
}
|
||||
|
||||
const count = item.count;
|
||||
if (typeof count !== 'number') {
|
||||
fail(`testSuite.count must be a number`);
|
||||
}
|
||||
|
||||
const type = item.type;
|
||||
if (type !== 'ftrConfig' && type !== 'group') {
|
||||
fail(`testSuite.type must be either "ftrConfig" or "group"`);
|
||||
}
|
||||
|
||||
if (item.type === 'ftrConfig') {
|
||||
const ftrConfig = item.ftrConfig;
|
||||
if (typeof ftrConfig !== 'string') {
|
||||
fail(`testSuite.ftrConfig must be a string`);
|
||||
}
|
||||
|
||||
testSuites.push({
|
||||
ftrConfig,
|
||||
count,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = item.key;
|
||||
if (typeof key !== 'string') {
|
||||
fail(`testSuite.key must be a string`);
|
||||
}
|
||||
testSuites.push({
|
||||
key,
|
||||
count,
|
||||
|
|
@ -65,13 +95,14 @@ function getTestSuitesFromJson(json) {
|
|||
}
|
||||
|
||||
const testSuites = getTestSuitesFromJson(configJson);
|
||||
const totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
|
||||
|
||||
if (totalJobs > 500) {
|
||||
const totalJobs = testSuites.reduce((acc, t) => acc + t.count, BASE_JOBS);
|
||||
|
||||
if (totalJobs > MAX_JOBS) {
|
||||
console.error('+++ Too many tests');
|
||||
console.error(
|
||||
`Buildkite builds can only contain 500 steps in total. Found ${totalJobs} in total. Make sure your test runs are less than ${
|
||||
500 - initialJobs
|
||||
`Buildkite builds can only contain ${MAX_JOBS} jobs in total. Found ${totalJobs} based on this config. Make sure your test runs are less than ${
|
||||
MAX_JOBS - BASE_JOBS
|
||||
}`
|
||||
);
|
||||
process.exit(1);
|
||||
|
|
@ -82,7 +113,7 @@ const pipeline = {
|
|||
env: {
|
||||
IGNORE_SHIP_CI_STATS_ERROR: 'true',
|
||||
},
|
||||
steps: steps,
|
||||
steps,
|
||||
};
|
||||
|
||||
steps.push({
|
||||
|
|
@ -94,76 +125,40 @@ steps.push({
|
|||
});
|
||||
|
||||
for (const testSuite of testSuites) {
|
||||
const TEST_SUITE = testSuite.key;
|
||||
const RUN_COUNT = testSuite.count;
|
||||
const UUID = process.env.UUID;
|
||||
|
||||
const JOB_PARTS = TEST_SUITE.split('/');
|
||||
const IS_XPACK = JOB_PARTS[0] === 'xpack';
|
||||
const TASK = JOB_PARTS[1];
|
||||
const CI_GROUP = JOB_PARTS.length > 2 ? JOB_PARTS[2] : '';
|
||||
|
||||
if (RUN_COUNT < 1) {
|
||||
if (testSuite.count <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (TASK) {
|
||||
case 'cigroup':
|
||||
if (IS_XPACK) {
|
||||
steps.push({
|
||||
command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
|
||||
label: `Default CI Group ${CI_GROUP}`,
|
||||
agents: { queue: 'n2-4' },
|
||||
depends_on: 'build',
|
||||
parallelism: RUN_COUNT,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: UUID,
|
||||
concurrency_method: 'eager',
|
||||
});
|
||||
} else {
|
||||
steps.push({
|
||||
command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
|
||||
label: `OSS CI Group ${CI_GROUP}`,
|
||||
agents: { queue: 'ci-group-4d' },
|
||||
depends_on: 'build',
|
||||
parallelism: RUN_COUNT,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: UUID,
|
||||
concurrency_method: 'eager',
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'firefox':
|
||||
steps.push({
|
||||
command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
|
||||
label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
|
||||
agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
|
||||
depends_on: 'build',
|
||||
parallelism: RUN_COUNT,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: UUID,
|
||||
concurrency_method: 'eager',
|
||||
});
|
||||
break;
|
||||
|
||||
case 'accessibility':
|
||||
steps.push({
|
||||
command: `.buildkite/scripts/steps/functional/${
|
||||
IS_XPACK ? 'xpack' : 'oss'
|
||||
}_accessibility.sh`,
|
||||
label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
|
||||
agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
|
||||
depends_on: 'build',
|
||||
parallelism: RUN_COUNT,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: UUID,
|
||||
concurrency_method: 'eager',
|
||||
});
|
||||
break;
|
||||
if (testSuite.ftrConfig) {
|
||||
steps.push({
|
||||
command: `.buildkite/scripts/steps/test/ftr_configs.sh`,
|
||||
env: {
|
||||
FTR_CONFIG: testSuite.ftrConfig,
|
||||
},
|
||||
label: `FTR Config: ${testSuite.ftrConfig}`,
|
||||
parallelism: testSuite.count,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: process.env.UUID,
|
||||
concurrency_method: 'eager',
|
||||
agents: {
|
||||
queue: 'n2-4-spot-2',
|
||||
},
|
||||
depends_on: 'build',
|
||||
timeout_in_minutes: 150,
|
||||
retry: {
|
||||
automatic: [
|
||||
{ exit_status: '-1', limit: 3 },
|
||||
// { exit_status: '*', limit: 1 },
|
||||
],
|
||||
},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const keyParts = testSuite.key.split('/');
|
||||
switch (keyParts[0]) {
|
||||
case 'cypress':
|
||||
const CYPRESS_SUITE = CI_GROUP;
|
||||
const CYPRESS_SUITE = keyParts[1];
|
||||
const group = groups.find((group) => group.key.includes(CYPRESS_SUITE));
|
||||
if (!group) {
|
||||
throw new Error(
|
||||
|
|
@ -175,12 +170,14 @@ for (const testSuite of testSuites) {
|
|||
label: group.name,
|
||||
agents: { queue: 'ci-group-6' },
|
||||
depends_on: 'build',
|
||||
parallelism: RUN_COUNT,
|
||||
parallelism: testSuite.count,
|
||||
concurrency: concurrency,
|
||||
concurrency_group: UUID,
|
||||
concurrency_group: process.env.UUID,
|
||||
concurrency_method: 'eager',
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new Error(`unknown test suite: ${testSuite.key}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,116 +49,8 @@ steps:
|
|||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
|
||||
label: 'Default CI Group'
|
||||
parallelism: 31
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 250
|
||||
key: default-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
|
||||
label: 'OSS CI Group'
|
||||
parallelism: 12
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
key: oss-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_accessibility.sh
|
||||
label: 'OSS Accessibility Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
|
||||
label: 'Default Accessibility Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_firefox.sh
|
||||
label: 'OSS Firefox Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_firefox.sh
|
||||
label: 'Default Firefox Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_misc.sh
|
||||
label: 'OSS Misc Functional Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
|
||||
label: 'Saved Object Field Metrics'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
|
||||
label: 'Pick Jest Config Run Order'
|
||||
- command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
|
||||
label: 'Pick Test Group Run Order'
|
||||
agents:
|
||||
queue: kibana-default
|
||||
retry:
|
||||
|
|
@ -166,17 +58,6 @@ steps:
|
|||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/api_integration.sh
|
||||
label: 'API Integration Tests'
|
||||
agents:
|
||||
queue: n2-2-spot
|
||||
timeout_in_minutes: 120
|
||||
key: api-integration
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
|
||||
- command: .buildkite/scripts/steps/lint.sh
|
||||
label: 'Linting'
|
||||
agents:
|
||||
|
|
|
|||
|
|
@ -15,116 +15,8 @@ steps:
|
|||
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
|
||||
timeout_in_minutes: 60
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
|
||||
label: 'Default CI Group'
|
||||
parallelism: 31
|
||||
agents:
|
||||
queue: n2-4-spot-2
|
||||
depends_on: build
|
||||
timeout_in_minutes: 150
|
||||
key: default-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
|
||||
label: 'OSS CI Group'
|
||||
parallelism: 12
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
key: oss-cigroup
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_accessibility.sh
|
||||
label: 'OSS Accessibility Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
|
||||
label: 'Default Accessibility Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_firefox.sh
|
||||
label: 'OSS Firefox Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_firefox.sh
|
||||
label: 'Default Firefox Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/oss_misc.sh
|
||||
label: 'OSS Misc Functional Tests'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
|
||||
label: 'Saved Object Field Metrics'
|
||||
agents:
|
||||
queue: n2-4-spot
|
||||
depends_on: build
|
||||
timeout_in_minutes: 120
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
|
||||
label: 'Pick Jest Config Run Order'
|
||||
- command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
|
||||
label: 'Pick Test Group Run Order'
|
||||
agents:
|
||||
queue: kibana-default
|
||||
retry:
|
||||
|
|
@ -132,17 +24,6 @@ steps:
|
|||
- exit_status: '*'
|
||||
limit: 1
|
||||
|
||||
- command: .buildkite/scripts/steps/test/api_integration.sh
|
||||
label: 'API Integration Tests'
|
||||
agents:
|
||||
queue: n2-2-spot
|
||||
timeout_in_minutes: 120
|
||||
key: api-integration
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: '-1'
|
||||
limit: 3
|
||||
|
||||
- command: .buildkite/scripts/steps/lint.sh
|
||||
label: 'Linting'
|
||||
agents:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
|
|||
.buildkite/scripts/steps/checks/commit/commit.sh
|
||||
.buildkite/scripts/steps/checks/bazel_packages.sh
|
||||
.buildkite/scripts/steps/checks/telemetry.sh
|
||||
.buildkite/scripts/steps/checks/validate_ci_groups.sh
|
||||
.buildkite/scripts/steps/checks/ts_projects.sh
|
||||
.buildkite/scripts/steps/checks/jest_configs.sh
|
||||
.buildkite/scripts/steps/checks/doc_api_changes.sh
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
|
||||
echo --- Ensure that all tests are in a CI Group
|
||||
checks-reporter-with-killswitch "Ensure that all tests are in a CI Group" \
|
||||
node scripts/ensure_all_tests_in_ci_group
|
||||
|
|
@ -11,10 +11,8 @@ export JOB=kibana-fleet-cypress
|
|||
|
||||
echo "--- Fleet Cypress tests"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "Fleet Cypress Tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/fleet_cypress/cli_config.ts
|
||||
--config x-pack/test/fleet_cypress/cli_config.ts
|
||||
|
|
|
|||
|
|
@ -12,10 +12,8 @@ export JOB=kibana-osquery-cypress
|
|||
|
||||
echo "--- Osquery Cypress tests"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "Osquery Cypress Tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--config test/osquery_cypress/cli_config.ts
|
||||
--config x-pack/test/osquery_cypress/cli_config.ts
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
echo --- OSS Accessibility Tests
|
||||
|
||||
checks-reporter-with-killswitch "Kibana accessibility tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/accessibility/config.ts
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
|
||||
export JOB=kibana-oss-ciGroup${CI_GROUP}
|
||||
|
||||
echo "--- OSS CI Group $CI_GROUP"
|
||||
|
||||
checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" \
|
||||
node scripts/functional_tests \
|
||||
--bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--include-tag "ciGroup$CI_GROUP"
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
echo --- OSS Firefox Smoke Tests
|
||||
|
||||
checks-reporter-with-killswitch "Firefox smoke test" \
|
||||
node scripts/functional_tests \
|
||||
--bail --debug \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--include-tag "includeFirefox" \
|
||||
--config test/functional/config.firefox.js
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
# Required, at least for plugin_functional tests
|
||||
.buildkite/scripts/build_kibana_plugins.sh
|
||||
|
||||
echo --- Plugin Functional Tests
|
||||
checks-reporter-with-killswitch "Plugin Functional Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/plugin_functional/config.ts \
|
||||
--bail \
|
||||
--debug
|
||||
|
||||
echo --- Interpreter Functional Tests
|
||||
checks-reporter-with-killswitch "Interpreter Functional Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/interpreter_functional/config.ts \
|
||||
--bail \
|
||||
--debug \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION"
|
||||
|
||||
echo --- Server Integration Tests
|
||||
checks-reporter-with-killswitch "Server Integration Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/server_integration/http/ssl/config.js \
|
||||
--config test/server_integration/http/ssl_redirect/config.js \
|
||||
--config test/server_integration/http/platform/config.ts \
|
||||
--config test/server_integration/http/ssl_with_p12/config.js \
|
||||
--config test/server_integration/http/ssl_with_p12_intermediate/config.js \
|
||||
--bail \
|
||||
--debug \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION"
|
||||
|
||||
# Tests that must be run against source in order to build test plugins
|
||||
echo --- Status Integration Tests
|
||||
checks-reporter-with-killswitch "Status Integration Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/server_integration/http/platform/config.status.ts \
|
||||
--bail \
|
||||
--debug
|
||||
|
||||
# Tests that must be run against source in order to build test plugins
|
||||
echo --- Analytics Integration Tests
|
||||
checks-reporter-with-killswitch "Analytics Integration Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/analytics/config.ts \
|
||||
--bail \
|
||||
--debug
|
||||
|
|
@ -18,8 +18,6 @@ export TEST_ES_DISABLE_STARTUP=true
|
|||
|
||||
sleep 120
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
|
||||
|
||||
for i in "${journeys[@]}"; do
|
||||
|
|
@ -31,8 +29,8 @@ for i in "${journeys[@]}"; do
|
|||
|
||||
checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \
|
||||
node scripts/functional_tests \
|
||||
--config test/performance/config.playwright.ts \
|
||||
--include "test/performance/tests/playwright/${i}.ts" \
|
||||
--config x-pack/test/performance/config.playwright.ts \
|
||||
--include "x-pack/test/performance/tests/playwright/${i}.ts" \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--debug \
|
||||
--bail
|
||||
|
|
@ -42,8 +40,8 @@ for i in "${journeys[@]}"; do
|
|||
|
||||
checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: TEST)" \
|
||||
node scripts/functional_tests \
|
||||
--config test/performance/config.playwright.ts \
|
||||
--include "test/performance/tests/playwright/${i}.ts" \
|
||||
--config x-pack/test/performance/config.playwright.ts \
|
||||
--include "x-pack/test/performance/tests/playwright/${i}.ts" \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--debug \
|
||||
--bail
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@ export JOB=kibana-security-solution-chrome
|
|||
|
||||
echo "--- Response Ops Cypress Tests on Security Solution"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "Response Ops Cypress Tests on Security Solution" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/security_solution_cypress/response_ops_cli_config.ts
|
||||
--config x-pack/test/security_solution_cypress/response_ops_cli_config.ts
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@ export JOB=kibana-security-solution-chrome
|
|||
|
||||
echo "--- Response Ops Cases Cypress Tests on Security Solution"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "Response Ops Cases Cypress Tests on Security Solution" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/security_solution_cypress/cases_cli_config.ts
|
||||
--config x-pack/test/security_solution_cypress/cases_cli_config.ts
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@ export JOB=kibana-security-solution-chrome
|
|||
|
||||
echo "--- Security Solution tests (Chrome)"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome)" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/security_solution_cypress/cli_config.ts
|
||||
--config x-pack/test/security_solution_cypress/cli_config.ts
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
echo --- Default Accessibility Tests
|
||||
|
||||
checks-reporter-with-killswitch "X-Pack accessibility tests" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/accessibility/config.ts
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
|
||||
export JOB=kibana-default-ciGroup${CI_GROUP}
|
||||
|
||||
echo "--- Default CI Group $CI_GROUP"
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
|
||||
node scripts/functional_tests \
|
||||
--bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--include-tag "ciGroup$CI_GROUP"
|
||||
|
||||
cd "$KIBANA_DIR"
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
echo --- Default Firefox Smoke Tests
|
||||
|
||||
checks-reporter-with-killswitch "X-Pack firefox smoke test" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--include-tag "includeFirefox" \
|
||||
--config test/functional/config.firefox.js \
|
||||
--config test/functional_embedded/config.firefox.ts
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
.buildkite/scripts/download_build_artifacts.sh
|
||||
|
||||
cd "$XPACK_DIR"
|
||||
|
||||
echo --- Capture Kibana Saved Objects field count metrics
|
||||
checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
|
||||
node scripts/functional_tests \
|
||||
--debug --bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config test/saved_objects_field_count/config.ts;
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
|
||||
is_test_execution_step
|
||||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
|
||||
echo --- API Integration Tests
|
||||
checks-reporter-with-killswitch "API Integration Tests" \
|
||||
node scripts/functional_tests \
|
||||
--config test/api_integration/config.js \
|
||||
--bail \
|
||||
--debug
|
||||
88
.buildkite/scripts/steps/test/ftr_configs.sh
Executable file
88
.buildkite/scripts/steps/test/ftr_configs.sh
Executable file
|
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/functional/common.sh
|
||||
source .buildkite/scripts/steps/test/test_group_env.sh
|
||||
|
||||
export JOB_NUM=$BUILDKITE_PARALLEL_JOB
|
||||
export JOB=ftr-configs-${JOB_NUM}
|
||||
|
||||
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}"
|
||||
|
||||
# a FTR failure will result in the script returning an exit code of 10
|
||||
exitCode=0
|
||||
|
||||
configs="${FTR_CONFIG:-}"
|
||||
|
||||
# The first retry should only run the configs that failed in the previous attempt
|
||||
# Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
|
||||
if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
|
||||
configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
|
||||
if [[ "$configs" ]]; then
|
||||
echo "--- Retrying only failed configs"
|
||||
echo "$configs"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$configs" == "" ]]; then
|
||||
echo "--- downloading ftr test run order"
|
||||
buildkite-agent artifact download ftr_run_order.json .
|
||||
configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
|
||||
fi
|
||||
|
||||
failedConfigs=""
|
||||
results=()
|
||||
|
||||
while read -r config; do
|
||||
if [[ ! "$config" ]]; then
|
||||
continue;
|
||||
fi
|
||||
|
||||
echo "--- $ node scripts/functional_tests --bail --config $config"
|
||||
start=$(date +%s)
|
||||
|
||||
# prevent non-zero exit code from breaking the loop
|
||||
set +e;
|
||||
node ./scripts/functional_tests \
|
||||
--bail \
|
||||
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
|
||||
--config="$config"
|
||||
lastCode=$?
|
||||
set -e;
|
||||
|
||||
timeSec=$(($(date +%s)-start))
|
||||
if [[ $timeSec -gt 60 ]]; then
|
||||
min=$((timeSec/60))
|
||||
sec=$((timeSec-(min*60)))
|
||||
duration="${min}m ${sec}s"
|
||||
else
|
||||
duration="${timeSec}s"
|
||||
fi
|
||||
|
||||
results+=("- $config
|
||||
duration: ${duration}
|
||||
result: ${lastCode}")
|
||||
|
||||
if [ $lastCode -ne 0 ]; then
|
||||
exitCode=10
|
||||
echo "FTR exited with code $lastCode"
|
||||
echo "^^^ +++"
|
||||
|
||||
if [[ "$failedConfigs" ]]; then
|
||||
failedConfigs="${failedConfigs}"$'\n'"$config"
|
||||
else
|
||||
failedConfigs="$config"
|
||||
fi
|
||||
fi
|
||||
done <<< "$configs"
|
||||
|
||||
if [[ "$failedConfigs" ]]; then
|
||||
buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
|
||||
fi
|
||||
|
||||
echo "--- FTR configs complete"
|
||||
printf "%s\n" "${results[@]}"
|
||||
echo ""
|
||||
|
||||
exit $exitCode
|
||||
|
|
@ -1,13 +1,14 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/steps/test/jest_env.sh
|
||||
source .buildkite/scripts/steps/test/test_group_env.sh
|
||||
|
||||
export JOB=$BUILDKITE_PARALLEL_JOB
|
||||
|
||||
# a jest failure will result in the script returning an exit code of 10
|
||||
exitCode=0
|
||||
results=()
|
||||
|
||||
if [[ "$1" == 'jest.config.js' ]]; then
|
||||
# run unit tests in parallel
|
||||
|
|
@ -20,14 +21,32 @@ else
|
|||
fi
|
||||
|
||||
export TEST_TYPE
|
||||
echo "--- downloading integration test run order"
|
||||
echo "--- downloading jest test run order"
|
||||
buildkite-agent artifact download jest_run_order.json .
|
||||
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
|
||||
|
||||
while read -r config; do
|
||||
echo "--- $ node scripts/jest --config $config"
|
||||
start=$(date +%s)
|
||||
|
||||
# prevent non-zero exit code from breaking the loop
|
||||
set +e;
|
||||
NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest --config="$config" "$parallelism" --coverage=false --passWithNoTests
|
||||
lastCode=$?
|
||||
set -e;
|
||||
|
||||
timeSec=$(($(date +%s)-start))
|
||||
if [[ $timeSec -gt 60 ]]; then
|
||||
min=$((timeSec/60))
|
||||
sec=$((timeSec-(min*60)))
|
||||
duration="${min}m ${sec}s"
|
||||
else
|
||||
duration="${timeSec}s"
|
||||
fi
|
||||
|
||||
results+=("- $config
|
||||
duration: ${duration}
|
||||
result: ${lastCode}")
|
||||
|
||||
if [ $lastCode -ne 0 ]; then
|
||||
exitCode=10
|
||||
|
|
@ -36,4 +55,8 @@ while read -r config; do
|
|||
fi
|
||||
done <<< "$configs"
|
||||
|
||||
echo "--- Jest configs complete"
|
||||
printf "%s\n" "${results[@]}"
|
||||
echo ""
|
||||
|
||||
exit $exitCode
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/steps/test/jest_env.sh
|
||||
|
||||
echo '--- Pick Jest Config Run Order'
|
||||
node "$(dirname "${0}")/pick_jest_config_run_order.js"
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/steps/test/test_group_env.sh
|
||||
|
||||
echo '--- Pick Test Group Run Order'
|
||||
node "$(dirname "${0}")/pick_test_group_run_order.js"
|
||||
|
|
@ -5,3 +5,4 @@ set -euo pipefail
|
|||
# keys used to associate test group data in ci-stats with Jest execution order
|
||||
export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
|
||||
export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"
|
||||
export TEST_GROUP_TYPE_FUNCTIONAL="Functional Tests"
|
||||
|
|
@ -6,7 +6,7 @@ We use functional tests to make sure the {kib} UI works as expected. It replaces
|
|||
[discrete]
|
||||
=== Running functional tests
|
||||
|
||||
The `FunctionalTestRunner` is very bare bones and gets most of its functionality from its config file, located at {blob}test/functional/config.js[test/functional/config.js] or {blob}x-pack/test/functional/config.js[x-pack/test/functional/config.js]. If you’re writing a plugin outside the {kib} repo, you will have your own config file.
|
||||
The `FunctionalTestRunner` (FTR) is very bare bones and gets most of its functionality from its config file. The {kib} repo contains many FTR config files which use slightly different configurations for the {kib} server or {es}, have different test files, and potentially other config differences. You can find a manifest of all the FTR config files in `.buildkite/ftr_configs.yml`. If you’re writing a plugin outside the {kib} repo, you will have your own config file.
|
||||
See <<external-plugin-functional-tests>> for more info.
|
||||
|
||||
There are three ways to run the tests depending on your goals:
|
||||
|
|
@ -96,7 +96,8 @@ node scripts/functional_test_runner --exclude-tag skipCloud
|
|||
|
||||
When run without any arguments the `FunctionalTestRunner` automatically loads the configuration in the standard location, but you can override that behavior with the `--config` flag. List configs with multiple --config arguments.
|
||||
|
||||
* `--config test/functional/config.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Chrome.
|
||||
* `--config test/functional/apps/app-name/config.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Chrome for a specific app. For example,
|
||||
`--config test/functional/apps/home/config.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Chrome for the home app.
|
||||
* `--config test/functional/config.firefox.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Firefox.
|
||||
* `--config test/api_integration/config.js` starts {es} and {kib} servers with the api integration tests configuration.
|
||||
* `--config test/accessibility/config.ts` starts {es} and {kib} servers with the WebDriver tests configured to run an accessibility audit using https://www.deque.com/axe/[axe].
|
||||
|
|
@ -416,7 +417,7 @@ export function SomethingUsefulProvider({ getService }) {
|
|||
-----------
|
||||
+
|
||||
* Re-export your provider from `services/index.js`
|
||||
* Import it into `src/functional/config.js` and add it to the services config:
|
||||
* Import it into `src/functional/config.base.js` and add it to the services config:
|
||||
+
|
||||
["source","js"]
|
||||
-----------
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ export default async function ({ readConfigFile }) {
|
|||
|
||||
// read the {kib} config file so that we can utilize some of
|
||||
// its services and PageObjects
|
||||
const kibanaConfig = await readConfigFile(resolve(REPO_ROOT, 'test/functional/config.js'));
|
||||
const kibanaConfig = await readConfigFile(resolve(REPO_ROOT, 'test/functional/config.base.js'));
|
||||
|
||||
return {
|
||||
// list paths to the files that contain your plugins tests
|
||||
|
|
|
|||
|
|
@ -37,19 +37,7 @@ async function withTimeout(
|
|||
ms: number,
|
||||
onTimeout: () => Promise<void>
|
||||
) {
|
||||
const TIMEOUT = Symbol('timeout');
|
||||
try {
|
||||
await Promise.race([
|
||||
attempt(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(TIMEOUT), ms)),
|
||||
]);
|
||||
} catch (error) {
|
||||
if (error === TIMEOUT) {
|
||||
await onTimeout();
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await Rx.lastValueFrom(Rx.race(Rx.defer(attempt), Rx.timer(ms).pipe(Rx.mergeMap(onTimeout))));
|
||||
}
|
||||
|
||||
export type Proc = ReturnType<typeof startProc>;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
*/
|
||||
|
||||
import * as Rx from 'rxjs';
|
||||
import { filter, first, catchError, map } from 'rxjs/operators';
|
||||
import exitHook from 'exit-hook';
|
||||
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
|
@ -93,29 +92,30 @@ export class ProcRunner {
|
|||
try {
|
||||
if (wait instanceof RegExp) {
|
||||
// wait for process to log matching line
|
||||
await Rx.race(
|
||||
proc.lines$.pipe(
|
||||
filter((line) => wait.test(line)),
|
||||
first(),
|
||||
catchError((err) => {
|
||||
if (err.name !== 'EmptyError') {
|
||||
throw createFailError(`[${name}] exited without matching pattern: ${wait}`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
})
|
||||
),
|
||||
waitTimeout === false
|
||||
? Rx.NEVER
|
||||
: Rx.timer(waitTimeout).pipe(
|
||||
map(() => {
|
||||
const sec = waitTimeout / SECOND;
|
||||
throw createFailError(
|
||||
`[${name}] failed to match pattern within ${sec} seconds [pattern=${wait}]`
|
||||
);
|
||||
})
|
||||
)
|
||||
).toPromise();
|
||||
await Rx.lastValueFrom(
|
||||
Rx.race(
|
||||
proc.lines$.pipe(
|
||||
Rx.filter((line) => wait.test(line)),
|
||||
Rx.take(1),
|
||||
Rx.defaultIfEmpty(undefined),
|
||||
Rx.map((line) => {
|
||||
if (line === undefined) {
|
||||
throw createFailError(`[${name}] exited without matching pattern: ${wait}`);
|
||||
}
|
||||
})
|
||||
),
|
||||
waitTimeout === false
|
||||
? Rx.NEVER
|
||||
: Rx.timer(waitTimeout).pipe(
|
||||
Rx.map(() => {
|
||||
const sec = waitTimeout / SECOND;
|
||||
throw createFailError(
|
||||
`[${name}] failed to match pattern within ${sec} seconds [pattern=${wait}]`
|
||||
);
|
||||
})
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (wait === true) {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import { Client, HttpConnection } from '@elastic/elasticsearch';
|
|||
import { EsArchiver } from './es_archiver';
|
||||
|
||||
const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v);
|
||||
const defaultConfigPath = resolveConfigPath('test/functional/config.js');
|
||||
const defaultConfigPath = resolveConfigPath('test/functional/config.base.js');
|
||||
|
||||
export function runCli() {
|
||||
new RunWithCommands({
|
||||
|
|
|
|||
|
|
@ -260,6 +260,24 @@ exports.Cluster = class Cluster {
|
|||
await this._outcome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops ES process, it it's running, without waiting for it to shutdown gracefully
|
||||
*/
|
||||
async kill() {
|
||||
if (this._stopCalled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._stopCalled;
|
||||
|
||||
if (!this._process || !this._outcome) {
|
||||
throw new Error('ES has not been started');
|
||||
}
|
||||
|
||||
await treeKillAsync(this._process.pid, 'SIGKILL');
|
||||
await this._outcome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common logic from this.start() and this.run()
|
||||
*
|
||||
|
|
|
|||
21
packages/kbn-pm/dist/index.js
vendored
21
packages/kbn-pm/dist/index.js
vendored
|
|
@ -2270,8 +2270,6 @@ exports.observeLines = observeLines;
|
|||
|
||||
var Rx = _interopRequireWildcard(__webpack_require__("../../node_modules/rxjs/dist/esm5/index.js"));
|
||||
|
||||
var _operators = __webpack_require__("../../node_modules/rxjs/dist/esm5/operators/index.js");
|
||||
|
||||
var _observe_readable = __webpack_require__("../../node_modules/@kbn/stdio-dev-helpers/target_node/observe_readable.js");
|
||||
|
||||
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
||||
|
|
@ -2297,8 +2295,8 @@ const SEP = /\r?\n/;
|
|||
* @return {Rx.Observable}
|
||||
*/
|
||||
function observeLines(readable) {
|
||||
const done$ = (0, _observe_readable.observeReadable)(readable).pipe((0, _operators.share)());
|
||||
const scan$ = Rx.fromEvent(readable, 'data').pipe((0, _operators.scan)(({
|
||||
const done$ = (0, _observe_readable.observeReadable)(readable).pipe(Rx.share());
|
||||
const scan$ = Rx.fromEvent(readable, 'data').pipe(Rx.scan(({
|
||||
buffer
|
||||
}, chunk) => {
|
||||
buffer += chunk;
|
||||
|
|
@ -2322,16 +2320,15 @@ function observeLines(readable) {
|
|||
}, {
|
||||
buffer: ''
|
||||
}), // stop if done completes or errors
|
||||
(0, _operators.takeUntil)(done$.pipe((0, _operators.materialize)())), (0, _operators.share)());
|
||||
Rx.takeUntil(done$.pipe(Rx.materialize())), Rx.share());
|
||||
return Rx.merge( // use done$ to provide completion/errors
|
||||
done$, // merge in the "lines" from each step
|
||||
scan$.pipe((0, _operators.mergeMap)(({
|
||||
scan$.pipe(Rx.mergeMap(({
|
||||
lines
|
||||
}) => lines || [])), // inject the "unsplit" data at the end
|
||||
scan$.pipe((0, _operators.last)(), (0, _operators.mergeMap)(({
|
||||
scan$.pipe(Rx.takeLast(1), Rx.mergeMap(({
|
||||
buffer
|
||||
}) => buffer ? [buffer] : []), // if there were no lines, last() will error, so catch and complete
|
||||
(0, _operators.catchError)(() => Rx.empty())));
|
||||
}) => buffer ? [buffer] : [])));
|
||||
}
|
||||
|
||||
/***/ }),
|
||||
|
|
@ -2349,8 +2346,6 @@ exports.observeReadable = observeReadable;
|
|||
|
||||
var Rx = _interopRequireWildcard(__webpack_require__("../../node_modules/rxjs/dist/esm5/index.js"));
|
||||
|
||||
var _operators = __webpack_require__("../../node_modules/rxjs/dist/esm5/operators/index.js");
|
||||
|
||||
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
||||
|
||||
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
||||
|
|
@ -2369,7 +2364,9 @@ function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj &&
|
|||
* - fails on the first "error" event
|
||||
*/
|
||||
function observeReadable(readable) {
|
||||
return Rx.race(Rx.fromEvent(readable, 'end').pipe((0, _operators.first)(), (0, _operators.ignoreElements)()), Rx.fromEvent(readable, 'error').pipe((0, _operators.first)(), (0, _operators.mergeMap)(err => Rx.throwError(err))));
|
||||
return Rx.race(Rx.fromEvent(readable, 'end').pipe(Rx.first(), Rx.ignoreElements()), Rx.fromEvent(readable, 'error').pipe(Rx.first(), Rx.map(err => {
|
||||
throw err;
|
||||
})));
|
||||
}
|
||||
|
||||
/***/ }),
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
import { Readable } from 'stream';
|
||||
|
||||
import * as Rx from 'rxjs';
|
||||
import { scan, takeUntil, share, materialize, mergeMap, last, catchError } from 'rxjs/operators';
|
||||
|
||||
const SEP = /\r?\n/;
|
||||
|
||||
|
|
@ -25,13 +24,13 @@ import { observeReadable } from './observe_readable';
|
|||
* @return {Rx.Observable}
|
||||
*/
|
||||
export function observeLines(readable: Readable): Rx.Observable<string> {
|
||||
const done$ = observeReadable(readable).pipe(share());
|
||||
const done$ = observeReadable(readable).pipe(Rx.share());
|
||||
|
||||
const scan$: Rx.Observable<{ buffer: string; lines?: string[] }> = Rx.fromEvent(
|
||||
readable,
|
||||
'data'
|
||||
).pipe(
|
||||
scan(
|
||||
Rx.scan(
|
||||
({ buffer }, chunk) => {
|
||||
buffer += chunk;
|
||||
|
||||
|
|
@ -53,9 +52,9 @@ export function observeLines(readable: Readable): Rx.Observable<string> {
|
|||
),
|
||||
|
||||
// stop if done completes or errors
|
||||
takeUntil(done$.pipe(materialize())),
|
||||
Rx.takeUntil(done$.pipe(Rx.materialize())),
|
||||
|
||||
share()
|
||||
Rx.share()
|
||||
);
|
||||
|
||||
return Rx.merge(
|
||||
|
|
@ -63,14 +62,12 @@ export function observeLines(readable: Readable): Rx.Observable<string> {
|
|||
done$,
|
||||
|
||||
// merge in the "lines" from each step
|
||||
scan$.pipe(mergeMap(({ lines }) => lines || [])),
|
||||
scan$.pipe(Rx.mergeMap(({ lines }) => lines || [])),
|
||||
|
||||
// inject the "unsplit" data at the end
|
||||
scan$.pipe(
|
||||
last(),
|
||||
mergeMap(({ buffer }) => (buffer ? [buffer] : [])),
|
||||
// if there were no lines, last() will error, so catch and complete
|
||||
catchError(() => Rx.empty())
|
||||
Rx.takeLast(1),
|
||||
Rx.mergeMap(({ buffer }) => (buffer ? [buffer] : []))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
import { Readable } from 'stream';
|
||||
|
||||
import * as Rx from 'rxjs';
|
||||
import { first, ignoreElements, mergeMap } from 'rxjs/operators';
|
||||
|
||||
/**
|
||||
* Produces an Observable from a ReadableSteam that:
|
||||
|
|
@ -18,11 +17,12 @@ import { first, ignoreElements, mergeMap } from 'rxjs/operators';
|
|||
*/
|
||||
export function observeReadable(readable: Readable): Rx.Observable<never> {
|
||||
return Rx.race(
|
||||
Rx.fromEvent(readable, 'end').pipe(first(), ignoreElements()),
|
||||
|
||||
Rx.fromEvent(readable, 'end').pipe(Rx.first(), Rx.ignoreElements()),
|
||||
Rx.fromEvent(readable, 'error').pipe(
|
||||
first(),
|
||||
mergeMap((err) => Rx.throwError(err))
|
||||
Rx.first(),
|
||||
Rx.map((err) => {
|
||||
throw err;
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ RUNTIME_DEPS = [
|
|||
"@npm//jest-snapshot",
|
||||
"@npm//jest-styled-components",
|
||||
"@npm//joi",
|
||||
"@npm//js-yaml",
|
||||
"@npm//mustache",
|
||||
"@npm//normalize-path",
|
||||
"@npm//parse-link-header",
|
||||
|
|
@ -108,6 +109,7 @@ TYPES_DEPS = [
|
|||
"@npm//@types/he",
|
||||
"@npm//@types/history",
|
||||
"@npm//@types/jest",
|
||||
"@npm//@types/js-yaml",
|
||||
"@npm//@types/joi",
|
||||
"@npm//@types/lodash",
|
||||
"@npm//@types/mustache",
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@ Functional testing methods exist in the `src/functional_tests` directory. They d
|
|||
#### runTests(configPaths: Array<string>)
|
||||
For each config file specified in configPaths, starts Elasticsearch and Kibana once, runs tests specified in that config file, and shuts down Elasticsearch and Kibana once completed. (Repeats for every config file.)
|
||||
|
||||
`configPaths`: array of strings, each an absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
|
||||
`configPaths`: array of strings, each an absolute path to a config file that looks like [this](../../test/functional/config.base.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
|
||||
|
||||
Internally the method that starts Elasticsearch comes from [kbn-es](../../packages/kbn-es).
|
||||
|
||||
#### startServers(configPath: string)
|
||||
Starts Elasticsearch and Kibana servers given a specified config.
|
||||
|
||||
`configPath`: absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
|
||||
`configPath`: absolute path to a config file that looks like [this](../../test/functional/config.base.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
|
||||
|
||||
Allows users to start another process to run just the tests while keeping the servers running with this method. Start servers _and_ run tests using the same config file ([see how](../../scripts/README.md)).
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ interface Node {
|
|||
) => Promise<{ insallPath: string }>;
|
||||
start: (installPath: string, opts: Record<string, unknown>) => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
kill: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface ICluster {
|
||||
|
|
@ -268,20 +269,26 @@ export function createTestEsCluster<
|
|||
}
|
||||
|
||||
async stop() {
|
||||
const nodeStopPromises = [];
|
||||
for (let i = 0; i < this.nodes.length; i++) {
|
||||
nodeStopPromises.push(async () => {
|
||||
await Promise.all(
|
||||
this.nodes.map(async (node, i) => {
|
||||
log.info(`[es] stopping node ${nodes[i].name}`);
|
||||
return await this.nodes[i].stop();
|
||||
});
|
||||
}
|
||||
await Promise.all(nodeStopPromises.map(async (stop) => await stop()));
|
||||
await node.stop();
|
||||
})
|
||||
);
|
||||
|
||||
log.info('[es] stopped');
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
await this.stop();
|
||||
log.info('[es] killing', this.nodes.length === 1 ? 'node' : `${this.nodes.length} nodes`);
|
||||
await Promise.all(
|
||||
this.nodes.map(async (node, i) => {
|
||||
log.info(`[es] stopping node ${nodes[i].name}`);
|
||||
// we are deleting this install, stop ES more aggressively
|
||||
await node.kill();
|
||||
})
|
||||
);
|
||||
|
||||
await del(config.installPath, { force: true });
|
||||
log.info('[es] cleanup complete');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,11 +98,7 @@ export function runFtrCli() {
|
|||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await functionalTestRunner.close();
|
||||
} finally {
|
||||
process.exit();
|
||||
}
|
||||
process.exit();
|
||||
};
|
||||
|
||||
process.on('unhandledRejection', (err) =>
|
||||
|
|
|
|||
|
|
@ -14,10 +14,9 @@ import { REPO_ROOT } from '@kbn/utils';
|
|||
import { Suite, Test } from './fake_mocha_types';
|
||||
import {
|
||||
Lifecycle,
|
||||
LifecyclePhase,
|
||||
TestMetadata,
|
||||
readConfigFile,
|
||||
ProviderCollection,
|
||||
Providers,
|
||||
readProviderSpec,
|
||||
setupMocha,
|
||||
runTests,
|
||||
|
|
@ -29,10 +28,6 @@ import {
|
|||
import { createEsClientForFtrConfig } from '../es';
|
||||
|
||||
export class FunctionalTestRunner {
|
||||
public readonly lifecycle = new Lifecycle();
|
||||
public readonly testMetadata = new TestMetadata(this.lifecycle);
|
||||
private closed = false;
|
||||
|
||||
private readonly esVersion: EsVersion;
|
||||
constructor(
|
||||
private readonly log: ToolingLog,
|
||||
|
|
@ -40,12 +35,6 @@ export class FunctionalTestRunner {
|
|||
private readonly configOverrides: any,
|
||||
esVersion?: string | EsVersion
|
||||
) {
|
||||
for (const [key, value] of Object.entries(this.lifecycle)) {
|
||||
if (value instanceof LifecyclePhase) {
|
||||
value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
|
||||
value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
|
||||
}
|
||||
}
|
||||
this.esVersion =
|
||||
esVersion === undefined
|
||||
? EsVersion.getDefault()
|
||||
|
|
@ -55,19 +44,28 @@ export class FunctionalTestRunner {
|
|||
}
|
||||
|
||||
async run() {
|
||||
return await this._run(async (config, coreProviders) => {
|
||||
SuiteTracker.startTracking(this.lifecycle, this.configFile);
|
||||
const testStats = await this.getTestStats();
|
||||
|
||||
const providers = new ProviderCollection(this.log, [
|
||||
...coreProviders,
|
||||
...readProviderSpec('Service', config.get('services')),
|
||||
...readProviderSpec('PageObject', config.get('pageObjects')),
|
||||
]);
|
||||
return await this.runHarness(async (config, lifecycle, coreProviders) => {
|
||||
SuiteTracker.startTracking(lifecycle, this.configFile);
|
||||
|
||||
if (providers.hasService('es')) {
|
||||
await this.validateEsVersion(config);
|
||||
const realServices =
|
||||
!testStats || (testStats.testCount > 0 && testStats.nonSkippedTestCount > 0);
|
||||
|
||||
const providers = realServices
|
||||
? new ProviderCollection(this.log, [
|
||||
...coreProviders,
|
||||
...readProviderSpec('Service', config.get('services')),
|
||||
...readProviderSpec('PageObject', config.get('pageObjects')),
|
||||
])
|
||||
: this.getStubProviderCollection(config, coreProviders);
|
||||
|
||||
if (realServices) {
|
||||
if (providers.hasService('es')) {
|
||||
await this.validateEsVersion(config);
|
||||
}
|
||||
await providers.loadAll();
|
||||
}
|
||||
await providers.loadAll();
|
||||
|
||||
const customTestRunner = config.get('testRunner');
|
||||
if (customTestRunner) {
|
||||
|
|
@ -90,7 +88,7 @@ export class FunctionalTestRunner {
|
|||
}
|
||||
|
||||
const mocha = await setupMocha(
|
||||
this.lifecycle,
|
||||
lifecycle,
|
||||
this.log,
|
||||
config,
|
||||
providers,
|
||||
|
|
@ -108,10 +106,10 @@ export class FunctionalTestRunner {
|
|||
return this.simulateMochaDryRun(mocha);
|
||||
}
|
||||
|
||||
await this.lifecycle.beforeTests.trigger(mocha.suite);
|
||||
await lifecycle.beforeTests.trigger(mocha.suite);
|
||||
this.log.info('Starting tests');
|
||||
|
||||
return await runTests(this.lifecycle, mocha);
|
||||
return await runTests(lifecycle, mocha);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -143,60 +141,73 @@ export class FunctionalTestRunner {
|
|||
}
|
||||
|
||||
async getTestStats() {
|
||||
return await this._run(async (config, coreProviders) => {
|
||||
return await this.runHarness(async (config, lifecycle, coreProviders) => {
|
||||
if (config.get('testRunner')) {
|
||||
throw new Error('Unable to get test stats for config that uses a custom test runner');
|
||||
return;
|
||||
}
|
||||
|
||||
// replace the function of custom service providers so that they return
|
||||
// promise-like objects which never resolve, essentially disabling them
|
||||
// allowing us to load the test files and populate the mocha suites
|
||||
const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
|
||||
readProviderSpec(type, providers).map((p) => ({
|
||||
...p,
|
||||
fn: skip.includes(p.name)
|
||||
? (ctx: any) => {
|
||||
const result = ProviderCollection.callProviderFn(p.fn, ctx);
|
||||
const providers = this.getStubProviderCollection(config, coreProviders);
|
||||
const mocha = await setupMocha(lifecycle, this.log, config, providers, this.esVersion);
|
||||
|
||||
if ('then' in result) {
|
||||
throw new Error(
|
||||
`Provider [${p.name}] returns a promise so it can't loaded during test analysis`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
: () => ({
|
||||
then: () => {},
|
||||
}),
|
||||
}));
|
||||
|
||||
const providers = new ProviderCollection(this.log, [
|
||||
...coreProviders,
|
||||
...readStubbedProviderSpec(
|
||||
'Service',
|
||||
config.get('services'),
|
||||
config.get('servicesRequiredForTestAnalysis')
|
||||
),
|
||||
...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
|
||||
]);
|
||||
|
||||
const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
|
||||
|
||||
const countTests = (suite: Suite): number =>
|
||||
suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length);
|
||||
const queue = new Set([mocha.suite]);
|
||||
const allTests: Test[] = [];
|
||||
for (const suite of queue) {
|
||||
for (const test of suite.tests) {
|
||||
allTests.push(test);
|
||||
}
|
||||
for (const childSuite of suite.suites) {
|
||||
queue.add(childSuite);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
testCount: countTests(mocha.suite),
|
||||
testCount: allTests.length,
|
||||
nonSkippedTestCount: allTests.filter((t) => !t.pending).length,
|
||||
testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async _run<T = any>(
|
||||
handler: (config: Config, coreProvider: ReturnType<typeof readProviderSpec>) => Promise<T>
|
||||
private getStubProviderCollection(config: Config, coreProviders: Providers) {
|
||||
// when we want to load the tests but not actually run anything we can
|
||||
// use stubbed providers which allow mocha to do it's thing without taking
|
||||
// too much time
|
||||
const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
|
||||
readProviderSpec(type, providers).map((p) => ({
|
||||
...p,
|
||||
fn: skip.includes(p.name)
|
||||
? (ctx: any) => {
|
||||
const result = ProviderCollection.callProviderFn(p.fn, ctx);
|
||||
|
||||
if ('then' in result) {
|
||||
throw new Error(
|
||||
`Provider [${p.name}] returns a promise so it can't loaded during test analysis`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
: () => ({
|
||||
then: () => {},
|
||||
}),
|
||||
}));
|
||||
|
||||
return new ProviderCollection(this.log, [
|
||||
...coreProviders,
|
||||
...readStubbedProviderSpec(
|
||||
'Service',
|
||||
config.get('services'),
|
||||
config.get('servicesRequiredForTestAnalysis')
|
||||
),
|
||||
...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
|
||||
]);
|
||||
}
|
||||
|
||||
private async runHarness<T = any>(
|
||||
handler: (config: Config, lifecycle: Lifecycle, coreProviders: Providers) => Promise<T>
|
||||
): Promise<T> {
|
||||
let runErrorOccurred = false;
|
||||
const lifecycle = new Lifecycle(this.log);
|
||||
|
||||
try {
|
||||
const config = await readConfigFile(
|
||||
|
|
@ -205,7 +216,7 @@ export class FunctionalTestRunner {
|
|||
this.configFile,
|
||||
this.configOverrides
|
||||
);
|
||||
this.log.info('Config loaded');
|
||||
this.log.debug('Config loaded');
|
||||
|
||||
if (
|
||||
(!config.get('testFiles') || config.get('testFiles').length === 0) &&
|
||||
|
|
@ -217,26 +228,25 @@ export class FunctionalTestRunner {
|
|||
const dockerServers = new DockerServersService(
|
||||
config.get('dockerServers'),
|
||||
this.log,
|
||||
this.lifecycle
|
||||
lifecycle
|
||||
);
|
||||
|
||||
// base level services that functional_test_runner exposes
|
||||
const coreProviders = readProviderSpec('Service', {
|
||||
lifecycle: () => this.lifecycle,
|
||||
lifecycle: () => lifecycle,
|
||||
log: () => this.log,
|
||||
testMetadata: () => this.testMetadata,
|
||||
config: () => config,
|
||||
dockerServers: () => dockerServers,
|
||||
esVersion: () => this.esVersion,
|
||||
});
|
||||
|
||||
return await handler(config, coreProviders);
|
||||
return await handler(config, lifecycle, coreProviders);
|
||||
} catch (runError) {
|
||||
runErrorOccurred = true;
|
||||
throw runError;
|
||||
} finally {
|
||||
try {
|
||||
await this.close();
|
||||
await lifecycle.cleanup.trigger();
|
||||
} catch (closeError) {
|
||||
if (runErrorOccurred) {
|
||||
this.log.error('failed to close functional_test_runner');
|
||||
|
|
@ -249,13 +259,6 @@ export class FunctionalTestRunner {
|
|||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.closed) return;
|
||||
|
||||
this.closed = true;
|
||||
await this.lifecycle.cleanup.trigger();
|
||||
}
|
||||
|
||||
simulateMochaDryRun(mocha: any) {
|
||||
interface TestEntry {
|
||||
file: string;
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ export {
|
|||
Lifecycle,
|
||||
LifecyclePhase,
|
||||
} from './lib';
|
||||
export type { ScreenshotRecord } from './lib';
|
||||
export { runFtrCli } from './cli';
|
||||
export * from './lib/docker_servers';
|
||||
export * from './public_types';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import Fs from 'fs';
|
||||
|
||||
import { REPO_ROOT } from '@kbn/utils';
|
||||
import JsYaml from 'js-yaml';
|
||||
|
||||
export const FTR_CONFIGS_MANIFEST_REL = '.buildkite/ftr_configs.yml';
|
||||
|
||||
const ftrConfigsManifest = JsYaml.safeLoad(
|
||||
Fs.readFileSync(Path.resolve(REPO_ROOT, FTR_CONFIGS_MANIFEST_REL), 'utf8')
|
||||
);
|
||||
|
||||
export const FTR_CONFIGS_MANIFEST_PATHS = (Object.values(ftrConfigsManifest) as string[][])
|
||||
.flat()
|
||||
.map((rel) => Path.resolve(REPO_ROOT, rel));
|
||||
|
|
@ -14,36 +14,35 @@ import { EsVersion } from '../es_version';
|
|||
const log = new ToolingLog();
|
||||
const esVersion = new EsVersion('8.0.0');
|
||||
|
||||
const CONFIG_PATH_1 = require.resolve('./__fixtures__/config.1.js');
|
||||
const CONFIG_PATH_2 = require.resolve('./__fixtures__/config.2.js');
|
||||
const CONFIG_PATH_INVALID = require.resolve('./__fixtures__/config.invalid.js');
|
||||
|
||||
describe('readConfigFile()', () => {
|
||||
it('reads config from a file, returns an instance of Config class', async () => {
|
||||
const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.1'));
|
||||
const config = await readConfigFile(log, esVersion, CONFIG_PATH_1);
|
||||
expect(config instanceof Config).toBeTruthy();
|
||||
expect(config.get('testFiles')).toEqual(['config.1']);
|
||||
});
|
||||
|
||||
it('merges setting overrides into log', async () => {
|
||||
const config = await readConfigFile(
|
||||
log,
|
||||
esVersion,
|
||||
require.resolve('./__fixtures__/config.1'),
|
||||
{
|
||||
screenshots: {
|
||||
directory: 'foo.bar',
|
||||
},
|
||||
}
|
||||
);
|
||||
const config = await readConfigFile(log, esVersion, CONFIG_PATH_1, {
|
||||
screenshots: {
|
||||
directory: 'foo.bar',
|
||||
},
|
||||
});
|
||||
|
||||
expect(config.get('screenshots.directory')).toBe('foo.bar');
|
||||
});
|
||||
|
||||
it('supports loading config files from within config files', async () => {
|
||||
const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.2'));
|
||||
const config = await readConfigFile(log, esVersion, CONFIG_PATH_2);
|
||||
expect(config.get('testFiles')).toEqual(['config.1', 'config.2']);
|
||||
});
|
||||
|
||||
it('throws if settings are invalid', async () => {
|
||||
try {
|
||||
await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.invalid'));
|
||||
await readConfigFile(log, esVersion, CONFIG_PATH_INVALID);
|
||||
throw new Error('expected readConfigFile() to fail');
|
||||
} catch (err) {
|
||||
expect(err.message).toMatch(/"foo"/);
|
||||
|
|
@ -6,25 +6,41 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { defaultsDeep } from 'lodash';
|
||||
import { createFlagError } from '@kbn/dev-utils';
|
||||
|
||||
import { Config } from './config';
|
||||
import { EsVersion } from '../es_version';
|
||||
import { FTR_CONFIGS_MANIFEST_REL, FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest';
|
||||
|
||||
const cache = new WeakMap();
|
||||
|
||||
async function getSettingsFromFile(
|
||||
log: ToolingLog,
|
||||
esVersion: EsVersion,
|
||||
path: string,
|
||||
settingOverrides: any
|
||||
options: {
|
||||
path: string;
|
||||
settingOverrides: any;
|
||||
primary: boolean;
|
||||
}
|
||||
) {
|
||||
const configModule = require(path); // eslint-disable-line @typescript-eslint/no-var-requires
|
||||
if (
|
||||
options.primary &&
|
||||
!FTR_CONFIGS_MANIFEST_PATHS.includes(options.path) &&
|
||||
!options.path.includes(`${Path.sep}__fixtures__${Path.sep}`)
|
||||
) {
|
||||
throw createFlagError(
|
||||
`Refusing to load FTR Config which is not listed in [${FTR_CONFIGS_MANIFEST_REL}]. All FTR Config files must be listed there, use the "enabled" key if the FTR Config should be run on automatically on PR CI, or the "disabled" key if it is run manually or by a special job.`
|
||||
);
|
||||
}
|
||||
|
||||
const configModule = require(options.path); // eslint-disable-line @typescript-eslint/no-var-requires
|
||||
const configProvider = configModule.__esModule ? configModule.default : configModule;
|
||||
|
||||
if (!cache.has(configProvider)) {
|
||||
log.debug('Loading config file from %j', path);
|
||||
log.debug('Loading config file from %j', options.path);
|
||||
cache.set(
|
||||
configProvider,
|
||||
configProvider({
|
||||
|
|
@ -32,7 +48,11 @@ async function getSettingsFromFile(
|
|||
esVersion,
|
||||
async readConfigFile(p: string, o: any) {
|
||||
return new Config({
|
||||
settings: await getSettingsFromFile(log, esVersion, p, o),
|
||||
settings: await getSettingsFromFile(log, esVersion, {
|
||||
path: p,
|
||||
settingOverrides: o,
|
||||
primary: false,
|
||||
}),
|
||||
primary: false,
|
||||
path: p,
|
||||
});
|
||||
|
|
@ -43,7 +63,7 @@ async function getSettingsFromFile(
|
|||
|
||||
const settingsWithDefaults: any = defaultsDeep(
|
||||
{},
|
||||
settingOverrides,
|
||||
options.settingOverrides,
|
||||
await cache.get(configProvider)!
|
||||
);
|
||||
|
||||
|
|
@ -57,7 +77,11 @@ export async function readConfigFile(
|
|||
settingOverrides: any = {}
|
||||
) {
|
||||
return new Config({
|
||||
settings: await getSettingsFromFile(log, esVersion, path, settingOverrides),
|
||||
settings: await getSettingsFromFile(log, esVersion, {
|
||||
path,
|
||||
settingOverrides,
|
||||
primary: true,
|
||||
}),
|
||||
primary: true,
|
||||
path,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -226,6 +226,11 @@ export const schema = Joi.object()
|
|||
wait: Joi.object()
|
||||
.regex()
|
||||
.default(/Kibana is now available/),
|
||||
|
||||
/**
|
||||
* Does this test config only work when run against source?
|
||||
*/
|
||||
alwaysUseSource: Joi.boolean().default(false),
|
||||
})
|
||||
.default(),
|
||||
env: Joi.object().unknown().default(),
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ export { readConfigFile, Config } from './config';
|
|||
export * from './providers';
|
||||
// @internal
|
||||
export { runTests, setupMocha } from './mocha';
|
||||
export * from './test_metadata';
|
||||
export * from './docker_servers';
|
||||
export { SuiteTracker } from './suite_tracker';
|
||||
|
||||
|
|
|
|||
|
|
@ -6,29 +6,51 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import * as Rx from 'rxjs';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
import { LifecyclePhase } from './lifecycle_phase';
|
||||
|
||||
import { Suite, Test } from '../fake_mocha_types';
|
||||
|
||||
export class Lifecycle {
|
||||
/** root subscription to cleanup lifecycle phases when lifecycle completes */
|
||||
private readonly sub = new Rx.Subscription();
|
||||
|
||||
/** lifecycle phase that will run handlers once before tests execute */
|
||||
public readonly beforeTests = new LifecyclePhase<[Suite]>({
|
||||
public readonly beforeTests = new LifecyclePhase<[Suite]>(this.sub, {
|
||||
singular: true,
|
||||
});
|
||||
/** lifecycle phase that runs handlers before each runnable (test and hooks) */
|
||||
public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
|
||||
public readonly beforeEachRunnable = new LifecyclePhase<[Test]>(this.sub);
|
||||
/** lifecycle phase that runs handlers before each suite */
|
||||
public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
|
||||
public readonly beforeTestSuite = new LifecyclePhase<[Suite]>(this.sub);
|
||||
/** lifecycle phase that runs handlers before each test */
|
||||
public readonly beforeEachTest = new LifecyclePhase<[Test]>();
|
||||
public readonly beforeEachTest = new LifecyclePhase<[Test]>(this.sub);
|
||||
/** lifecycle phase that runs handlers after each suite */
|
||||
public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
|
||||
public readonly afterTestSuite = new LifecyclePhase<[Suite]>(this.sub);
|
||||
/** lifecycle phase that runs handlers after a test fails */
|
||||
public readonly testFailure = new LifecyclePhase<[Error, Test]>();
|
||||
public readonly testFailure = new LifecyclePhase<[Error, Test]>(this.sub);
|
||||
/** lifecycle phase that runs handlers after a hook fails */
|
||||
public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
|
||||
public readonly testHookFailure = new LifecyclePhase<[Error, Test]>(this.sub);
|
||||
/** lifecycle phase that runs handlers at the very end of execution */
|
||||
public readonly cleanup = new LifecyclePhase<[]>({
|
||||
public readonly cleanup = new LifecyclePhase<[]>(this.sub, {
|
||||
singular: true,
|
||||
});
|
||||
|
||||
constructor(log: ToolingLog) {
|
||||
for (const [name, phase] of Object.entries(this)) {
|
||||
if (phase instanceof LifecyclePhase) {
|
||||
phase.before$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
|
||||
phase.after$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
|
||||
}
|
||||
}
|
||||
|
||||
// after the singular cleanup lifecycle phase completes unsubscribe from the root subscription
|
||||
this.cleanup.after$.pipe(Rx.materialize()).subscribe((n) => {
|
||||
if (n.kind === 'C') {
|
||||
this.sub.unsubscribe();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ describe('with randomness', () => {
|
|||
});
|
||||
|
||||
it('calls handlers in random order', async () => {
|
||||
const phase = new LifecyclePhase();
|
||||
const phase = new LifecyclePhase(new Rx.Subscription());
|
||||
const order: string[] = [];
|
||||
|
||||
phase.add(
|
||||
|
|
@ -69,7 +69,7 @@ describe('without randomness', () => {
|
|||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('calls all handlers and throws first error', async () => {
|
||||
const phase = new LifecyclePhase();
|
||||
const phase = new LifecyclePhase(new Rx.Subscription());
|
||||
const fn1 = jest.fn();
|
||||
phase.add(fn1);
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ describe('without randomness', () => {
|
|||
});
|
||||
|
||||
it('triggers before$ just before calling handler and after$ once it resolves', async () => {
|
||||
const phase = new LifecyclePhase();
|
||||
const phase = new LifecyclePhase(new Rx.Subscription());
|
||||
const order: string[] = [];
|
||||
|
||||
const beforeSub = jest.fn(() => order.push('before'));
|
||||
|
|
@ -116,7 +116,7 @@ describe('without randomness', () => {
|
|||
});
|
||||
|
||||
it('completes before$ and after$ if phase is singular', async () => {
|
||||
const phase = new LifecyclePhase({ singular: true });
|
||||
const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
|
||||
|
||||
const beforeNotifs: Array<Rx.Notification<unknown>> = [];
|
||||
phase.before$.pipe(materialize()).subscribe((n) => beforeNotifs.push(n));
|
||||
|
|
@ -160,7 +160,7 @@ describe('without randomness', () => {
|
|||
});
|
||||
|
||||
it('completes before$ subscribers after trigger of singular phase', async () => {
|
||||
const phase = new LifecyclePhase({ singular: true });
|
||||
const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
|
||||
await phase.trigger();
|
||||
|
||||
await expect(phase.before$.pipe(materialize(), toArray()).toPromise()).resolves
|
||||
|
|
@ -177,7 +177,7 @@ describe('without randomness', () => {
|
|||
});
|
||||
|
||||
it('replays after$ event subscribers after trigger of singular phase', async () => {
|
||||
const phase = new LifecyclePhase({ singular: true });
|
||||
const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
|
||||
await phase.trigger();
|
||||
|
||||
await expect(phase.after$.pipe(materialize(), toArray()).toPromise()).resolves
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export class LifecyclePhase<Args extends readonly any[]> {
|
|||
public readonly after$: Rx.Observable<void>;
|
||||
|
||||
constructor(
|
||||
sub: Rx.Subscription,
|
||||
private readonly options: {
|
||||
singular?: boolean;
|
||||
} = {}
|
||||
|
|
@ -35,6 +36,12 @@ export class LifecyclePhase<Args extends readonly any[]> {
|
|||
|
||||
this.afterSubj = this.options.singular ? new Rx.ReplaySubject<void>(1) : new Rx.Subject<void>();
|
||||
this.after$ = this.afterSubj.asObservable();
|
||||
|
||||
sub.add(() => {
|
||||
this.beforeSubj.complete();
|
||||
this.afterSubj.complete();
|
||||
this.handlers.length = 0;
|
||||
});
|
||||
}
|
||||
|
||||
public add(fn: (...args: Args) => Promise<void> | void) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,32 @@ import { createAssignmentProxy } from './assignment_proxy';
|
|||
import { wrapFunction } from './wrap_function';
|
||||
import { wrapRunnableArgs } from './wrap_runnable_args';
|
||||
|
||||
const allTestsSkippedCache = new WeakMap();
|
||||
function allTestsAreSkipped(suite) {
|
||||
// cache result for each suite so we don't have to traverse over and over
|
||||
const cache = allTestsSkippedCache.get(suite);
|
||||
if (cache) {
|
||||
return cache;
|
||||
}
|
||||
|
||||
// if this suite is skipped directly then all it's children are skipped
|
||||
if (suite.pending) {
|
||||
allTestsSkippedCache.set(suite, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
// if any of this suites own tests are not skipped, then we don't need to traverse to child suites
|
||||
if (suite.tests.some((t) => !t.pending)) {
|
||||
allTestsSkippedCache.set(suite, false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// otherwise traverse down through the child suites and return true only if all children are all skipped
|
||||
const childrenSkipped = suite.suites.every(allTestsAreSkipped);
|
||||
allTestsSkippedCache.set(suite, childrenSkipped);
|
||||
return childrenSkipped;
|
||||
}
|
||||
|
||||
export function decorateMochaUi(log, lifecycle, context, { rootTags }) {
|
||||
// incremented at the start of each suite, decremented after
|
||||
// so that in each non-suite call we can know if we are within
|
||||
|
|
@ -71,6 +97,12 @@ export function decorateMochaUi(log, lifecycle, context, { rootTags }) {
|
|||
|
||||
provider.call(this);
|
||||
|
||||
if (allTestsAreSkipped(this)) {
|
||||
// all the children in this suite are skipped, so make sure the suite is
|
||||
// marked as pending so that its hooks are not run
|
||||
this.pending = true;
|
||||
}
|
||||
|
||||
after('afterTestSuite.trigger', async () => {
|
||||
await lifecycle.afterTestSuite.trigger(this);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ function setup({ include, exclude, esVersion }) {
|
|||
info(...args) {
|
||||
history.push(`info: ${format(...args)}`);
|
||||
},
|
||||
debug(...args) {
|
||||
history.push(`debg: ${format(...args)}`);
|
||||
},
|
||||
},
|
||||
mocha,
|
||||
include,
|
||||
|
|
@ -221,7 +224,7 @@ it(`excludes tests which don't meet the esVersionRequirement`, async () => {
|
|||
|
||||
expect(history).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
"info: Only running suites which are compatible with ES version 9.0.0",
|
||||
"debg: Only running suites which are compatible with ES version 9.0.0",
|
||||
"suite: ",
|
||||
"suite: level 1",
|
||||
"suite: level 1 level 1a",
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ export function filterSuites({ log, mocha, include, exclude, esVersion }: Option
|
|||
|
||||
if (esVersion) {
|
||||
// traverse the test graph and exclude any tests which don't meet their esVersionRequirement
|
||||
log.info('Only running suites which are compatible with ES version', esVersion.toString());
|
||||
log.debug('Only running suites which are compatible with ES version', esVersion.toString());
|
||||
(function recurse(parentSuite: SuiteInternal) {
|
||||
const children = parentSuite.suites;
|
||||
parentSuite.suites = [];
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import {
|
|||
|
||||
import { Config } from '../../config';
|
||||
import { Runner } from '../../../fake_mocha_types';
|
||||
import { TestMetadata, ScreenshotRecord } from '../../test_metadata';
|
||||
import { Lifecycle } from '../../lifecycle';
|
||||
import { getSnapshotOfRunnableLogs } from '../../../../mocha';
|
||||
|
||||
|
|
@ -36,7 +35,6 @@ interface Runnable {
|
|||
file: string;
|
||||
title: string;
|
||||
parent: Suite;
|
||||
_screenshots?: ScreenshotRecord[];
|
||||
}
|
||||
|
||||
function getHookType(hook: Runnable): CiStatsTestType {
|
||||
|
|
@ -60,15 +58,18 @@ export function setupCiStatsFtrTestGroupReporter({
|
|||
config,
|
||||
lifecycle,
|
||||
runner,
|
||||
testMetadata,
|
||||
reporter,
|
||||
}: {
|
||||
config: Config;
|
||||
lifecycle: Lifecycle;
|
||||
runner: Runner;
|
||||
testMetadata: TestMetadata;
|
||||
reporter: CiStatsReporter;
|
||||
}) {
|
||||
const testGroupType = process.env.TEST_GROUP_TYPE_FUNCTIONAL;
|
||||
if (!testGroupType) {
|
||||
throw new Error('missing process.env.TEST_GROUP_TYPE_FUNCTIONAL');
|
||||
}
|
||||
|
||||
let startMs: number | undefined;
|
||||
runner.on('start', () => {
|
||||
startMs = Date.now();
|
||||
|
|
@ -78,7 +79,7 @@ export function setupCiStatsFtrTestGroupReporter({
|
|||
const group: CiStatsReportTestsOptions['group'] = {
|
||||
startTime: new Date(start).toJSON(),
|
||||
durationMs: 0,
|
||||
type: config.path.startsWith('x-pack') ? 'X-Pack Functional Tests' : 'Functional Tests',
|
||||
type: testGroupType,
|
||||
name: Path.relative(REPO_ROOT, config.path),
|
||||
result: 'skip',
|
||||
meta: {
|
||||
|
|
@ -106,10 +107,6 @@ export function setupCiStatsFtrTestGroupReporter({
|
|||
type,
|
||||
error: error?.stack,
|
||||
stdout: getSnapshotOfRunnableLogs(runnable),
|
||||
screenshots: testMetadata.getScreenshots(runnable).map((s) => ({
|
||||
base64Png: s.base64Png,
|
||||
name: s.name,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ export function MochaReporterProvider({ getService }) {
|
|||
const log = getService('log');
|
||||
const config = getService('config');
|
||||
const lifecycle = getService('lifecycle');
|
||||
const testMetadata = getService('testMetadata');
|
||||
let originalLogWriters;
|
||||
let reporterCaptureStartTime;
|
||||
|
||||
|
|
@ -61,7 +60,6 @@ export function MochaReporterProvider({ getService }) {
|
|||
config,
|
||||
lifecycle,
|
||||
runner,
|
||||
testMetadata,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,8 +48,10 @@ export function validateCiGroupTags(log, mocha) {
|
|||
const queue = [mocha.suite];
|
||||
while (queue.length) {
|
||||
const suite = queue.shift();
|
||||
if (getCiGroups(suite).length > 1) {
|
||||
suitesWithMultipleCiGroups.push(suite);
|
||||
if (getCiGroups(suite).length) {
|
||||
throw new Error(
|
||||
'ciGroups are no longer needed and should be removed. If you need to split up your FTR config because it is taking too long to complete then create one or more a new FTR config files and split your test files amoungst them'
|
||||
);
|
||||
} else {
|
||||
queue.push(...(suite.suites ?? []));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,6 @@
|
|||
*/
|
||||
|
||||
export { ProviderCollection } from './provider_collection';
|
||||
export { readProviderSpec } from './read_provider_spec';
|
||||
export * from './read_provider_spec';
|
||||
export { createAsyncInstance } from './async_instance';
|
||||
export type { Provider } from './read_provider_spec';
|
||||
|
|
|
|||
|
|
@ -6,11 +6,14 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
import { Suite, Test } from '../../fake_mocha_types';
|
||||
import { Lifecycle } from '../lifecycle';
|
||||
import { decorateSnapshotUi, expectSnapshot } from './decorate_snapshot_ui';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
const createRootSuite = () => {
|
||||
const suite = {
|
||||
|
|
@ -65,7 +68,7 @@ describe('decorateSnapshotUi', () => {
|
|||
let lifecycle: Lifecycle;
|
||||
let rootSuite: Suite;
|
||||
beforeEach(async () => {
|
||||
lifecycle = new Lifecycle();
|
||||
lifecycle = new Lifecycle(new ToolingLog());
|
||||
rootSuite = createRootSuite();
|
||||
decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });
|
||||
|
||||
|
|
@ -116,7 +119,7 @@ describe('decorateSnapshotUi', () => {
|
|||
let lifecycle: Lifecycle;
|
||||
let rootSuite: Suite;
|
||||
beforeEach(async () => {
|
||||
lifecycle = new Lifecycle();
|
||||
lifecycle = new Lifecycle(new ToolingLog());
|
||||
rootSuite = createRootSuite();
|
||||
decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });
|
||||
|
||||
|
|
@ -162,7 +165,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
|
|||
let lifecycle: Lifecycle;
|
||||
let rootSuite: Suite;
|
||||
beforeEach(async () => {
|
||||
lifecycle = new Lifecycle();
|
||||
lifecycle = new Lifecycle(new ToolingLog());
|
||||
rootSuite = createRootSuite();
|
||||
decorateSnapshotUi({ lifecycle, updateSnapshots: true, isCi: false });
|
||||
|
||||
|
|
@ -185,7 +188,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
|
|||
fs.writeFileSync(
|
||||
snapshotFile,
|
||||
`// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
|
||||
exports[\`Test 1\`] = \`"foo"\`;
|
||||
`,
|
||||
{ encoding: 'utf-8' }
|
||||
|
|
@ -219,7 +222,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
|
|||
let lifecycle: Lifecycle;
|
||||
let rootSuite: Suite;
|
||||
beforeEach(async () => {
|
||||
lifecycle = new Lifecycle();
|
||||
lifecycle = new Lifecycle(new ToolingLog());
|
||||
rootSuite = createRootSuite();
|
||||
decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: true });
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
import fs from 'fs';
|
||||
import { join, resolve } from 'path';
|
||||
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
jest.mock('fs');
|
||||
jest.mock('@kbn/utils', () => {
|
||||
return { REPO_ROOT: '/dev/null/root' };
|
||||
|
|
@ -60,7 +62,7 @@ describe('SuiteTracker', () => {
|
|||
};
|
||||
|
||||
const runLifecycleWithMocks = async (mocks: Suite[], fn: (objs: any) => any = () => {}) => {
|
||||
const lifecycle = new Lifecycle();
|
||||
const lifecycle = new Lifecycle(new ToolingLog());
|
||||
const suiteTracker = SuiteTracker.startTracking(
|
||||
lifecycle,
|
||||
resolve(REPO_ROOT, MOCK_CONFIG_PATH)
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { Lifecycle } from './lifecycle';
|
||||
|
||||
export interface ScreenshotRecord {
|
||||
name: string;
|
||||
base64Png: string;
|
||||
baselinePath?: string;
|
||||
failurePath?: string;
|
||||
}
|
||||
|
||||
export class TestMetadata {
|
||||
// mocha's global types mean we can't import Mocha or it will override the global jest types..............
|
||||
private currentRunnable?: any;
|
||||
|
||||
constructor(lifecycle: Lifecycle) {
|
||||
lifecycle.beforeEachRunnable.add((runnable) => {
|
||||
this.currentRunnable = runnable;
|
||||
});
|
||||
}
|
||||
|
||||
addScreenshot(screenshot: ScreenshotRecord) {
|
||||
this.currentRunnable._screenshots = (this.currentRunnable._screenshots || []).concat(
|
||||
screenshot
|
||||
);
|
||||
}
|
||||
|
||||
getScreenshots(test: any): ScreenshotRecord[] {
|
||||
if (!test || typeof test !== 'object' || !test._screenshots) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return test._screenshots.slice();
|
||||
}
|
||||
}
|
||||
|
|
@ -8,10 +8,10 @@
|
|||
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
import type { Config, Lifecycle, TestMetadata, DockerServersService, EsVersion } from './lib';
|
||||
import type { Config, Lifecycle, DockerServersService, EsVersion } from './lib';
|
||||
import type { Test, Suite } from './fake_mocha_types';
|
||||
|
||||
export { Lifecycle, Config, TestMetadata };
|
||||
export { Lifecycle, Config };
|
||||
|
||||
export interface AsyncInstance<T> {
|
||||
/**
|
||||
|
|
@ -56,9 +56,7 @@ export interface GenericFtrProviderContext<
|
|||
* Determine if a service is avaliable
|
||||
* @param serviceName
|
||||
*/
|
||||
hasService(
|
||||
serviceName: 'config' | 'log' | 'lifecycle' | 'testMetadata' | 'dockerServers' | 'esVersion'
|
||||
): true;
|
||||
hasService(serviceName: 'config' | 'log' | 'lifecycle' | 'dockerServers' | 'esVersion'): true;
|
||||
hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
|
||||
hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
|
||||
|
||||
|
|
@ -71,7 +69,6 @@ export interface GenericFtrProviderContext<
|
|||
getService(serviceName: 'log'): ToolingLog;
|
||||
getService(serviceName: 'lifecycle'): Lifecycle;
|
||||
getService(serviceName: 'dockerServers'): DockerServersService;
|
||||
getService(serviceName: 'testMetadata'): TestMetadata;
|
||||
getService(serviceName: 'esVersion'): EsVersion;
|
||||
getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
|
||||
|
||||
|
|
|
|||
|
|
@ -90,6 +90,9 @@ export async function assertNoneExcluded({ configPath, options }: CreateFtrParam
|
|||
}
|
||||
|
||||
const stats = await ftr.getTestStats();
|
||||
if (!stats) {
|
||||
throw new Error('unable to get test stats');
|
||||
}
|
||||
if (stats.testsExcludedByTag.length > 0) {
|
||||
throw new CliError(`
|
||||
${stats.testsExcludedByTag.length} tests in the ${configPath} config
|
||||
|
|
@ -122,5 +125,8 @@ export async function hasTests({ configPath, options }: CreateFtrParams) {
|
|||
return true;
|
||||
}
|
||||
const stats = await ftr.getTestStats();
|
||||
return stats.testCount > 0;
|
||||
if (!stats) {
|
||||
throw new Error('unable to get test stats');
|
||||
}
|
||||
return stats.nonSkippedTestCount > 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,13 +36,13 @@ export async function runKibanaServer({
|
|||
config: Config;
|
||||
options: { installDir?: string; extraKbnOpts?: string[] };
|
||||
}) {
|
||||
const { installDir } = options;
|
||||
const runOptions = config.get('kbnTestServer.runOptions');
|
||||
const installDir = runOptions.alwaysUseSource ? undefined : options.installDir;
|
||||
const env = config.get('kbnTestServer.env');
|
||||
|
||||
await procs.run('kibana', {
|
||||
cmd: getKibanaCmd(installDir),
|
||||
args: filterCliArgs(collectCliArgs(config, options)),
|
||||
args: filterCliArgs(collectCliArgs(config, installDir, options.extraKbnOpts)),
|
||||
env: {
|
||||
FORCE_COLOR: 1,
|
||||
...process.env,
|
||||
|
|
@ -70,10 +70,7 @@ function getKibanaCmd(installDir?: string) {
|
|||
* passed, we run from source code. We also allow passing in extra
|
||||
* Kibana server options, so we tack those on here.
|
||||
*/
|
||||
function collectCliArgs(
|
||||
config: Config,
|
||||
{ installDir, extraKbnOpts }: { installDir?: string; extraKbnOpts?: string[] }
|
||||
) {
|
||||
function collectCliArgs(config: Config, installDir?: string, extraKbnOpts: string[] = []) {
|
||||
const buildArgs: string[] = config.get('kbnTestServer.buildArgs') || [];
|
||||
const sourceArgs: string[] = config.get('kbnTestServer.sourceArgs') || [];
|
||||
const serverArgs: string[] = config.get('kbnTestServer.serverArgs') || [];
|
||||
|
|
@ -82,7 +79,7 @@ function collectCliArgs(
|
|||
serverArgs,
|
||||
(args) => (installDir ? args.filter((a: string) => a !== '--oss') : args),
|
||||
(args) => (installDir ? [...buildArgs, ...args] : [KIBANA_EXEC_PATH, ...sourceArgs, ...args]),
|
||||
(args) => args.concat(extraKbnOpts || [])
|
||||
(args) => args.concat(extraKbnOpts)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
import { relative } from 'path';
|
||||
import * as Rx from 'rxjs';
|
||||
import { setTimeout } from 'timers/promises';
|
||||
import { startWith, switchMap, take } from 'rxjs/operators';
|
||||
import { withProcRunner } from '@kbn/dev-utils';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
|
|
@ -63,7 +64,7 @@ interface RunTestsParams extends CreateFtrOptions {
|
|||
assertNoneExcluded: boolean;
|
||||
}
|
||||
export async function runTests(options: RunTestsParams) {
|
||||
if (!process.env.KBN_NP_PLUGINS_BUILT && !options.assertNoneExcluded) {
|
||||
if (!process.env.CI && !options.assertNoneExcluded) {
|
||||
const log = options.createLogger();
|
||||
log.warning('❗️❗️❗️');
|
||||
log.warning('❗️❗️❗️');
|
||||
|
|
@ -91,21 +92,18 @@ export async function runTests(options: RunTestsParams) {
|
|||
return;
|
||||
}
|
||||
|
||||
log.write('--- determining which ftr configs to run');
|
||||
const configPathsWithTests: string[] = [];
|
||||
for (const configPath of options.configs) {
|
||||
log.info('testing', relative(REPO_ROOT, configPath));
|
||||
await log.indent(4, async () => {
|
||||
if (await hasTests({ configPath, options: { ...options, log } })) {
|
||||
configPathsWithTests.push(configPath);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const [i, configPath] of configPathsWithTests.entries()) {
|
||||
for (const [i, configPath] of options.configs.entries()) {
|
||||
await log.indent(0, async () => {
|
||||
const progress = `${i + 1}/${configPathsWithTests.length}`;
|
||||
log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
|
||||
if (options.configs.length > 1) {
|
||||
const progress = `${i + 1}/${options.configs.length}`;
|
||||
log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
|
||||
}
|
||||
|
||||
if (!(await hasTests({ configPath, options: { ...options, log } }))) {
|
||||
// just run the FTR, no Kibana or ES, which will quickly report a skipped test group to ci-stats and continue
|
||||
await runFtr({ configPath, options: { ...options, log } });
|
||||
return;
|
||||
}
|
||||
|
||||
await withProcRunner(log, async (procs) => {
|
||||
const config = await readConfigFile(log, options.esVersion, configPath);
|
||||
|
|
@ -122,7 +120,7 @@ export async function runTests(options: RunTestsParams) {
|
|||
const delay = config.get('kbnTestServer.delayShutdown');
|
||||
if (typeof delay === 'number') {
|
||||
log.info('Delaying shutdown of Kibana for', delay, 'ms');
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
await setTimeout(delay);
|
||||
}
|
||||
|
||||
await procs.stop('kibana');
|
||||
|
|
|
|||
|
|
@ -17,13 +17,13 @@ This directory is excluded from the build and tools within it should help users
|
|||
|
||||
## Functional Test Scripts
|
||||
|
||||
**`node scripts/functional_tests [--config test/functional/config.js --config test/api_integration/config.js]`**
|
||||
**`node scripts/functional_tests [--config test/functional/config.base.js --config test/api_integration/config.js]`**
|
||||
|
||||
Runs all the functional tests: selenium tests and api integration tests. List configs with multiple `--config` arguments. Uses the [@kbn/test](../packages/kbn-test) library to run Elasticsearch and Kibana servers and tests against those servers, for multiple server+test setups. In particular, calls out to [`runTests()`](../packages/kbn-test/src/functional_tests/tasks.js). Can be run on a single config.
|
||||
|
||||
**`node scripts/functional_tests_server [--config test/functional/config.js]`**
|
||||
**`node scripts/functional_tests_server [--config test/functional/config.base.js]`**
|
||||
|
||||
Starts just the Elasticsearch and Kibana servers given a single config, i.e. via `--config test/functional/config.js` or `--config test/api_integration/config`. Allows the user to start just the servers with this script, and keep them running while running tests against these servers. The idea is that the same config file configures both Elasticsearch and Kibana servers. Uses the [`startServers()`](../packages/kbn-test/src/functional_tests/tasks.js#L52-L80) method from [@kbn/test](../packages/kbn-test) library.
|
||||
Starts just the Elasticsearch and Kibana servers given a single config, i.e. via `--config test/functional/config.base.js` or `--config test/api_integration/config`. Allows the user to start just the servers with this script, and keep them running while running tests against these servers. The idea is that the same config file configures both Elasticsearch and Kibana servers. Uses the [`startServers()`](../packages/kbn-test/src/functional_tests/tasks.js#L52-L80) method from [@kbn/test](../packages/kbn-test) library.
|
||||
|
||||
Example. Start servers _and_ run tests, separately, but using the same config:
|
||||
|
||||
|
|
@ -51,7 +51,7 @@ If you wish to load up specific es archived data for your test, you can do so vi
|
|||
node scripts/es_archiver.js load <archive> [--es-url=http://username:password@localhost:9200] [--kibana-url=http://username:password@localhost:5601/{basepath?}]
|
||||
```
|
||||
|
||||
That will load the specified archive located in the archive directory specified by the default functional config file, located in `test/functional/config.js`. To load archives from other function config files you can pass `--config path/to/config.js`.
|
||||
That will load the specified archive located in the archive directory specified by the default functional config file, located in `test/functional/config.base.js`. To load archives from other function config files you can pass `--config path/to/config.js`.
|
||||
|
||||
*Note:* The `--es-url` and `--kibana-url` options may or may not be neccessary depending on your current Kibana configuration settings, and their values
|
||||
may also change based on those settings (for example if you are not running with security you will not need the `username:password` portion).
|
||||
|
|
|
|||
|
|
@ -7,26 +7,4 @@
|
|||
*/
|
||||
|
||||
require('../src/setup_node_env');
|
||||
require('@kbn/test').runTestsCli([
|
||||
require.resolve('../test/functional/config.ccs.ts'),
|
||||
require.resolve('../test/functional/config.js'),
|
||||
require.resolve('../test/plugin_functional/config.ts'),
|
||||
require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
|
||||
require.resolve('../test/new_visualize_flow/config.ts'),
|
||||
require.resolve('../test/interactive_setup_api_integration/enrollment_flow.config.ts'),
|
||||
require.resolve('../test/interactive_setup_api_integration/manual_configuration_flow.config.ts'),
|
||||
require.resolve(
|
||||
'../test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts'
|
||||
),
|
||||
require.resolve('../test/interactive_setup_functional/enrollment_token.config.ts'),
|
||||
require.resolve('../test/interactive_setup_functional/manual_configuration.config.ts'),
|
||||
require.resolve(
|
||||
'../test/interactive_setup_functional/manual_configuration_without_security.config.ts'
|
||||
),
|
||||
require.resolve(
|
||||
'../test/interactive_setup_functional/manual_configuration_without_tls.config.ts'
|
||||
),
|
||||
require.resolve('../test/api_integration/config.js'),
|
||||
require.resolve('../test/interpreter_functional/config.ts'),
|
||||
require.resolve('../test/examples/config.js'),
|
||||
]);
|
||||
require('@kbn/test').runTestsCli();
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@
|
|||
*/
|
||||
|
||||
require('../src/setup_node_env');
|
||||
require('@kbn/test').startServersCli(require.resolve('../test/functional/config.js'));
|
||||
require('@kbn/test').startServersCli(require.resolve('../test/functional/config.base.js'));
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { services } from './services';
|
|||
import { pageObjects } from './page_objects';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import { services } from './services';
|
|||
*/
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const commonConfig = await readConfigFile(require.resolve('../common/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
|
||||
|
||||
return {
|
||||
testFiles: [require.resolve('./tests')],
|
||||
|
|
|
|||
|
|
@ -14,5 +14,6 @@ export async function getKibanaVersion(getService: FtrProviderContext['getServic
|
|||
const kibanaVersion = await kibanaServer.version.get();
|
||||
expect(typeof kibanaVersion).to.eql('string');
|
||||
expect(kibanaVersion.length).to.be.greaterThan(0);
|
||||
return kibanaVersion;
|
||||
// mimic SavedObjectsService.stripVersionQualifier()
|
||||
return kibanaVersion.split('-')[0];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { services } from './services';
|
|||
|
||||
export default async function ({ readConfigFile }) {
|
||||
const commonConfig = await readConfigFile(require.resolve('../common/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
|
||||
|
||||
return {
|
||||
rootTags: ['runOutsideOfCiGroups'],
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
|
|||
const PageObjects = getPageObjects(['common', 'header']);
|
||||
|
||||
describe('bfetch explorer', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1300, 900);
|
||||
await PageObjects.common.navigateToApp('bfetch-explorer', { insertTimestamp: false });
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import fs from 'fs';
|
|||
import { KIBANA_ROOT } from '@kbn/test';
|
||||
|
||||
export default async function ({ readConfigFile }) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
|
||||
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
|
||||
|
||||
// Find all folders in /examples and /x-pack/examples since we treat all them as plugin folder
|
||||
const examplesFiles = fs.readdirSync(resolve(KIBANA_ROOT, 'examples'));
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ export default function ({
|
|||
const PageObjects = getPageObjects(['common', 'header', 'settings']);
|
||||
|
||||
describe('data view field editor example', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await esArchiver.emptyKibanaIndex();
|
||||
await browser.setWindowSize(1300, 900);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ export default function ({
|
|||
const PageObjects = getPageObjects(['common', 'header']);
|
||||
|
||||
describe('embeddable explorer', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1300, 900);
|
||||
await PageObjects.common.navigateToApp('embeddableExplorer');
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ export default function ({
|
|||
const PageObjects = getPageObjects(['common', 'header']);
|
||||
|
||||
describe('expressions explorer', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1300, 900);
|
||||
await PageObjects.common.navigateToApp('expressionsExplorer');
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
|
||||
describe('Field formats example', function () {
|
||||
before(async () => {
|
||||
this.tags('ciGroup11');
|
||||
await PageObjects.common.navigateToApp('fieldFormatsExample');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
|
|||
|
||||
describe('Hello world', function () {
|
||||
before(async () => {
|
||||
this.tags('ciGroup11');
|
||||
await PageObjects.common.navigateToApp('helloWorld');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
|
||||
describe('Partial Results Example', function () {
|
||||
before(async () => {
|
||||
this.tags('ciGroup11');
|
||||
await PageObjects.common.navigateToApp('partialResultsExample');
|
||||
|
||||
const element = await testSubjects.find('example-help');
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
|
|||
|
||||
describe('routing examples', function () {
|
||||
before(async () => {
|
||||
this.tags('ciGroup11');
|
||||
await PageObjects.common.navigateToApp('routingExample');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ export default function ({
|
|||
const browser = getService('browser');
|
||||
|
||||
describe('state sync examples', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1300, 900);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ export default function ({
|
|||
const PageObjects = getPageObjects(['common', 'header']);
|
||||
|
||||
describe('ui actions explorer', function () {
|
||||
this.tags('ciGroup11');
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1300, 900);
|
||||
await PageObjects.common.navigateToApp('uiActionsExplorer');
|
||||
|
|
|
|||
|
|
@ -6,18 +6,13 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export default async function ({ readConfigFile }) {
|
||||
const defaultConfig = await readConfigFile(require.resolve('./config'));
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
|
||||
|
||||
return {
|
||||
...defaultConfig.getAll(),
|
||||
|
||||
suiteTags: {
|
||||
exclude: ['skipCoverage'],
|
||||
},
|
||||
|
||||
junit: {
|
||||
reportName: 'Code Coverage for Functional Tests',
|
||||
},
|
||||
...functionalConfig.getAll(),
|
||||
testFiles: [require.resolve('.')],
|
||||
};
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ export default function ({ getService }) {
|
|||
const supertest = getService('supertest');
|
||||
|
||||
describe('bundle compression', function () {
|
||||
this.tags(['ciGroup11', 'skipCoverage']);
|
||||
this.tags('skipCoverage');
|
||||
|
||||
let buildNum;
|
||||
before(async () => {
|
||||
|
|
|
|||
18
test/functional/apps/console/config.ts
Normal file
18
test/functional/apps/console/config.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
testFiles: [require.resolve('.')],
|
||||
};
|
||||
}
|
||||
|
|
@ -10,8 +10,6 @@ export default function ({ getService, loadTestFile }) {
|
|||
const browser = getService('browser');
|
||||
|
||||
describe('console app', function () {
|
||||
this.tags('ciGroup1');
|
||||
|
||||
before(async function () {
|
||||
await browser.setWindowSize(1300, 1100);
|
||||
});
|
||||
|
|
|
|||
18
test/functional/apps/context/config.ts
Normal file
18
test/functional/apps/context/config.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
testFiles: [require.resolve('.')],
|
||||
};
|
||||
}
|
||||
|
|
@ -15,8 +15,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
|
|||
const kibanaServer = getService('kibanaServer');
|
||||
|
||||
describe('context app', function () {
|
||||
this.tags('ciGroup1');
|
||||
|
||||
before(async () => {
|
||||
await browser.setWindowSize(1200, 800);
|
||||
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
|
||||
|
|
|
|||
7
test/functional/apps/dashboard/README.md
Normal file
7
test/functional/apps/dashboard/README.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# What are all these groups?
|
||||
|
||||
These tests take a while so they have been broken up into groups with their own `config.ts` and `index.ts` file, causing each of these groups to be independent bundles of tests which can be run on some worker in CI without taking an incredible amount of time.
|
||||
|
||||
Want to change the groups to something more logical? Have fun! Just make sure that each group executes on CI in less than 10 minutes or so. We don't currently have any mechanism for validating this right now, you just need to look at the times in the log output on CI, but we'll be working on tooling for making this information more accessible soon.
|
||||
|
||||
- Kibana Operations
|
||||
18
test/functional/apps/dashboard/group1/config.ts
Normal file
18
test/functional/apps/dashboard/group1/config.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
testFiles: [require.resolve('.')],
|
||||
};
|
||||
}
|
||||
|
|
@ -10,7 +10,7 @@ import expect from '@kbn/expect';
|
|||
|
||||
import { VisualizeConstants } from '@kbn/visualizations-plugin/common/constants';
|
||||
import { VISUALIZE_ENABLE_LABS_SETTING } from '@kbn/visualizations-plugin/common/constants';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const retry = getService('retry');
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const kibanaServer = getService('kibanaServer');
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects(['dashboard', 'header', 'common']);
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const retry = getService('retry');
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const esArchiver = getService('esArchiver');
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']);
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue