github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ftr] automatically determine config run order

Browse source
This commit is contained in:
spalger 2022-04-28 13:00:07 -05:00
parent ef6a4e4498
commit 8bc10881a1
451 changed files with 2637 additions and 1447 deletions
Download patch file Download diff file Expand all files Collapse all files

181
.buildkite/ftr_configs.yml Normal file
View file

@ -0,0 +1,181 @@
disabled:
# TODO: Enable once RBAC timeline search strategy test updated
- x-pack/test/timeline/security_and_spaces/config_basic.ts
enabled:
- test/accessibility/config.ts
- test/analytics/config.ts
- test/api_integration/config.js
- test/api_integration/config.js
- test/examples/config.js
- test/functional/apps/console/config.ts
- test/functional/apps/context/config.ts
- test/functional/apps/dashboard_elements/config.ts
- test/functional/apps/dashboard/group1/config.ts
- test/functional/apps/dashboard/group2/config.ts
- test/functional/apps/dashboard/group3/config.ts
- test/functional/apps/dashboard/group4/config.ts
- test/functional/apps/dashboard/group5/config.ts
- test/functional/apps/discover/config.ts
- test/functional/apps/getting_started/config.ts
- test/functional/apps/home/config.ts
- test/functional/apps/management/config.ts
- test/functional/apps/saved_objects_management/config.ts
- test/functional/apps/status_page/config.ts
- test/functional/apps/visualize/group1/config.ts
- test/functional/apps/visualize/group2/config.ts
- test/functional/apps/visualize/group3/config.ts
- test/functional/apps/visualize/group4/config.ts
- test/functional/apps/visualize/group5/config.ts
- test/functional/apps/visualize/group6/config.ts
- test/functional/apps/visualize/replaced_vislib_chart_types/config.ts
- test/functional/config.ccs.ts
- test/functional/config.firefox.js
- test/interactive_setup_api_integration/enrollment_flow.config.ts
- test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
- test/interactive_setup_api_integration/manual_configuration_flow.config.ts
- test/interactive_setup_functional/enrollment_token.config.ts
- test/interactive_setup_functional/manual_configuration_without_security.config.ts
- test/interactive_setup_functional/manual_configuration_without_tls.config.ts
- test/interactive_setup_functional/manual_configuration.config.ts
- test/interpreter_functional/config.ts
- test/interpreter_functional/config.ts
- test/new_visualize_flow/config.ts
- test/plugin_functional/config.ts
- test/plugin_functional/config.ts
- test/server_integration/http/platform/config.status.ts
- test/server_integration/http/platform/config.ts
- test/server_integration/http/ssl_redirect/config.js
- test/server_integration/http/ssl_with_p12_intermediate/config.js
- test/server_integration/http/ssl_with_p12/config.js
- test/server_integration/http/ssl/config.js
- test/ui_capabilities/newsfeed_err/config.ts
- x-pack/test/accessibility/config.ts
- x-pack/test/alerting_api_integration/basic/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/config.ts
- x-pack/test/alerting_api_integration/spaces_only/config.ts
- x-pack/test/api_integration_basic/config.ts
- x-pack/test/api_integration/config_security_basic.ts
- x-pack/test/api_integration/config_security_trial.ts
- x-pack/test/api_integration/config.ts
- x-pack/test/apm_api_integration/basic/config.ts
- x-pack/test/apm_api_integration/rules/config.ts
- x-pack/test/apm_api_integration/trial/config.ts
- x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts
- x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts
- x-pack/test/cases_api_integration/spaces_only/config.ts
- x-pack/test/detection_engine_api_integration/basic/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group2/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group3/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
- x-pack/test/encrypted_saved_objects_api_integration/config.ts
- x-pack/test/endpoint_api_integration_no_ingest/config.ts
- x-pack/test/examples/config.ts
- x-pack/test/fleet_api_integration/config.ts
- x-pack/test/fleet_functional/config.ts
- x-pack/test/functional_basic/config.ts
- x-pack/test/functional_cors/config.ts
- x-pack/test/functional_embedded/config.ts
- x-pack/test/functional_enterprise_search/without_host_configured.config.ts
- x-pack/test/functional_execution_context/config.ts
- x-pack/test/functional_synthetics/config.js
- x-pack/test/functional_with_es_ssl/config.ts
- x-pack/test/functional/apps/advanced_settings/config.ts
- x-pack/test/functional/apps/api_keys/config.ts
- x-pack/test/functional/apps/apm/config.ts
- x-pack/test/functional/apps/canvas/config.ts
- x-pack/test/functional/apps/cross_cluster_replication/config.ts
- x-pack/test/functional/apps/dashboard/config.ts
- x-pack/test/functional/apps/data_views/config.ts
- x-pack/test/functional/apps/dev_tools/config.ts
- x-pack/test/functional/apps/discover/config.ts
- x-pack/test/functional/apps/graph/config.ts
- x-pack/test/functional/apps/grok_debugger/config.ts
- x-pack/test/functional/apps/home/config.ts
- x-pack/test/functional/apps/index_lifecycle_management/config.ts
- x-pack/test/functional/apps/index_management/config.ts
- x-pack/test/functional/apps/infra/config.ts
- x-pack/test/functional/apps/ingest_pipelines/config.ts
- x-pack/test/functional/apps/lens/config.ts
- x-pack/test/functional/apps/license_management/config.ts
- x-pack/test/functional/apps/logstash/config.ts
- x-pack/test/functional/apps/management/config.ts
- x-pack/test/functional/apps/maps/config.ts
- x-pack/test/functional/apps/ml/group1/config.ts
- x-pack/test/functional/apps/ml/group2/config.ts
- x-pack/test/functional/apps/ml/group3/config.ts
- x-pack/test/functional/apps/monitoring/config.ts
- x-pack/test/functional/apps/remote_clusters/config.ts
- x-pack/test/functional/apps/reporting_management/config.ts
- x-pack/test/functional/apps/rollup_job/config.ts
- x-pack/test/functional/apps/saved_objects_management/config.ts
- x-pack/test/functional/apps/security/config.ts
- x-pack/test/functional/apps/snapshot_restore/config.ts
- x-pack/test/functional/apps/spaces/config.ts
- x-pack/test/functional/apps/status_page/config.ts
- x-pack/test/functional/apps/transform/config.ts
- x-pack/test/functional/apps/upgrade_assistant/config.ts
- x-pack/test/functional/apps/uptime/config.ts
- x-pack/test/functional/apps/visualize/config.ts
- x-pack/test/functional/apps/watcher/config.ts
- x-pack/test/functional/config_security_basic.ts
- x-pack/test/functional/config.ccs.ts
- x-pack/test/functional/config.firefox.js
- x-pack/test/licensing_plugin/config.public.ts
- x-pack/test/licensing_plugin/config.ts
- x-pack/test/lists_api_integration/security_and_spaces/config.ts
- x-pack/test/observability_api_integration/basic/config.ts
- x-pack/test/observability_api_integration/trial/config.ts
- x-pack/test/observability_functional/with_rac_write.config.ts
- x-pack/test/plugin_api_integration/config.ts
- x-pack/test/plugin_functional/config.ts
- x-pack/test/reporting_api_integration/reporting_and_security.config.ts
- x-pack/test/reporting_api_integration/reporting_without_security.config.ts
- x-pack/test/reporting_functional/reporting_and_deprecated_security.config.ts
- x-pack/test/reporting_functional/reporting_and_security.config.ts
- x-pack/test/reporting_functional/reporting_without_security.config.ts
- x-pack/test/rule_registry/security_and_spaces/config_basic.ts
- x-pack/test/rule_registry/security_and_spaces/config_trial.ts
- x-pack/test/rule_registry/spaces_only/config_basic.ts
- x-pack/test/rule_registry/spaces_only/config_trial.ts
- x-pack/test/saved_object_api_integration/security_and_spaces/config_basic.ts
- x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts
- x-pack/test/saved_object_api_integration/spaces_only/config.ts
- x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts
- x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts
- x-pack/test/saved_object_tagging/functional/config.ts
- x-pack/test/saved_objects_field_count/config.ts
- x-pack/test/search_sessions_integration/config.ts
- x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts
- x-pack/test/security_api_integration/anonymous.config.ts
- x-pack/test/security_api_integration/audit.config.ts
- x-pack/test/security_api_integration/http_bearer.config.ts
- x-pack/test/security_api_integration/http_no_auth_providers.config.ts
- x-pack/test/security_api_integration/kerberos_anonymous_access.config.ts
- x-pack/test/security_api_integration/kerberos.config.ts
- x-pack/test/security_api_integration/login_selector.config.ts
- x-pack/test/security_api_integration/oidc_implicit_flow.config.ts
- x-pack/test/security_api_integration/oidc.config.ts
- x-pack/test/security_api_integration/pki.config.ts
- x-pack/test/security_api_integration/saml.config.ts
- x-pack/test/security_api_integration/session_idle.config.ts
- x-pack/test/security_api_integration/session_invalidate.config.ts
- x-pack/test/security_api_integration/session_lifespan.config.ts
- x-pack/test/security_api_integration/token.config.ts
- x-pack/test/security_functional/login_selector.config.ts
- x-pack/test/security_functional/oidc.config.ts
- x-pack/test/security_functional/saml.config.ts
- x-pack/test/security_solution_endpoint_api_int/config.ts
- x-pack/test/security_solution_endpoint/config.ts
- x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
- x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts
- x-pack/test/spaces_api_integration/spaces_only/config.ts
- x-pack/test/timeline/security_and_spaces/config_trial.ts
- x-pack/test/ui_capabilities/security_and_spaces/config.ts
- x-pack/test/ui_capabilities/spaces_only/config.ts
- x-pack/test/upgrade_assistant_integration/config.js
- x-pack/test/usage_collection/config.ts

10
.buildkite/package-lock.json generated
View file

@ -8,7 +8,7 @@
"name": "kibana-buildkite",
"version": "1.0.0",
"dependencies": {
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
"kibana-buildkite-library": "git+https://git@github.com/spalger/kibana-buildkite-library#implement/ftr-config-run-order"
}
},
"node_modules/@nodelib/fs.scandir": {
@ -355,8 +355,7 @@
},
"node_modules/kibana-buildkite-library": {
"version": "1.0.0",
"resolved": "git+https://git@github.com/elastic/kibana-buildkite-library.git#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
"integrity": "sha512-Ayiyy3rAE/jOWcR65vxiv4zacMlpxuRZ+WKvly6magfClWTWIUTcW1aiOH2/PYWP3faiCbIDHOyxLeGGajk5dQ==",
"resolved": "git+https://git@github.com/spalger/kibana-buildkite-library.git#bd36e8374e812d9f522089213683283b9e207281",
"license": "MIT",
"dependencies": {
"@octokit/rest": "^18.10.0",
@ -801,9 +800,8 @@
}
},
"kibana-buildkite-library": {
"version": "git+https://git@github.com/elastic/kibana-buildkite-library.git#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
"integrity": "sha512-Ayiyy3rAE/jOWcR65vxiv4zacMlpxuRZ+WKvly6magfClWTWIUTcW1aiOH2/PYWP3faiCbIDHOyxLeGGajk5dQ==",
"from": "kibana-buildkite-library@git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e",
"version": "git+https://git@github.com/spalger/kibana-buildkite-library.git#bd36e8374e812d9f522089213683283b9e207281",
"from": "kibana-buildkite-library@git+https://git@github.com/spalger/kibana-buildkite-library#implement/ftr-config-run-order",
"requires": {
"@octokit/rest": "^18.10.0",
"axios": "^0.21.4",

2
.buildkite/package.json
View file

@ -3,6 +3,6 @@
"version": "1.0.0",
"private": true,
"dependencies": {
"kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
"kibana-buildkite-library": "git+https://git@github.com/spalger/kibana-buildkite-library#implement/ftr-config-run-order"
}
}

172
.buildkite/pipelines/pull_request/base.yml
View file

@ -1,3 +1,6 @@
env:
KIBANA_BUILD_ID: 148e1c71-94a7-460e-8ca4-7257919cf4e1
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build
@ -15,176 +18,11 @@ steps:
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
timeout_in_minutes: 60
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Default CI Group'
parallelism: 31
agents:
queue: n2-4-spot-2
depends_on: build
timeout_in_minutes: 150
key: default-cigroup
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
label: 'OSS CI Group'
parallelism: 12
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
key: oss-cigroup
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/oss_accessibility.sh
label: 'OSS Accessibility Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
label: 'Default Accessibility Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/oss_firefox.sh
label: 'OSS Firefox Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/xpack_firefox.sh
label: 'Default Firefox Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/oss_misc.sh
label: 'OSS Misc Functional Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
label: 'Saved Object Field Metrics'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
label: 'Pick Jest Config Run Order'
- command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
label: 'Pick Test Group Run Order'
agents:
queue: kibana-default
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/test/api_integration.sh
label: 'API Integration Tests'
agents:
queue: n2-2-spot
timeout_in_minutes: 120
key: api-integration
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: .buildkite/scripts/steps/lint.sh
label: 'Linting'
agents:
queue: n2-2
key: linting
timeout_in_minutes: 90
- command: .buildkite/scripts/steps/lint_with_types.sh
label: 'Linting (with types)'
agents:
queue: c2-16
key: linting_with_types
timeout_in_minutes: 90
- command: .buildkite/scripts/steps/checks.sh
label: 'Checks'
agents:
queue: c2-8
key: checks
timeout_in_minutes: 120
- command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
label: 'Build Storybooks'
agents:
queue: c2-4
key: storybooks
timeout_in_minutes: 60
- command: .buildkite/scripts/steps/build_api_docs.sh
label: 'Build API Docs'
agents:
queue: n2-4-spot
key: build_api_docs
timeout_in_minutes: 60
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
label: 'Build Webpack Bundle Analyzer reports'
agents:
queue: c2-4
key: webpack_bundle_analyzer
timeout_in_minutes: 60

67
.buildkite/scripts/pipelines/pull_request/pipeline.js
View file

@ -8,7 +8,7 @@
const execSync = require('child_process').execSync;
const fs = require('fs');
const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
const { areChangesSkippable } = require('kibana-buildkite-library');
const SKIPPABLE_PATHS = [
/^docs\//,
@ -61,71 +61,6 @@ const uploadPipeline = (pipelineContent) => {
const pipeline = [];
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/lists/,
/^x-pack\/plugins\/security_solution/,
/^x-pack\/plugins\/timelines/,
/^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
/^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
/^x-pack\/test\/security_solution_cypress/,
])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
}
if (
(await doAnyChangesMatch([
/^src\/plugins\/data/,
/^x-pack\/plugins\/actions/,
/^x-pack\/plugins\/alerting/,
/^x-pack\/plugins\/event_log/,
/^x-pack\/plugins\/rule_registry/,
/^x-pack\/plugins\/task_manager/,
])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
}
if (
(await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops_cases.yml'));
}
if (
(await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
}
if (
(await doAnyChangesMatch([/^x-pack\/plugins\/fleet/, /^x-pack\/test\/fleet_cypress/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fleet_cypress.yml'));
}
if (
(await doAnyChangesMatch([/^x-pack\/plugins\/osquery/, /^x-pack\/test\/osquery_cypress/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/osquery_cypress.yml'));
}
if (await doAnyChangesMatch([/^x-pack\/plugins\/uptime/])) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime.yml'));
}
if (process.env.GITHUB_PR_LABELS.includes('ci:deploy-cloud')) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
}
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
uploadPipeline(pipeline.join('\n'));

13
.buildkite/scripts/steps/functional/oss_accessibility.sh
View file

@ -1,13 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
echo --- OSS Accessibility Tests
checks-reporter-with-killswitch "Kibana accessibility tests" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config test/accessibility/config.ts

16
.buildkite/scripts/steps/functional/oss_cigroup.sh
View file

@ -1,16 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
export JOB=kibana-oss-ciGroup${CI_GROUP}
echo "--- OSS CI Group $CI_GROUP"
checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" \
node scripts/functional_tests \
--bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--include-tag "ciGroup$CI_GROUP"

14
.buildkite/scripts/steps/functional/oss_firefox.sh
View file

@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
echo --- OSS Firefox Smoke Tests
checks-reporter-with-killswitch "Firefox smoke test" \
node scripts/functional_tests \
--bail --debug \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--include-tag "includeFirefox" \
--config test/functional/config.firefox.js

51
.buildkite/scripts/steps/functional/oss_misc.sh
View file

@ -1,51 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
# Required, at least for plugin_functional tests
.buildkite/scripts/build_kibana_plugins.sh
echo --- Plugin Functional Tests
checks-reporter-with-killswitch "Plugin Functional Tests" \
node scripts/functional_tests \
--config test/plugin_functional/config.ts \
--bail \
--debug
echo --- Interpreter Functional Tests
checks-reporter-with-killswitch "Interpreter Functional Tests" \
node scripts/functional_tests \
--config test/interpreter_functional/config.ts \
--bail \
--debug \
--kibana-install-dir "$KIBANA_BUILD_LOCATION"
echo --- Server Integration Tests
checks-reporter-with-killswitch "Server Integration Tests" \
node scripts/functional_tests \
--config test/server_integration/http/ssl/config.js \
--config test/server_integration/http/ssl_redirect/config.js \
--config test/server_integration/http/platform/config.ts \
--config test/server_integration/http/ssl_with_p12/config.js \
--config test/server_integration/http/ssl_with_p12_intermediate/config.js \
--bail \
--debug \
--kibana-install-dir "$KIBANA_BUILD_LOCATION"
# Tests that must be run against source in order to build test plugins
echo --- Status Integration Tests
checks-reporter-with-killswitch "Status Integration Tests" \
node scripts/functional_tests \
--config test/server_integration/http/platform/config.status.ts \
--bail \
--debug
# Tests that must be run against source in order to build test plugins
echo --- Analytics Integration Tests
checks-reporter-with-killswitch "Analytics Integration Tests" \
node scripts/functional_tests \
--config test/analytics/config.ts \
--bail \
--debug

15
.buildkite/scripts/steps/functional/xpack_accessibility.sh
View file

@ -1,15 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
cd "$XPACK_DIR"
echo --- Default Accessibility Tests
checks-reporter-with-killswitch "X-Pack accessibility tests" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config test/accessibility/config.ts

20
.buildkite/scripts/steps/functional/xpack_cigroup.sh
View file

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
export CI_GROUP=${CI_GROUP:-$((BUILDKITE_PARALLEL_JOB+1))}
export JOB=kibana-default-ciGroup${CI_GROUP}
echo "--- Default CI Group $CI_GROUP"
cd "$XPACK_DIR"
checks-reporter-with-killswitch "X-Pack Chrome Functional tests / Group ${CI_GROUP}" \
node scripts/functional_tests \
--bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--include-tag "ciGroup$CI_GROUP"
cd "$KIBANA_DIR"

17
.buildkite/scripts/steps/functional/xpack_firefox.sh
View file

@ -1,17 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
cd "$XPACK_DIR"
echo --- Default Firefox Smoke Tests
checks-reporter-with-killswitch "X-Pack firefox smoke test" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--include-tag "includeFirefox" \
--config test/functional/config.firefox.js \
--config test/functional_embedded/config.firefox.ts

17
.buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
View file

@ -1,17 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh
cd "$XPACK_DIR"
echo --- Capture Kibana Saved Objects field count metrics
checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config test/saved_objects_field_count/config.ts;

16
.buildkite/scripts/steps/test/api_integration.sh
View file

@ -1,16 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
is_test_execution_step
.buildkite/scripts/bootstrap.sh
echo --- API Integration Tests
checks-reporter-with-killswitch "API Integration Tests" \
node scripts/functional_tests \
--config test/api_integration/config.js \
--bail \
--debug

36
.buildkite/scripts/steps/test/ftr_configs.sh Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
source .buildkite/scripts/steps/test/test_group_env.sh
export JOB_NUM=$BUILDKITE_PARALLEL_JOB
export JOB=ftr-configs-${JOB_NUM}
# a FTR failure will result in the script returning an exit code of 10
exitCode=0
echo "--- downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
while read -r config; do
echo "--- $ node scripts/functional_tests --bail --config $config"
# prevent non-zero exit code from breaking the loop
set +e;
node ./scripts/functional_tests \
--bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config="$config"
lastCode=$?
set -e;
if [ $lastCode -ne 0 ]; then
exitCode=10
echo "FTR exited with code $lastCode"
echo "^^^ +++"
fi
done <<< "$configs"
exit $exitCode

2
.buildkite/scripts/steps/test/jest_parallel.sh
View file

@ -20,7 +20,7 @@ else
fi
export TEST_TYPE
echo "--- downloading integration test run order"
echo "--- downloading jest test run order"
buildkite-agent artifact download jest_run_order.json .
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)

9
.buildkite/scripts/steps/test/pick_jest_config_run_order.sh
View file

@ -1,9 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/test/jest_env.sh
echo '--- Pick Jest Config Run Order'
node "$(dirname "${0}")/pick_jest_config_run_order.js"

0
.buildkite/scripts/steps/test/pick_jest_config_run_order.js → .buildkite/scripts/steps/test/pick_test_group_run_order.js
View file

9
.buildkite/scripts/steps/test/pick_test_group_run_order.sh Normal file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/test/test_group_env.sh
echo '--- Pick Test Group Run Order'
node "$(dirname "${0}")/pick_test_group_run_order.js"

1
.buildkite/scripts/steps/test/jest_env.sh → .buildkite/scripts/steps/test/test_group_env.sh
View file

@ -5,3 +5,4 @@ set -euo pipefail
# keys used to associate test group data in ci-stats with Jest execution order
export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"
export TEST_GROUP_TYPE_FUNCTIONAL="Functional Tests"

6
docs/developer/contributing/development-functional-tests.asciidoc
View file

@ -6,7 +6,7 @@ We use functional tests to make sure the {kib} UI works as expected. It replaces
[discrete]
=== Running functional tests
The `FunctionalTestRunner` is very bare bones and gets most of its functionality from its config file, located at {blob}test/functional/config.js[test/functional/config.js] or {blob}x-pack/test/functional/config.js[x-pack/test/functional/config.js]. If you’re writing a plugin outside the {kib} repo, you will have your own config file.
The `FunctionalTestRunner` is very bare bones and gets most of its functionality from its config file, located at {blob}test/functional/config.base.js[test/functional/config.base.js] or {blob}x-pack/test/functional/config.base.js[x-pack/test/functional/config.base.js]. If you’re writing a plugin outside the {kib} repo, you will have your own config file.
See <<external-plugin-functional-tests>> for more info.
There are three ways to run the tests depending on your goals:
@ -96,7 +96,7 @@ node scripts/functional_test_runner --exclude-tag skipCloud
When run without any arguments the `FunctionalTestRunner` automatically loads the configuration in the standard location, but you can override that behavior with the `--config` flag. List configs with multiple --config arguments.
* `--config test/functional/config.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Chrome.
* `--config test/functional/config.base.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Chrome.
* `--config test/functional/config.firefox.js` starts {es} and {kib} servers with the WebDriver tests configured to run in Firefox.
* `--config test/api_integration/config.js` starts {es} and {kib} servers with the api integration tests configuration.
* `--config test/accessibility/config.ts` starts {es} and {kib} servers with the WebDriver tests configured to run an accessibility audit using https://www.deque.com/axe/[axe].
@ -416,7 +416,7 @@ export function SomethingUsefulProvider({ getService }) {
-----------
+
* Re-export your provider from `services/index.js`
* Import it into `src/functional/config.js` and add it to the services config:
* Import it into `src/functional/config.base.js` and add it to the services config:
+
["source","js"]
-----------

4
docs/developer/plugin/external-plugin-functional-tests.asciidoc
View file

@ -8,7 +8,7 @@ Plugins use the `FunctionalTestRunner` by running it out of the {kib} repo. Ensu
Every project or plugin should have its own `FunctionalTestRunner` config file. Just like {kib}'s, this config file will define all of the test files to load, providers for Services and PageObjects, as well as configuration options for certain services.
To get started copy and paste this example to `test/functional/config.js`:
To get started copy and paste this example to `test/functional/config.base.js`:
["source","js"]
-----------
@ -24,7 +24,7 @@ export default async function ({ readConfigFile }) {
// read the {kib} config file so that we can utilize some of
// its services and PageObjects
const kibanaConfig = await readConfigFile(resolve(REPO_ROOT, 'test/functional/config.js'));
const kibanaConfig = await readConfigFile(resolve(REPO_ROOT, 'test/functional/config.base.js'));
return {
// list paths to the files that contain your plugins tests

2
packages/kbn-es-archiver/src/cli.ts
View file

@ -24,7 +24,7 @@ import { Client, HttpConnection } from '@elastic/elasticsearch';
import { EsArchiver } from './es_archiver';
const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v);
const defaultConfigPath = resolveConfigPath('test/functional/config.js');
const defaultConfigPath = resolveConfigPath('test/functional/config.base.js');
export function runCli() {
new RunWithCommands({

4
packages/kbn-test/README.md
View file

@ -15,14 +15,14 @@ Functional testing methods exist in the `src/functional_tests` directory. They d
#### runTests(configPaths: Array<string>)
For each config file specified in configPaths, starts Elasticsearch and Kibana once, runs tests specified in that config file, and shuts down Elasticsearch and Kibana once completed. (Repeats for every config file.)
`configPaths`: array of strings, each an absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
`configPaths`: array of strings, each an absolute path to a config file that looks like [this](../../test/functional/config.base.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
Internally the method that starts Elasticsearch comes from [kbn-es](../../packages/kbn-es).
#### startServers(configPath: string)
Starts Elasticsearch and Kibana servers given a specified config.
`configPath`: absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
`configPath`: absolute path to a config file that looks like [this](../../test/functional/config.base.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
Allows users to start another process to run just the tests while keeping the servers running with this method. Start servers _and_ run tests using the same config file ([see how](../../scripts/README.md)).

115
packages/kbn-test/src/functional_test_runner/functional_test_runner.ts
View file

@ -18,6 +18,7 @@ import {
TestMetadata,
readConfigFile,
ProviderCollection,
Providers,
readProviderSpec,
setupMocha,
runTests,
@ -55,19 +56,29 @@ export class FunctionalTestRunner {
}
async run() {
const testStats = await this.getTestStats();
return await this._run(async (config, coreProviders) => {
SuiteTracker.startTracking(this.lifecycle, this.configFile);
const providers = new ProviderCollection(this.log, [
...coreProviders,
...readProviderSpec('Service', config.get('services')),
...readProviderSpec('PageObject', config.get('pageObjects')),
]);
const realServices =
!!config.get('testRunner') ||
(testStats.testCount > 0 && testStats.nonSkippedTestCount > 0);
if (providers.hasService('es')) {
await this.validateEsVersion(config);
const providers = realServices
? new ProviderCollection(this.log, [
...coreProviders,
...readProviderSpec('Service', config.get('services')),
...readProviderSpec('PageObject', config.get('pageObjects')),
])
: this.getStubProviderCollection(config, coreProviders);
if (realServices) {
if (providers.hasService('es')) {
await this.validateEsVersion(config);
}
await providers.loadAll();
}
await providers.loadAll();
const customTestRunner = config.get('testRunner');
if (customTestRunner) {
@ -148,53 +159,65 @@ export class FunctionalTestRunner {
throw new Error('Unable to get test stats for config that uses a custom test runner');
}
// replace the function of custom service providers so that they return
// promise-like objects which never resolve, essentially disabling them
// allowing us to load the test files and populate the mocha suites
const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
readProviderSpec(type, providers).map((p) => ({
...p,
fn: skip.includes(p.name)
? (ctx: any) => {
const result = ProviderCollection.callProviderFn(p.fn, ctx);
if ('then' in result) {
throw new Error(
`Provider [${p.name}] returns a promise so it can't loaded during test analysis`
);
}
return result;
}
: () => ({
then: () => {},
}),
}));
const providers = new ProviderCollection(this.log, [
...coreProviders,
...readStubbedProviderSpec(
'Service',
config.get('services'),
config.get('servicesRequiredForTestAnalysis')
),
...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
]);
const providers = this.getStubProviderCollection(config, coreProviders);
const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
const countTests = (suite: Suite): number =>
suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length);
const queue = new Set([mocha.suite]);
const allTests: Test[] = [];
for (const suite of queue) {
for (const test of suite.tests) {
allTests.push(test);
}
for (const childSuite of suite.suites) {
queue.add(childSuite);
}
}
return {
testCount: countTests(mocha.suite),
testCount: allTests.length,
nonSkippedTestCount: allTests.filter((t) => !t.pending).length,
testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()),
};
});
}
private getStubProviderCollection(config: Config, coreProviders: Providers) {
// when we want to load the tests but not actually run anything we can
// use stubbed providers which allow mocha to do it's thing without taking
// too much time
const readStubbedProviderSpec = (type: string, providers: any, skip: string[]) =>
readProviderSpec(type, providers).map((p) => ({
...p,
fn: skip.includes(p.name)
? (ctx: any) => {
const result = ProviderCollection.callProviderFn(p.fn, ctx);
if ('then' in result) {
throw new Error(
`Provider [${p.name}] returns a promise so it can't loaded during test analysis`
);
}
return result;
}
: () => ({
then: () => {},
}),
}));
return new ProviderCollection(this.log, [
...coreProviders,
...readStubbedProviderSpec(
'Service',
config.get('services'),
config.get('servicesRequiredForTestAnalysis')
),
...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
]);
}
async _run<T = any>(
handler: (config: Config, coreProvider: ReturnType<typeof readProviderSpec>) => Promise<T>
handler: (config: Config, coreProviders: Providers) => Promise<T>
): Promise<T> {
let runErrorOccurred = false;
@ -205,7 +228,7 @@ export class FunctionalTestRunner {
this.configFile,
this.configOverrides
);
this.log.info('Config loaded');
this.log.debug('Config loaded');
if (
(!config.get('testFiles') || config.get('testFiles').length === 0) &&

32
packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js
View file

@ -12,6 +12,32 @@ import { createAssignmentProxy } from './assignment_proxy';
import { wrapFunction } from './wrap_function';
import { wrapRunnableArgs } from './wrap_runnable_args';
const allTestsSkippedCache = new WeakMap();
function allTestsAreSkipped(suite) {
// cache result for each suite so we don't have to traverse over and over
const cache = allTestsSkippedCache.get(suite);
if (cache) {
return cache;
}
// if this suite is skipped directly then all it's children are skipped
if (suite.pending) {
allTestsSkippedCache.set(suite, true);
return true;
}
// if any of this suites own tests are not skipped, then we don't need to traverse to child suites
if (suite.tests.some((t) => !t.pending)) {
allTestsSkippedCache.set(suite, false);
return false;
}
// otherwise traverse down through the child suites and return true only if all children are all skipped
const childrenSkipped = suite.suites.every(allTestsAreSkipped);
allTestsSkippedCache.set(suite, childrenSkipped);
return childrenSkipped;
}
export function decorateMochaUi(log, lifecycle, context, { rootTags }) {
// incremented at the start of each suite, decremented after
// so that in each non-suite call we can know if we are within
@ -71,6 +97,12 @@ export function decorateMochaUi(log, lifecycle, context, { rootTags }) {
provider.call(this);
if (allTestsAreSkipped(this)) {
// all the children in this suite are skipped, so make sure the suite is
// marked as pending so that its hooks are not run
this.pending = true;
}
after('afterTestSuite.trigger', async () => {
await lifecycle.afterTestSuite.trigger(this);
});

2
packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.ts
View file

@ -44,7 +44,7 @@ export function filterSuites({ log, mocha, include, exclude, esVersion }: Option
if (esVersion) {
// traverse the test graph and exclude any tests which don't meet their esVersionRequirement
log.info('Only running suites which are compatible with ES version', esVersion.toString());
log.debug('Only running suites which are compatible with ES version', esVersion.toString());
(function recurse(parentSuite: SuiteInternal) {
const children = parentSuite.suites;
parentSuite.suites = [];

7
packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/ci_stats_ftr_reporter.ts
View file

@ -69,6 +69,11 @@ export function setupCiStatsFtrTestGroupReporter({
testMetadata: TestMetadata;
reporter: CiStatsReporter;
}) {
const testGroupType = process.env.TEST_GROUP_TYPE_FUNCTIONAL;
if (!testGroupType) {
throw new Error('missing process.env.TEST_GROUP_TYPE_FUNCTIONAL');
}
let startMs: number | undefined;
runner.on('start', () => {
startMs = Date.now();
@ -78,7 +83,7 @@ export function setupCiStatsFtrTestGroupReporter({
const group: CiStatsReportTestsOptions['group'] = {
startTime: new Date(start).toJSON(),
durationMs: 0,
type: config.path.startsWith('x-pack') ? 'X-Pack Functional Tests' : 'Functional Tests',
type: testGroupType,
name: Path.relative(REPO_ROOT, config.path),
result: 'skip',
meta: {

2
packages/kbn-test/src/functional_test_runner/lib/providers/index.ts
View file

@ -7,6 +7,6 @@
*/
export { ProviderCollection } from './provider_collection';
export { readProviderSpec } from './read_provider_spec';
export * from './read_provider_spec';
export { createAsyncInstance } from './async_instance';
export type { Provider } from './read_provider_spec';

2
packages/kbn-test/src/functional_tests/lib/run_ftr.ts
View file

@ -122,5 +122,5 @@ export async function hasTests({ configPath, options }: CreateFtrParams) {
return true;
}
const stats = await ftr.getTestStats();
return stats.testCount > 0;
return stats.nonSkippedTestCount > 0;
}

27
packages/kbn-test/src/functional_tests/tasks.ts
View file

@ -63,7 +63,7 @@ interface RunTestsParams extends CreateFtrOptions {
assertNoneExcluded: boolean;
}
export async function runTests(options: RunTestsParams) {
if (!process.env.KBN_NP_PLUGINS_BUILT && !options.assertNoneExcluded) {
if (!process.env.CI && !options.assertNoneExcluded) {
const log = options.createLogger();
log.warning('❗️❗️❗️');
log.warning('❗️❗️❗️');
@ -91,21 +91,18 @@ export async function runTests(options: RunTestsParams) {
return;
}
log.write('--- determining which ftr configs to run');
const configPathsWithTests: string[] = [];
for (const configPath of options.configs) {
log.info('testing', relative(REPO_ROOT, configPath));
await log.indent(4, async () => {
if (await hasTests({ configPath, options: { ...options, log } })) {
configPathsWithTests.push(configPath);
}
});
}
for (const [i, configPath] of configPathsWithTests.entries()) {
for (const [i, configPath] of options.configs.entries()) {
await log.indent(0, async () => {
const progress = `${i + 1}/${configPathsWithTests.length}`;
log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
if (options.configs.length > 1) {
const progress = `${i + 1}/${options.configs.length}`;
log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
}
if (!(await hasTests({ configPath, options: { ...options, log } }))) {
// just run the FTR, no Kibana or ES, which will quickly report a skipped test group to ci-stats and continue
await runFtr({ configPath, options: { ...options, log } });
return;
}
await withProcRunner(log, async (procs) => {
const config = await readConfigFile(log, options.esVersion, configPath);

8
scripts/README.md
View file

@ -17,13 +17,13 @@ This directory is excluded from the build and tools within it should help users
## Functional Test Scripts
**`node scripts/functional_tests [--config test/functional/config.js --config test/api_integration/config.js]`**
**`node scripts/functional_tests [--config test/functional/config.base.js --config test/api_integration/config.js]`**
Runs all the functional tests: selenium tests and api integration tests. List configs with multiple `--config` arguments. Uses the [@kbn/test](../packages/kbn-test) library to run Elasticsearch and Kibana servers and tests against those servers, for multiple server+test setups. In particular, calls out to [`runTests()`](../packages/kbn-test/src/functional_tests/tasks.js). Can be run on a single config.
**`node scripts/functional_tests_server [--config test/functional/config.js]`**
**`node scripts/functional_tests_server [--config test/functional/config.base.js]`**
Starts just the Elasticsearch and Kibana servers given a single config, i.e. via `--config test/functional/config.js` or `--config test/api_integration/config`. Allows the user to start just the servers with this script, and keep them running while running tests against these servers. The idea is that the same config file configures both Elasticsearch and Kibana servers. Uses the [`startServers()`](../packages/kbn-test/src/functional_tests/tasks.js#L52-L80) method from [@kbn/test](../packages/kbn-test) library.
Starts just the Elasticsearch and Kibana servers given a single config, i.e. via `--config test/functional/config.base.js` or `--config test/api_integration/config`. Allows the user to start just the servers with this script, and keep them running while running tests against these servers. The idea is that the same config file configures both Elasticsearch and Kibana servers. Uses the [`startServers()`](../packages/kbn-test/src/functional_tests/tasks.js#L52-L80) method from [@kbn/test](../packages/kbn-test) library.
Example. Start servers _and_ run tests, separately, but using the same config:
@ -51,7 +51,7 @@ If you wish to load up specific es archived data for your test, you can do so vi
node scripts/es_archiver.js load <archive> [--es-url=http://username:password@localhost:9200] [--kibana-url=http://username:password@localhost:5601/{basepath?}]
```
That will load the specified archive located in the archive directory specified by the default functional config file, located in `test/functional/config.js`. To load archives from other function config files you can pass `--config path/to/config.js`.
That will load the specified archive located in the archive directory specified by the default functional config file, located in `test/functional/config.base.js`. To load archives from other function config files you can pass `--config path/to/config.js`.
*Note:* The `--es-url` and `--kibana-url` options may or may not be neccessary depending on your current Kibana configuration settings, and their values
may also change based on those settings (for example if you are not running with security you will not need the `username:password` portion).

24
scripts/functional_tests.js
View file

@ -7,26 +7,4 @@
*/
require('../src/setup_node_env');
require('@kbn/test').runTestsCli([
require.resolve('../test/functional/config.ccs.ts'),
require.resolve('../test/functional/config.js'),
require.resolve('../test/plugin_functional/config.ts'),
require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
require.resolve('../test/new_visualize_flow/config.ts'),
require.resolve('../test/interactive_setup_api_integration/enrollment_flow.config.ts'),
require.resolve('../test/interactive_setup_api_integration/manual_configuration_flow.config.ts'),
require.resolve(
'../test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts'
),
require.resolve('../test/interactive_setup_functional/enrollment_token.config.ts'),
require.resolve('../test/interactive_setup_functional/manual_configuration.config.ts'),
require.resolve(
'../test/interactive_setup_functional/manual_configuration_without_security.config.ts'
),
require.resolve(
'../test/interactive_setup_functional/manual_configuration_without_tls.config.ts'
),
require.resolve('../test/api_integration/config.js'),
require.resolve('../test/interpreter_functional/config.ts'),
require.resolve('../test/examples/config.js'),
]);
require('@kbn/test').runTestsCli();

2
scripts/functional_tests_server.js
View file

@ -7,4 +7,4 @@
*/
require('../src/setup_node_env');
require('@kbn/test').startServersCli(require.resolve('../test/functional/config.js'));
require('@kbn/test').startServersCli(require.resolve('../test/functional/config.base.js'));

2
test/accessibility/config.ts
View file

@ -11,7 +11,7 @@ import { services } from './services';
import { pageObjects } from './page_objects';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
return {
...functionalConfig.getAll(),

2
test/analytics/config.ts
View file

@ -19,7 +19,7 @@ import { services } from './services';
*/
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const commonConfig = await readConfigFile(require.resolve('../common/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
return {
testFiles: [require.resolve('./tests')],

3
test/api_integration/apis/saved_objects/lib/saved_objects_test_utils.ts
View file

@ -14,5 +14,6 @@ export async function getKibanaVersion(getService: FtrProviderContext['getServic
const kibanaVersion = await kibanaServer.version.get();
expect(typeof kibanaVersion).to.eql('string');
expect(kibanaVersion.length).to.be.greaterThan(0);
return kibanaVersion;
// mimic SavedObjectsService.stripVersionQualifier()
return kibanaVersion.split('-')[0];
}

2
test/api_integration/config.js
View file

@ -10,7 +10,7 @@ import { services } from './services';
export default async function ({ readConfigFile }) {
const commonConfig = await readConfigFile(require.resolve('../common/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
return {
rootTags: ['runOutsideOfCiGroups'],

2
test/examples/config.js
View file

@ -12,7 +12,7 @@ import fs from 'fs';
import { KIBANA_ROOT } from '@kbn/test';
export default async function ({ readConfigFile }) {
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
// Find all folders in /examples and /x-pack/examples since we treat all them as plugin folder
const examplesFiles = fs.readdirSync(resolve(KIBANA_ROOT, 'examples'));

17
test/functional/config.coverage.js → test/functional/apps/bundles/config.ts
View file

@ -6,18 +6,13 @@
* Side Public License, v 1.
*/
export default async function ({ readConfigFile }) {
const defaultConfig = await readConfigFile(require.resolve('./config'));
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...defaultConfig.getAll(),
suiteTags: {
exclude: ['skipCoverage'],
},
junit: {
reportName: 'Code Coverage for Functional Tests',
},
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/console/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/context/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

7
test/functional/apps/dashboard/README.md Normal file
View file

@ -0,0 +1,7 @@
# What are all these groups?
The tests take a while so they have been broken up into groups with their own `config.ts` and `index.ts` file, causing each of these groups to be independent bundles of tests which can be run on some worker in CI without taking an incredible amount of time.
Want to change the groups to something more logical? Have fun! Just make sure that each group executes on CI in less than 10 minutes or so. We don't currently have any mechanism for validating this right now, you just need to look at the times in the log output on CI, but we'll be working on tooling for making this information more accessible soon.
- Kibana Operations

18
test/functional/apps/dashboard/group1/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

2
test/functional/apps/dashboard/create_and_add_embeddables.ts → test/functional/apps/dashboard/group1/create_and_add_embeddables.ts
View file

@ -10,7 +10,7 @@ import expect from '@kbn/expect';
import { VisualizeConstants } from '@kbn/visualizations-plugin/common/constants';
import { VISUALIZE_ENABLE_LABS_SETTING } from '@kbn/visualizations-plugin/common/constants';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

2
test/functional/apps/dashboard/dashboard_back_button.ts → test/functional/apps/dashboard/group1/dashboard_back_button.ts
View file

@ -7,7 +7,7 @@
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');

2
test/functional/apps/dashboard/dashboard_error_handling.ts → test/functional/apps/dashboard/group1/dashboard_error_handling.ts
View file

@ -7,7 +7,7 @@
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'common']);

2
test/functional/apps/dashboard/dashboard_options.ts → test/functional/apps/dashboard/group1/dashboard_options.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

2
test/functional/apps/dashboard/dashboard_query_bar.ts → test/functional/apps/dashboard/group1/dashboard_query_bar.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');

2
test/functional/apps/dashboard/dashboard_unsaved_listing.ts → test/functional/apps/dashboard/group1/dashboard_unsaved_listing.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']);

2
test/functional/apps/dashboard/dashboard_unsaved_state.ts → test/functional/apps/dashboard/group1/dashboard_unsaved_state.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']);

2
test/functional/apps/dashboard/data_shared_attributes.ts → test/functional/apps/dashboard/group1/data_shared_attributes.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

2
test/functional/apps/dashboard/edit_embeddable_redirects.ts → test/functional/apps/dashboard/group1/edit_embeddable_redirects.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']);

0
test/functional/apps/dashboard/edit_visualizations.js → test/functional/apps/dashboard/group1/edit_visualizations.js
View file

2
test/functional/apps/dashboard/embed_mode.ts → test/functional/apps/dashboard/group1/embed_mode.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');

2
test/functional/apps/dashboard/embeddable_data_grid.ts → test/functional/apps/dashboard/group1/embeddable_data_grid.ts
View file

@ -7,7 +7,7 @@
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dashboardAddPanel = getService('dashboardAddPanel');

2
test/functional/apps/dashboard/embeddable_rendering.ts → test/functional/apps/dashboard/group1/embeddable_rendering.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
/**
* This tests both that one of each visualization can be added to a dashboard (as opposed to opening an existing

2
test/functional/apps/dashboard/empty_dashboard.ts → test/functional/apps/dashboard/group1/empty_dashboard.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');

54
test/functional/apps/dashboard/group1/index.ts Normal file
View file

@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
async function loadCurrentData() {
await browser.setWindowSize(1300, 900);
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/dashboard/current/data');
}
async function unloadCurrentData() {
await esArchiver.unload('test/functional/fixtures/es_archiver/dashboard/current/data');
}
describe('dashboard app - group 1', function () {
before(loadCurrentData);
after(unloadCurrentData);
// This has to be first since the other tests create some embeddables as side affects and our counting assumes
// a fresh index.
loadTestFile(require.resolve('./empty_dashboard'));
loadTestFile(require.resolve('./url_field_formatter'));
loadTestFile(require.resolve('./embeddable_rendering'));
loadTestFile(require.resolve('./embeddable_data_grid'));
loadTestFile(require.resolve('./create_and_add_embeddables'));
loadTestFile(require.resolve('./edit_embeddable_redirects'));
loadTestFile(require.resolve('./dashboard_unsaved_state'));
loadTestFile(require.resolve('./dashboard_unsaved_listing'));
loadTestFile(require.resolve('./edit_visualizations'));
loadTestFile(require.resolve('./dashboard_options'));
loadTestFile(require.resolve('./data_shared_attributes'));
loadTestFile(require.resolve('./share'));
loadTestFile(require.resolve('./embed_mode'));
loadTestFile(require.resolve('./dashboard_back_button'));
loadTestFile(require.resolve('./dashboard_error_handling'));
loadTestFile(require.resolve('./legacy_urls'));
loadTestFile(require.resolve('./saved_search_embeddable'));
// Note: This one must be last because it unloads some data for one of its tests!
// No, this isn't ideal, but loading/unloading takes so much time and these are all bunched
// to improve efficiency...
loadTestFile(require.resolve('./dashboard_query_bar'));
});
}

2
test/functional/apps/dashboard/legacy_urls.ts → test/functional/apps/dashboard/group1/legacy_urls.ts
View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects([

2
test/functional/apps/dashboard/saved_search_embeddable.ts → test/functional/apps/dashboard/group1/saved_search_embeddable.ts
View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dashboardAddPanel = getService('dashboardAddPanel');

2
test/functional/apps/dashboard/share.ts → test/functional/apps/dashboard/group1/share.ts
View file

@ -7,7 +7,7 @@
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');

4
test/functional/apps/dashboard/url_field_formatter.ts → test/functional/apps/dashboard/group1/url_field_formatter.ts
View file

@ -7,8 +7,8 @@
*/
import expect from '@kbn/expect';
import { WebElementWrapper } from '../../services/lib/web_element_wrapper';
import { FtrProviderContext } from '../../ftr_provider_context';
import { WebElementWrapper } from '../../../services/lib/web_element_wrapper';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const { common, dashboard, settings, timePicker, visChart } = getPageObjects([

18
test/functional/apps/dashboard/group2/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

2
test/functional/apps/dashboard/dashboard_filter_bar.ts → test/functional/apps/dashboard/group2/dashboard_filter_bar.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dataGrid = getService('dataGrid');

2
test/functional/apps/dashboard/dashboard_filtering.ts → test/functional/apps/dashboard/group2/dashboard_filtering.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
/**
* Test the querying capabilities of dashboard, and make sure visualizations show the expected results, especially

2
test/functional/apps/dashboard/dashboard_grid.ts → test/functional/apps/dashboard/group2/dashboard_grid.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

2
test/functional/apps/dashboard/dashboard_saved_query.ts → test/functional/apps/dashboard/group2/dashboard_saved_query.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');

2
test/functional/apps/dashboard/dashboard_snapshots.ts → test/functional/apps/dashboard/group2/dashboard_snapshots.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({
getService,

2
test/functional/apps/dashboard/embeddable_library.ts → test/functional/apps/dashboard/group2/embeddable_library.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']);

2
test/functional/apps/dashboard/full_screen_mode.ts → test/functional/apps/dashboard/group2/full_screen_mode.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

43
test/functional/apps/dashboard/group2/index.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
async function loadCurrentData() {
await browser.setWindowSize(1300, 900);
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/dashboard/current/data');
}
async function unloadCurrentData() {
await esArchiver.unload('test/functional/fixtures/es_archiver/dashboard/current/data');
}
describe('dashboard app - group 2', function () {
before(loadCurrentData);
after(unloadCurrentData);
loadTestFile(require.resolve('./full_screen_mode'));
loadTestFile(require.resolve('./dashboard_filter_bar'));
loadTestFile(require.resolve('./dashboard_filtering'));
loadTestFile(require.resolve('./panel_expand_toggle'));
loadTestFile(require.resolve('./dashboard_grid'));
loadTestFile(require.resolve('./view_edit'));
loadTestFile(require.resolve('./dashboard_saved_query'));
// Order of test suites *shouldn't* be important but there's a bug for the view_edit test above
// https://github.com/elastic/kibana/issues/46752
// The dashboard_snapshot test below requires the timestamped URL which breaks the view_edit test.
// If we don't use the timestamp in the URL, the colors in the charts will be different.
loadTestFile(require.resolve('./dashboard_snapshots'));
loadTestFile(require.resolve('./embeddable_library'));
});
}

2
test/functional/apps/dashboard/panel_expand_toggle.ts → test/functional/apps/dashboard/group2/panel_expand_toggle.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

2
test/functional/apps/dashboard/view_edit.ts → test/functional/apps/dashboard/group2/view_edit.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const queryBar = getService('queryBar');

2
test/functional/apps/dashboard/bwc_shared_urls.ts → test/functional/apps/dashboard/group3/bwc_shared_urls.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header']);

18
test/functional/apps/dashboard/group3/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

2
test/functional/apps/dashboard/copy_panel_to.ts → test/functional/apps/dashboard/group3/copy_panel_to.ts
View file

@ -7,7 +7,7 @@
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dashboardVisualizations = getService('dashboardVisualizations');

4
test/functional/apps/dashboard/dashboard_state.ts → test/functional/apps/dashboard/group3/dashboard_state.ts
View file

@ -10,8 +10,8 @@ import expect from '@kbn/expect';
import chroma from 'chroma-js';
import { DEFAULT_PANEL_WIDTH } from '@kbn/dashboard-plugin/public/application/embeddable/dashboard_constants';
import { PIE_CHART_VIS_NAME, AREA_CHART_VIS_NAME } from '../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../ftr_provider_context';
import { PIE_CHART_VIS_NAME, AREA_CHART_VIS_NAME } from '../../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects([

4
test/functional/apps/dashboard/dashboard_time_picker.ts → test/functional/apps/dashboard/group3/dashboard_time_picker.ts
View file

@ -8,8 +8,8 @@
import expect from '@kbn/expect';
import { PIE_CHART_VIS_NAME } from '../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../ftr_provider_context';
import { PIE_CHART_VIS_NAME } from '../../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dashboardExpect = getService('dashboardExpect');

37
test/functional/apps/dashboard/group3/index.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
async function loadLogstash() {
await browser.setWindowSize(1200, 900);
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
}
async function unloadLogstash() {
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
}
describe('dashboard app', function () {
this.tags('ciGroup4');
before(loadLogstash);
after(unloadLogstash);
loadTestFile(require.resolve('./dashboard_time_picker'));
loadTestFile(require.resolve('./bwc_shared_urls'));
loadTestFile(require.resolve('./panel_replacing'));
loadTestFile(require.resolve('./panel_cloning'));
loadTestFile(require.resolve('./copy_panel_to'));
loadTestFile(require.resolve('./panel_context_menu'));
loadTestFile(require.resolve('./dashboard_state'));
});
}

4
test/functional/apps/dashboard/panel_cloning.ts → test/functional/apps/dashboard/group3/panel_cloning.ts
View file

@ -7,8 +7,8 @@
*/
import expect from '@kbn/expect';
import { PIE_CHART_VIS_NAME } from '../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../ftr_provider_context';
import { PIE_CHART_VIS_NAME } from '../../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const dashboardPanelActions = getService('dashboardPanelActions');

4
test/functional/apps/dashboard/panel_context_menu.ts → test/functional/apps/dashboard/group3/panel_context_menu.ts
View file

@ -8,8 +8,8 @@
import expect from '@kbn/expect';
import { VisualizeConstants } from '@kbn/visualizations-plugin/common/constants';
import { PIE_CHART_VIS_NAME } from '../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../ftr_provider_context';
import { PIE_CHART_VIS_NAME } from '../../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

4
test/functional/apps/dashboard/panel_replacing.ts → test/functional/apps/dashboard/group3/panel_replacing.ts
View file

@ -11,8 +11,8 @@ import {
PIE_CHART_VIS_NAME,
AREA_CHART_VIS_NAME,
LINE_CHART_VIS_NAME,
} from '../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../ftr_provider_context';
} from '../../../page_objects/dashboard_page';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

18
test/functional/apps/dashboard/group4/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

2
test/functional/apps/dashboard/dashboard_clone.ts → test/functional/apps/dashboard/group4/dashboard_clone.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');

2
test/functional/apps/dashboard/dashboard_listing.ts → test/functional/apps/dashboard/group4/dashboard_listing.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'common']);

2
test/functional/apps/dashboard/dashboard_save.ts → test/functional/apps/dashboard/group4/dashboard_save.ts
View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize']);

2
test/functional/apps/dashboard/dashboard_time.ts → test/functional/apps/dashboard/group4/dashboard_time.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
const dashboardName = 'Dashboard Test Time';

34
test/functional/apps/dashboard/group4/index.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
async function loadLogstash() {
await browser.setWindowSize(1200, 900);
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
}
async function unloadLogstash() {
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
}
describe('dashboard app - group 4', function () {
this.tags('ciGroup5');
before(loadLogstash);
after(unloadLogstash);
loadTestFile(require.resolve('./dashboard_save'));
loadTestFile(require.resolve('./dashboard_time'));
loadTestFile(require.resolve('./dashboard_listing'));
loadTestFile(require.resolve('./dashboard_clone'));
});
}

18
test/functional/apps/dashboard/group5/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

50
test/functional/apps/dashboard/group5/index.ts Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
async function loadLogstash() {
await browser.setWindowSize(1200, 900);
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
}
async function unloadLogstash() {
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
}
describe('dashboard app', function () {
// TODO: Remove when vislib is removed
// https://github.com/elastic/kibana/issues/56143
describe('new charts library', function () {
this.tags('ciGroup5');
before(async () => {
await loadLogstash();
await kibanaServer.uiSettings.update({
'visualization:visualize:legacyPieChartsLibrary': false,
});
await browser.refresh();
});
after(async () => {
await unloadLogstash();
await kibanaServer.uiSettings.update({
'visualization:visualize:legacyPieChartsLibrary': true,
});
await browser.refresh();
});
loadTestFile(require.resolve('../group3/dashboard_state'));
});
});
}

139
test/functional/apps/dashboard/index.ts
View file

@ -1,139 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
async function loadCurrentData() {
await browser.setWindowSize(1300, 900);
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/dashboard/current/data');
}
async function unloadCurrentData() {
await esArchiver.unload('test/functional/fixtures/es_archiver/dashboard/current/data');
}
async function loadLogstash() {
await browser.setWindowSize(1200, 900);
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
}
async function unloadLogstash() {
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
}
describe('dashboard app', function () {
// This has to be first since the other tests create some embeddables as side affects and our counting assumes
// a fresh index.
describe('using current data', function () {
this.tags('ciGroup2');
before(loadCurrentData);
after(unloadCurrentData);
loadTestFile(require.resolve('./empty_dashboard'));
loadTestFile(require.resolve('./url_field_formatter'));
loadTestFile(require.resolve('./embeddable_rendering'));
loadTestFile(require.resolve('./embeddable_data_grid'));
loadTestFile(require.resolve('./create_and_add_embeddables'));
loadTestFile(require.resolve('./edit_embeddable_redirects'));
loadTestFile(require.resolve('./dashboard_unsaved_state'));
loadTestFile(require.resolve('./dashboard_unsaved_listing'));
loadTestFile(require.resolve('./edit_visualizations'));
loadTestFile(require.resolve('./dashboard_options'));
loadTestFile(require.resolve('./data_shared_attributes'));
loadTestFile(require.resolve('./share'));
loadTestFile(require.resolve('./embed_mode'));
loadTestFile(require.resolve('./dashboard_back_button'));
loadTestFile(require.resolve('./dashboard_error_handling'));
loadTestFile(require.resolve('./legacy_urls'));
loadTestFile(require.resolve('./saved_search_embeddable'));
// Note: This one must be last because it unloads some data for one of its tests!
// No, this isn't ideal, but loading/unloading takes so much time and these are all bunched
// to improve efficiency...
loadTestFile(require.resolve('./dashboard_query_bar'));
});
describe('using current data', function () {
this.tags('ciGroup3');
before(loadCurrentData);
after(unloadCurrentData);
loadTestFile(require.resolve('./full_screen_mode'));
loadTestFile(require.resolve('./dashboard_filter_bar'));
loadTestFile(require.resolve('./dashboard_filtering'));
loadTestFile(require.resolve('./panel_expand_toggle'));
loadTestFile(require.resolve('./dashboard_grid'));
loadTestFile(require.resolve('./view_edit'));
loadTestFile(require.resolve('./dashboard_saved_query'));
// Order of test suites *shouldn't* be important but there's a bug for the view_edit test above
// https://github.com/elastic/kibana/issues/46752
// The dashboard_snapshot test below requires the timestamped URL which breaks the view_edit test.
// If we don't use the timestamp in the URL, the colors in the charts will be different.
loadTestFile(require.resolve('./dashboard_snapshots'));
loadTestFile(require.resolve('./embeddable_library'));
});
// Each of these tests call initTests themselves, the way it was originally written. The above tests only load
// the data once to save on time. Eventually, all of these tests should just use current data and we can reserve
// legacy data only for specifically testing BWC situations.
describe('using legacy data', function () {
this.tags('ciGroup4');
before(loadLogstash);
after(unloadLogstash);
loadTestFile(require.resolve('./dashboard_time_picker'));
loadTestFile(require.resolve('./bwc_shared_urls'));
loadTestFile(require.resolve('./panel_replacing'));
loadTestFile(require.resolve('./panel_cloning'));
loadTestFile(require.resolve('./copy_panel_to'));
loadTestFile(require.resolve('./panel_context_menu'));
loadTestFile(require.resolve('./dashboard_state'));
});
describe('using legacy data', function () {
this.tags('ciGroup5');
before(loadLogstash);
after(unloadLogstash);
loadTestFile(require.resolve('./dashboard_save'));
loadTestFile(require.resolve('./dashboard_time'));
loadTestFile(require.resolve('./dashboard_listing'));
loadTestFile(require.resolve('./dashboard_clone'));
});
// TODO: Remove when vislib is removed
// https://github.com/elastic/kibana/issues/56143
describe('new charts library', function () {
this.tags('ciGroup5');
before(async () => {
await loadLogstash();
await kibanaServer.uiSettings.update({
'visualization:visualize:legacyPieChartsLibrary': false,
});
await browser.refresh();
});
after(async () => {
await unloadLogstash();
await kibanaServer.uiSettings.update({
'visualization:visualize:legacyPieChartsLibrary': true,
});
await browser.refresh();
});
loadTestFile(require.resolve('./dashboard_state'));
});
});
}

18
test/functional/apps/dashboard_elements/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/discover/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/getting_started/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/home/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/management/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/saved_objects_management/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

18
test/functional/apps/status_page/config.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

7
test/functional/apps/visualize/README.md Normal file
View file

@ -0,0 +1,7 @@
# What are all these groups?
The tests take a while so they have been broken up into groups with their own `config.ts` and `index.ts` file, causing each of these groups to be independent bundles of tests which can be run on some worker in CI without taking an incredible amount of time.
Want to change the groups to something more logical? Have fun! Just make sure that each group executes on CI in less than 10 minutes or so. We don't currently have any mechanism for validating this right now, you just need to look at the times in the log output on CI, but we'll be working on tooling for making this information more accessible soon.
- Kibana Operations

Some files were not shown because too many files have changed in this diff Show more