github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ci] Merge test_serverless into ftr configs (#159607)

Browse source 
This moves all FTR and cypress based serverless tests into the default
test pipeline.  By moving these tests, failures will
now fail pull request and on-merge pipelines.
This commit is contained in:
Jon 2023-09-13 04:15:55 -05:00 • committed by GitHub
parent 15d4e42a96
commit 09e8e9a9f2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 50 additions and 322 deletions
Download patch file Download diff file Expand all files Collapse all files

30
.buildkite/ftr_configs.yml
View file

@ -79,25 +79,12 @@ disabled:
- x-pack/test_serverless/functional/config.base.ts
- x-pack/test_serverless/shared/config.base.ts
# Serverless configs, currently only for manual tests runs, CI integration planned
- x-pack/test_serverless/api_integration/test_suites/common/config.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/common/config.ts
- x-pack/test_serverless/functional/test_suites/observability/config.ts
# Serverless feature flag config files (move to enabled after tests are added)
- x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
- x-pack/test_serverless/functional/test_suites/search/config.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/search/config.examples.ts
- x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
- x-pack/test_serverless/functional/test_suites/security/config.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/security/config.examples.ts
defaultQueue: 'n2-4-spot'
enabled:
@ -411,6 +398,17 @@ enabled:
- x-pack/test/ui_capabilities/spaces_only/config.ts
- x-pack/test/upgrade_assistant_integration/config.js
- x-pack/test/usage_collection/config.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.ts
- x-pack/test_serverless/functional/test_suites/observability/config.ts
- x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/search/config.ts
- x-pack/test_serverless/functional/test_suites/search/config.examples.ts
- x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
- x-pack/test_serverless/functional/test_suites/security/config.ts
- x-pack/test_serverless/functional/test_suites/security/config.examples.ts
- x-pack/performance/journeys/ecommerce_dashboard.ts
- x-pack/performance/journeys/ecommerce_dashboard_map_only.ts
- x-pack/performance/journeys/flight_dashboard.ts

44
.buildkite/pipelines/on_merge.yml
View file

@ -78,6 +78,42 @@ steps:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless.sh
label: 'Serverless Security Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
label: 'Serverless Explore - Security Solution Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
label: 'Serverless Investigations - Security Solution Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
label: Trigger unsupported ftr tests
timeout_in_minutes: 10
@ -86,14 +122,6 @@ steps:
agents:
queue: 'kibana-default'
- command: '.buildkite/scripts/steps/functional/on_merge_serverless_ftrs.sh'
label: Trigger serverless ftr tests
timeout_in_minutes: 10
depends_on:
- build
agents:
queue: 'kibana-default'
- command: .buildkite/scripts/steps/lint.sh
label: 'Linting'
agents:

82
.buildkite/pipelines/pull_request/base.yml
View file

@ -56,84 +56,6 @@ steps:
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Observability Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Observability Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Search Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Search Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Security Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Security Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
soft_fail:
- exit_status: 10
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: .buildkite/scripts/steps/functional/security_serverless.sh
label: 'Serverless Security Cypress Tests'
agents:
@ -141,7 +63,6 @@ steps:
depends_on: build
timeout_in_minutes: 40
parallelism: 2
soft_fail: true
retry:
automatic:
- exit_status: '*'
@ -154,7 +75,6 @@ steps:
# queue: n2-4-spot
# depends_on: build
# timeout_in_minutes: 40
# soft_fail: true
# retry:
# automatic:
# - exit_status: '*'
@ -167,7 +87,6 @@ steps:
depends_on: build
timeout_in_minutes: 40
parallelism: 2
soft_fail: true
retry:
automatic:
- exit_status: '*'
@ -180,7 +99,6 @@ steps:
depends_on: build
timeout_in_minutes: 40
parallelism: 2
soft_fail: true
retry:
automatic:
- exit_status: '*'

137
.buildkite/pipelines/serverless.yml
View file

@ -1,137 +0,0 @@
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build
timeout_in_minutes: 10
agents:
queue: kibana-default
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/build_kibana.sh
label: Build Kibana Distribution and Plugins
agents:
queue: n2-16-spot
key: build
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
timeout_in_minutes: 60
retry:
automatic:
- exit_status: '-1'
limit: 3
- command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Observability Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Observability Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Search Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Search Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Security Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
label: 'Serverless Security Examples Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless.sh
label: 'Serverless Security Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 40
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
label: 'Serverless Security Investigations Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
label: 'Serverless Security Explore Cypress Tests'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 60
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1

8
.buildkite/scripts/steps/functional/on_merge_serverless_ftrs.sh
View file

@ -1,8 +0,0 @@
#!/bin/bash
set -euo pipefail
if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
echo "--- Trigger serverless ftr tests"
ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-serverless "$BUILDKITE_BRANCH" "$BUILDKITE_COMMIT" "$BUILDKITE_BUILD_ID"
fi

71
.buildkite/scripts/steps/functional/serverless_ftr.sh
View file

@ -1,71 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT"
if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/search/config.ts"
"x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/search/config.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "search.examples" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/functional/test_suites/search/config.examples.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "observability" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/observability/config.ts"
"x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/observability/config.ts"
"x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "observability.examples" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/functional/test_suites/observability/config.examples.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "security" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/security/config.ts"
"x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/security/config.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "security.examples" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/functional/test_suites/security/config.examples.ts"
)
fi
EXIT_CODE=0
OFFENDING_CONFIG=
for CONFIG in "${SERVERLESS_CONFIGS[@]}"
do
echo "--- $ node scripts/functional_tests --bail --config $CONFIG"
set +e;
node ./scripts/functional_tests \
--bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config="$CONFIG"
LAST_CODE=$?
set -e;
if [ $LAST_CODE -ne 0 ]; then
EXIT_CODE=10
OFFENDING_CONFIG=$CONFIG
fi
done
echo "--- Serverless FTR Results for $JOB"
if [ $EXIT_CODE -eq 0 ]; then
echo "✅ Success!"
elif [ $EXIT_CODE -eq 10 ]; then
echo "❌ Failed in config: $OFFENDING_CONFIG, exit code set to 10 for soft-failure"
else
echo "❌ Failed."
fi
exit $EXIT_CODE