mirror of
https://github.com/elastic/logstash.git
synced 2025-04-19 04:15:23 -04:00
3e0f488df2
* tests: make integration split quantity configurable Refactors shared splitter bash function to take a list of files on stdin and split into a configurable number of partitions, emitting only those from the currently-selected partition to stdout. Also refactors the only caller in the integration_tests launcher script to accept an optional partition_count parameter (defaulting to `2` for backward- compatibility), to provide the list of specs to the function's stdin, and to output relevant information about the quantity of partition splits and which was selected. * ci: run integration tests in 3 parts
243 lines
7.2 KiB
YAML
243 lines
7.2 KiB
YAML
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
|
|
|
|
steps:
|
|
- label: ":passport_control: License check"
|
|
key: "license-check"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
|
|
cpu: "4"
|
|
memory: "6Gi"
|
|
ephemeralStorage: "100Gi"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
export JRUBY_OPTS="-J-Xmx1g"
|
|
export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
|
|
ci/license_check.sh -m 4G
|
|
|
|
- label: ":rspec: Ruby unit tests"
|
|
key: "ruby-unit-tests"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "4"
|
|
memory: "8Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as a non-root user
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
ci/unit_tests.sh ruby
|
|
artifact_paths:
|
|
- "coverage/coverage.json"
|
|
|
|
- label: ":java: Java unit tests"
|
|
key: "java-unit-tests"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
env:
|
|
ENABLE_SONARQUBE: true
|
|
command: |
|
|
set -euo pipefail
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
ci/unit_tests.sh java
|
|
artifact_paths:
|
|
- "**/build/test-results/javaTests/TEST-*.xml"
|
|
- "**/jacocoTestReport.xml"
|
|
- "**/build/classes/**/*.*"
|
|
|
|
- label: ":sonarqube: Continuous Code Inspection"
|
|
if: |
|
|
build.pull_request.id != null ||
|
|
build.branch == "main" ||
|
|
build.branch =~ /^[0-9]+\.[0-9]+\$/
|
|
env:
|
|
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/logstash/sonar-analyze-token"
|
|
agents:
|
|
image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
|
|
command:
|
|
- "buildkite-agent artifact download --step ruby-unit-tests coverage/coverage.json ."
|
|
- "buildkite-agent artifact download --step java-unit-tests **/jacocoTestReport.xml ."
|
|
- "buildkite-agent artifact download --step java-unit-tests **/build/classes/**/*.* ."
|
|
- "/scan-source-code.sh"
|
|
depends_on:
|
|
- "ruby-unit-tests"
|
|
- "java-unit-tests"
|
|
retry:
|
|
manual:
|
|
allowed: true
|
|
|
|
- label: ":lab_coat: Integration Tests / part 1-of-3"
|
|
key: "integration-tests-part-1-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as a non-root user
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
ci/integration_tests.sh split 0 3
|
|
|
|
- label: ":lab_coat: Integration Tests / part 2-of-3"
|
|
key: "integration-tests-part-2-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as a non-root user
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
ci/integration_tests.sh split 1 3
|
|
|
|
- label: ":lab_coat: Integration Tests / part 3-of-3"
|
|
key: "integration-tests-part-3-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as a non-root user
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
ci/integration_tests.sh split 2 3
|
|
|
|
- label: ":lab_coat: IT Persistent Queues / part 1-of-3"
|
|
key: "integration-tests-qa-part-1-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as non root (logstash) user. UID is hardcoded in image.
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
export FEATURE_FLAG=persistent_queues
|
|
ci/integration_tests.sh split 0 3
|
|
|
|
- label: ":lab_coat: IT Persistent Queues / part 2-of-3"
|
|
key: "integration-tests-qa-part-2-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as non root (logstash) user. UID is hardcoded in image.
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
export FEATURE_FLAG=persistent_queues
|
|
ci/integration_tests.sh split 1 3
|
|
|
|
- label: ":lab_coat: IT Persistent Queues / part 3-of-3"
|
|
key: "integration-tests-qa-part-3-of-3"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as non root (logstash) user. UID is hardcoded in image.
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
export FEATURE_FLAG=persistent_queues
|
|
ci/integration_tests.sh split 2 3
|
|
|
|
- label: ":lab_coat: x-pack unit tests"
|
|
key: "x-pack-unit-tests"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as non root (logstash) user. UID is hardcoded in image.
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
x-pack/ci/unit_tests.sh
|
|
|
|
- label: ":lab_coat: x-pack integration"
|
|
key: "integration-tests-x-pack"
|
|
agents:
|
|
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
ephemeralStorage: "100Gi"
|
|
# Run as non root (logstash) user. UID is hardcoded in image.
|
|
imageUID: "1002"
|
|
retry:
|
|
automatic:
|
|
- limit: 3
|
|
command: |
|
|
set -euo pipefail
|
|
|
|
source .buildkite/scripts/common/container-agent.sh
|
|
x-pack/ci/integration_tests.sh
|
|
|
|
- wait: ~
|
|
continue_on_failure: true
|
|
|
|
- label: "🏁 Annotate JUnit results"
|
|
# the plugin requires docker run, hence the use of a VM
|
|
agents:
|
|
provider: gcp
|
|
imageProject: elastic-images-prod
|
|
image: family/platform-ingest-logstash-ubuntu-2204
|
|
machineType: "n2-standard-2"
|
|
plugins:
|
|
- junit-annotate#v2.4.1:
|
|
artifacts: "**/TEST-*.xml"
|