github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 22:57:16 -04:00
Code Issues Projects Releases Packages Wiki Activity

Manually backport #14654, #14657, #14659 (#14665)

Browse source 
* DRA: Improve shell scripts for debuggability (#14654). The changes remove some code duplication by introducing a common file that can be sourced between all scripts. It also improves debuggability by adding better messages.
* Fix dra_common sourcing (#14657). Fixes the source of dra_common.sh. It will now first check the directory of the file from which this dra_common.sh script is being called. This allows the common script to be sourced regardless of where the sourcing script is being called from.
* Fix sourcing on dra_upload (#14659). Fix sourcing on dra_upload.sh
This commit is contained in:
Andres Rodriguez 2022-10-17 18:30:18 -04:00 • committed by GitHub
parent a441638474
commit 5c30a394af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 104 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

47
ci/dra_aarch64.sh
View file

@ -1,20 +1,15 @@
#!/bin/bash -ie
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
echo "####################################################################"
echo "##################### Starting $0"
echo "####################################################################"
# Since we are using the system jruby, we need to make sure our jvm process
# uses at least 1g of memory, If we don't do this we can get OOM issues when
# installing gems. See https://github.com/elastic/logstash/issues/5179
export JRUBY_OPTS="-J-Xmx1g"
# Extract the version number from the version.yml file
# e.g.: 8.6.0
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
source ./$(dirname "$0")/dra_common.sh
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
case "$WORKFLOW_TYPE" in
snapshot)
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
rake artifact:docker
rake artifact:docker_oss
@ -28,10 +23,11 @@ case "$WORKFLOW_TYPE" in
# e.g: 8.0.0-alpha1
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
fi
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
;;
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
;;
staging)
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
RELEASE=1 rake artifact:docker
RELEASE=1 rake artifact:docker_oss
@ -45,22 +41,25 @@ case "$WORKFLOW_TYPE" in
# e.g: 8.0.0-alpha1
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
fi
;;
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
;;
*)
exit 1
;;
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
;;
esac
echo "Saving tar.gz for docker images"
docker save docker.elastic.co/logstash/logstash:${STACK_VERSION} | gzip -c > build/logstash-${STACK_VERSION}-docker-image-aarch64.tar.gz
docker save docker.elastic.co/logstash/logstash-oss:${STACK_VERSION} | gzip -c > build/logstash-oss-${STACK_VERSION}-docker-image-aarch64.tar.gz
docker save docker.elastic.co/logstash/logstash-ubi8:${STACK_VERSION} | gzip -c > build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
info "Saving tar.gz for docker images"
save_docker_tarballs "aarch64" "${STACK_VERSION}"
echo "GENERATED ARTIFACTS"
info "GENERATED ARTIFACTS"
for file in build/logstash-*; do shasum $file;done
echo "UPLOADING TO INTERMEDIATE BUCKET"
info "UPLOADING TO INTERMEDIATE BUCKET"
# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the dra_x86_64.sh
gsutil cp build/logstash-${STACK_VERSION}-docker-image-aarch64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
gsutil cp build/logstash-oss-${STACK_VERSION}-docker-image-aarch64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
gsutil cp build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
for image in logstash logstash-oss logstash-ubi8; do
upload_to_bucket "build/$image-${STACK_VERSION}-docker-image-aarch64.tar.gz" ${STACK_VERSION}
done
echo "####################################################################"
echo "##################### Finishing $0"
echo "####################################################################"

32
ci/dra_common.sh Executable file
View file

@ -0,0 +1,32 @@
function info {
echo "INFO: $1"
}
function error {
echo "ERROR: $1"
exit 1
}
function save_docker_tarballs {
local arch="${1:?architecture required}"
local version="${2:?stack-version required}"
for image in logstash logstash-oss logstash-ubi8; do
docker save "docker.elastic.co/logstash/${image}:${version}" | gzip -c > "build/${image}-${version}-docker-image-${arch}.tar.gz"
done
}
function upload_to_bucket {
local file="${1:?file required}"
local version="${2:?stack-version required}"
gsutil cp "${file}" "gs://logstash-ci-artifacts/dra/${version}/"
}
# Since we are using the system jruby, we need to make sure our jvm process
# uses at least 1g of memory, If we don't do this we can get OOM issues when
# installing gems. See https://github.com/elastic/logstash/issues/5179
export JRUBY_OPTS="-J-Xmx1g"
# Extract the version number from the version.yml file
# e.g.: 8.6.0
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`

34
ci/dra_upload.sh
View file

@ -1,15 +1,10 @@
#!/bin/bash -ie
#!/bin/bash -iex
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
echo "####################################################################"
echo "##################### Starting $0"
echo "####################################################################"
# Since we are using the system jruby, we need to make sure our jvm process
# uses at least 1g of memory, If we don't do this we can get OOM issues when
# installing gems. See https://github.com/elastic/logstash/issues/5179
export JRUBY_OPTS="-J-Xmx1g"
# Extract the version number from the version.yml file
# e.g.: 8.6.0
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
source ./$(dirname "$0")/dra_common.sh
PLAIN_STACK_VERSION=$STACK_VERSION
# This is the branch selector that needs to be passed to the release-manager
@ -32,12 +27,17 @@ fi
case "$WORKFLOW_TYPE" in
snapshot)
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
;;
;;
staging)
;;
*)
error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
;;
esac
echo "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
echo "Download all the artifacts for version ${STACK_VERSION}"
info "Download all the artifacts for version ${STACK_VERSION}"
mkdir build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-no-jdk.deb build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}.csv build/
@ -70,9 +70,11 @@ gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-oss-${STACK_V
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-arm64.deb build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-aarch64.rpm build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-linux-aarch64.tar.gz build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-darwin-aarch64.tar.gz build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-oss-${STACK_VERSION}-arm64.deb build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-oss-${STACK_VERSION}-aarch64.rpm build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-oss-${STACK_VERSION}-linux-aarch64.tar.gz build/
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-oss-${STACK_VERSION}-darwin-aarch64.tar.gz build/
# docker
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}-docker-build-context.tar.gz build/
@ -92,7 +94,7 @@ gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-ubi8-${STACK_
gsutil cp gs://logstash-ci-artifacts/dra/${STACK_VERSION}/logstash-${STACK_VERSION}.csv build/
echo "Downloaded ARTIFACTS"
info "Downloaded ARTIFACTS"
for file in build/logstash-*; do shasum $file;done
mkdir -p build/distributions/dependencies-reports/
@ -123,3 +125,7 @@ docker run --rm \
--workflow "${WORKFLOW_TYPE}" \
--version "${PLAIN_STACK_VERSION}" \
--artifact-set main
echo "####################################################################"
echo "##################### Finishing $0"
echo "####################################################################"

56
ci/dra_x86_64.sh
View file

@ -1,20 +1,15 @@
#!/bin/bash -ie
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
echo "####################################################################"
echo "##################### Starting $0"
echo "####################################################################"
# Since we are using the system jruby, we need to make sure our jvm process
# uses at least 1g of memory, If we don't do this we can get OOM issues when
# installing gems. See https://github.com/elastic/logstash/issues/5179
export JRUBY_OPTS="-J-Xmx1g"
# Extract the version number from the version.yml file
# e.g.: 8.6.0
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
source ./$(dirname "$0")/dra_common.sh
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
case "$WORKFLOW_TYPE" in
snapshot)
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
rake artifact:all
else
@ -25,8 +20,10 @@ case "$WORKFLOW_TYPE" in
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
fi
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
;;
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
;;
staging)
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
RELEASE=1 rake artifact:all
else
@ -36,33 +33,38 @@ case "$WORKFLOW_TYPE" in
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:all
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
fi
;;
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
;;
*)
exit 1
;;
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
;;
esac
echo "Saving tar.gz for docker images"
docker save docker.elastic.co/logstash/logstash:${STACK_VERSION} | gzip -c > build/logstash-${STACK_VERSION}-docker-image-x86_64.tar.gz
docker save docker.elastic.co/logstash/logstash-oss:${STACK_VERSION} | gzip -c > build/logstash-oss-${STACK_VERSION}-docker-image-x86_64.tar.gz
docker save docker.elastic.co/logstash/logstash-ubi8:${STACK_VERSION} | gzip -c > build/logstash-ubi8-${STACK_VERSION}-docker-image-x86_64.tar.gz
info "Saving tar.gz for docker images"
save_docker_tarballs "x86_64" "${STACK_VERSION}"
echo "GENERATED ARTIFACTS"
info "GENERATED ARTIFACTS"
for file in build/logstash-*; do shasum $file;done
echo "Creating dependencies report for ${STACK_VERSION}"
info "Creating dependencies report for ${STACK_VERSION}"
mkdir -p build/distributions/dependencies-reports/
bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
echo "GENERATED DEPENDENCIES REPORT"
info "GENERATED DEPENDENCIES REPORT"
shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
echo "UPLOADING TO INTERMEDIATE BUCKET"
info "UPLOADING TO INTERMEDIATE BUCKET"
for file in build/logstash-*; do
gsutil cp $file gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
upload_to_bucket $file ${STACK_VERSION}
done
gsutil cp build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
gsutil cp build/logstash-${STACK_VERSION}-docker-image-x86_64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
gsutil cp build/logstash-oss-${STACK_VERSION}-docker-image-x86_64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
gsutil cp build/logstash-ubi8-${STACK_VERSION}-docker-image-x86_64.tar.gz gs://logstash-ci-artifacts/dra/${STACK_VERSION}/
# Upload Dependencies Report
upload_to_bucket "build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv" ${STACK_VERSION}
for image in logstash logstash-oss logstash-ubi8; do
upload_to_bucket "build/$image-${STACK_VERSION}-docker-image-x86_64.tar.gz" ${STACK_VERSION}
done
echo "####################################################################"
echo "##################### Finishing $0"
echo "####################################################################"