mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 14:47:19 -04:00
[ci] Backport Buildkite DRA pipeline+scripts to 8.10 (#15392)
This is a backport of the DRA pipeline and related scripts from: - #15366 - #15365 - #15356 - #15352 - #15344 - #15343 - #15337 - #15312 Note that it's a manual backport because some PRs (e.g. #15312) contain files (`catalog-info.yaml`) that should only live on the `main` branch.
This commit is contained in:
Dimitrios Liappis
GitHub
parent
63339bee98
commit
bdc7f26c23
8 changed files with 474 additions and 0 deletions
90
.buildkite/dra/build_docker.sh
Executable file
90
.buildkite/dra/build_docker.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
||||||
|
#!/bin/bash -ie
|
||||||
|
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Starting $0"
|
||||||
|
echo "####################################################################"
|
||||||
|
|
||||||
|
source ./$(dirname "$0")/common.sh
|
||||||
|
|
||||||
|
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
|
||||||
|
case "$WORKFLOW_TYPE" in
|
||||||
|
snapshot)
|
||||||
|
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||||
|
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||||
|
rake artifact:docker || error "artifact:docker build failed."
|
||||||
|
rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||||
|
rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||||
|
fi
|
||||||
|
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||||
|
# in case of alpha or beta releases:
|
||||||
|
# e.g: 8.0.0-alpha1
|
||||||
|
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
fi
|
||||||
|
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||||
|
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||||
|
;;
|
||||||
|
staging)
|
||||||
|
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||||
|
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||||
|
RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
|
||||||
|
RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||||
|
RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||||
|
fi
|
||||||
|
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||||
|
# in case of alpha or beta releases:
|
||||||
|
# e.g: 8.0.0-alpha1
|
||||||
|
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
fi
|
||||||
|
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
info "Saving tar.gz for docker images"
|
||||||
|
save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
|
||||||
|
|
||||||
|
info "Generated Artifacts"
|
||||||
|
for file in build/logstash-*; do shasum $file;done
|
||||||
|
|
||||||
|
info "Uploading DRA artifacts in buildkite's artifact store ..."
|
||||||
|
# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
|
||||||
|
images="logstash logstash-oss"
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
# No logstash-ubi8 for AARCH64
|
||||||
|
images="logstash logstash-oss logstash-ubi8"
|
||||||
|
fi
|
||||||
|
for image in ${images}; do
|
||||||
|
buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
|
||||||
|
# overwritten when building aarch64 (or viceversa).
|
||||||
|
if [ "$ARCH" != "aarch64" ]; then
|
||||||
|
for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do
|
||||||
|
buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Finishing $0"
|
||||||
|
echo "####################################################################"
|
||||||
58
.buildkite/dra/build_packages.sh
Executable file
58
.buildkite/dra/build_packages.sh
Executable file
|
|
@ -0,0 +1,58 @@
|
||||||
|
#!/bin/bash -ie
|
||||||
|
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Starting $0"
|
||||||
|
echo "####################################################################"
|
||||||
|
|
||||||
|
source ./$(dirname "$0")/common.sh
|
||||||
|
|
||||||
|
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
|
||||||
|
case "$WORKFLOW_TYPE" in
|
||||||
|
snapshot)
|
||||||
|
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||||
|
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||||
|
SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||||
|
else
|
||||||
|
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||||
|
# in case of alpha or beta releases:
|
||||||
|
# e.g: 8.0.0-alpha1
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||||
|
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
fi
|
||||||
|
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||||
|
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||||
|
;;
|
||||||
|
staging)
|
||||||
|
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||||
|
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||||
|
RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||||
|
else
|
||||||
|
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||||
|
# in case of alpha or beta releases:
|
||||||
|
# e.g: 8.0.0-alpha1
|
||||||
|
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||||
|
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
fi
|
||||||
|
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
info "Generated Artifacts"
|
||||||
|
for file in build/logstash-*; do shasum $file;done
|
||||||
|
|
||||||
|
info "Creating dependencies report for ${STACK_VERSION}"
|
||||||
|
mkdir -p build/distributions/dependencies-reports/
|
||||||
|
bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
|
||||||
|
|
||||||
|
info "Generated dependencies report"
|
||||||
|
shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
|
||||||
|
|
||||||
|
info "Uploading DRA artifacts in buildkite's artifact store ..."
|
||||||
|
buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*"
|
||||||
|
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Finishing $0"
|
||||||
|
echo "####################################################################"
|
||||||
46
.buildkite/dra/common.sh
Executable file
46
.buildkite/dra/common.sh
Executable file
|
|
@ -0,0 +1,46 @@
|
||||||
|
function info {
|
||||||
|
echo "--- INFO: $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
function error {
|
||||||
|
echo "--- ERROR: $1"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
function save_docker_tarballs {
|
||||||
|
local arch="${1:?architecture required}"
|
||||||
|
local version="${2:?stack-version required}"
|
||||||
|
local images="logstash logstash-oss"
|
||||||
|
if [ "${arch}" != "aarch64" ]; then
|
||||||
|
# No logstash-ubi8 for AARCH64
|
||||||
|
images="logstash logstash-oss logstash-ubi8"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for image in ${images}; do
|
||||||
|
tar_file="${image}-${version}-docker-image-${arch}.tar"
|
||||||
|
docker save -o "build/${tar_file}" \
|
||||||
|
"docker.elastic.co/logstash/${image}:${version}" || \
|
||||||
|
error "Unable to save tar file ${tar_file} for ${image} image."
|
||||||
|
# NOTE: if docker save exited with non-zero the error log already exited the script
|
||||||
|
gzip "build/${tar_file}"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Since we are using the system jruby, we need to make sure our jvm process
|
||||||
|
# uses at least 1g of memory, If we don't do this we can get OOM issues when
|
||||||
|
# installing gems. See https://github.com/elastic/logstash/issues/5179
|
||||||
|
export JRUBY_OPTS="-J-Xmx1g"
|
||||||
|
|
||||||
|
# Extract the version number from the version.yml file
|
||||||
|
# e.g.: 8.6.0
|
||||||
|
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
|
||||||
|
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
|
||||||
|
|
||||||
|
info "Agent is running on architecture [$(uname -i)]"
|
||||||
|
|
||||||
|
export VERSION_QUALIFIER_OPT=${VERSION_QUALIFIER_OPT:-""}
|
||||||
|
export DRA_DRY_RUN=${DRA_DRY_RUN:-""}
|
||||||
|
|
||||||
|
if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then
|
||||||
|
info "Release manager will run in dry-run mode [$DRA_DRY_RUN]"
|
||||||
|
fi
|
||||||
23
.buildkite/dra/docker-env-setup.sh
Executable file
23
.buildkite/dra/docker-env-setup.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
DOCKER_REGISTRY="docker.elastic.co"
|
||||||
|
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
|
||||||
|
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
|
||||||
|
|
||||||
|
|
||||||
|
function docker_login {
|
||||||
|
DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
|
||||||
|
DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
|
||||||
|
docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
|
||||||
|
unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
|
||||||
|
}
|
||||||
|
|
||||||
|
function release_manager_login {
|
||||||
|
DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
|
||||||
|
VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
|
||||||
|
VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
|
||||||
|
VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
|
||||||
|
export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID
|
||||||
|
}
|
||||||
15
.buildkite/dra/docker-env-teardown.sh
Executable file
15
.buildkite/dra/docker-env-teardown.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Unset all variables ending with _SECRET or _TOKEN
|
||||||
|
for var in $(printenv | sed 's;=.*;;' | sort); do
|
||||||
|
if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then
|
||||||
|
unset "$var"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if command -v docker &>/dev/null; then
|
||||||
|
DOCKER_REGISTRY="docker.elastic.co"
|
||||||
|
docker logout $DOCKER_REGISTRY
|
||||||
|
fi
|
||||||
131
.buildkite/dra/generatesteps.py
Normal file
131
.buildkite/dra/generatesteps.py
Normal file
|
|
@ -0,0 +1,131 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
def to_bk_key_friendly_string(key):
|
||||||
|
"""
|
||||||
|
Convert and return key to an acceptable format for Buildkite's key: field
|
||||||
|
Only alphanumerics, dashes and underscores are allowed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
mapping_table = str.maketrans({'.': '_'})
|
||||||
|
|
||||||
|
return key.translate(mapping_table)
|
||||||
|
|
||||||
|
def package_x86_step(branch, workflow_type):
|
||||||
|
step = f'''
|
||||||
|
- label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||||
|
key: "logstash_build_packages_dra"
|
||||||
|
agents:
|
||||||
|
provider: gcp
|
||||||
|
imageProject: elastic-images-prod
|
||||||
|
image: family/platform-ingest-logstash-ubuntu-2204
|
||||||
|
machineType: "n2-standard-16"
|
||||||
|
diskSizeGb: 200
|
||||||
|
command: |
|
||||||
|
export WORKFLOW_TYPE="{workflow_type}"
|
||||||
|
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||||
|
eval "$(rbenv init -)"
|
||||||
|
.buildkite/scripts/dra/build_packages.sh
|
||||||
|
'''
|
||||||
|
|
||||||
|
return step
|
||||||
|
|
||||||
|
def package_x86_docker_step(branch, workflow_type):
|
||||||
|
step = f'''
|
||||||
|
- label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||||
|
key: "logstash_build_x86_64_docker_dra"
|
||||||
|
agents:
|
||||||
|
provider: gcp
|
||||||
|
imageProject: elastic-images-prod
|
||||||
|
image: family/platform-ingest-logstash-ubuntu-2204
|
||||||
|
machineType: "n2-standard-16"
|
||||||
|
diskSizeGb: 200
|
||||||
|
command: |
|
||||||
|
export WORKFLOW_TYPE="{workflow_type}"
|
||||||
|
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||||
|
export ARCH="x86_64"
|
||||||
|
eval "$(rbenv init -)"
|
||||||
|
.buildkite/scripts/dra/build_docker.sh
|
||||||
|
'''
|
||||||
|
|
||||||
|
return step
|
||||||
|
|
||||||
|
def package_aarch64_docker_step(branch, workflow_type):
|
||||||
|
step = f'''
|
||||||
|
- label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||||
|
key: "logstash_build_aarch64_docker_dra"
|
||||||
|
agents:
|
||||||
|
provider: aws
|
||||||
|
imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
|
||||||
|
instanceType: "m6g.4xlarge"
|
||||||
|
diskSizeGb: 200
|
||||||
|
command: |
|
||||||
|
export WORKFLOW_TYPE="{workflow_type}"
|
||||||
|
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||||
|
export ARCH="aarch64"
|
||||||
|
eval "$(rbenv init -)"
|
||||||
|
.buildkite/scripts/dra/build_docker.sh
|
||||||
|
'''
|
||||||
|
|
||||||
|
return step
|
||||||
|
|
||||||
|
def publish_dra_step(branch, workflow_type, depends_on):
|
||||||
|
step = f'''
|
||||||
|
- label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||||
|
key: "logstash_publish_dra"
|
||||||
|
depends_on: "{depends_on}"
|
||||||
|
agents:
|
||||||
|
provider: gcp
|
||||||
|
imageProject: elastic-images-prod
|
||||||
|
image: family/platform-ingest-logstash-ubuntu-2204
|
||||||
|
machineType: "n2-standard-16"
|
||||||
|
diskSizeGb: 200
|
||||||
|
command: |
|
||||||
|
echo "+++ Restoring Artifacts"
|
||||||
|
buildkite-agent artifact download "build/logstash*" .
|
||||||
|
buildkite-agent artifact download "build/distributions/**/*" .
|
||||||
|
echo "+++ Changing permissions for the release manager"
|
||||||
|
sudo chown -R :1000 build
|
||||||
|
echo "+++ Running DRA publish step"
|
||||||
|
export WORKFLOW_TYPE="{workflow_type}"
|
||||||
|
.buildkite/scripts/dra/publish.sh
|
||||||
|
'''
|
||||||
|
|
||||||
|
return step
|
||||||
|
|
||||||
|
def build_steps_to_yaml(branch, workflow_type):
|
||||||
|
steps = []
|
||||||
|
steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type)))
|
||||||
|
steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type)))
|
||||||
|
steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type)))
|
||||||
|
|
||||||
|
return steps
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
workflow_type = os.environ["WORKFLOW_TYPE"]
|
||||||
|
except ImportError:
|
||||||
|
print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=<staging|snapshot>\n.Exiting.")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
branch = os.environ["BUILDKITE_BRANCH"]
|
||||||
|
|
||||||
|
structure = {"steps": []}
|
||||||
|
|
||||||
|
# Group defining parallel steps that build and save artifacts
|
||||||
|
group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
|
||||||
|
|
||||||
|
structure["steps"].append({
|
||||||
|
"group": f":Build Artifacts - {workflow_type.upper()}",
|
||||||
|
"key": group_key,
|
||||||
|
"steps": build_steps_to_yaml(branch, workflow_type),
|
||||||
|
})
|
||||||
|
|
||||||
|
# Final step: pull artifacts built above and publish them via the release-manager
|
||||||
|
structure["steps"].extend(
|
||||||
|
yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
|
||||||
|
)
|
||||||
|
|
||||||
|
print(yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))
|
||||||
100
.buildkite/dra/publish.sh
Executable file
100
.buildkite/dra/publish.sh
Executable file
|
|
@ -0,0 +1,100 @@
|
||||||
|
#!/bin/bash -i
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Starting $0"
|
||||||
|
echo "####################################################################"
|
||||||
|
|
||||||
|
source ./$(dirname "$0")/common.sh
|
||||||
|
|
||||||
|
PLAIN_STACK_VERSION=$STACK_VERSION
|
||||||
|
|
||||||
|
# This is the branch selector that needs to be passed to the release-manager
|
||||||
|
# It has to be the name of the branch which originates the artifacts.
|
||||||
|
RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'`
|
||||||
|
if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then
|
||||||
|
RELEASE_BRANCH=$RELEASE_VER
|
||||||
|
else
|
||||||
|
RELEASE_BRANCH=main
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$VERSION_QUALIFIER_OPT" ]; then
|
||||||
|
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||||
|
# in case of alpha or beta releases:
|
||||||
|
# e.g: 8.0.0-alpha1
|
||||||
|
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$WORKFLOW_TYPE" in
|
||||||
|
snapshot)
|
||||||
|
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||||
|
;;
|
||||||
|
staging)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
|
||||||
|
|
||||||
|
if [ "$RELEASE_VER" != "7.17" ]; then
|
||||||
|
# Version 7.17.x doesn't generates ARM artifacts for Darwin
|
||||||
|
# TODO see if we need to do anything here
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected
|
||||||
|
# by the release manager.
|
||||||
|
# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle
|
||||||
|
# for more details.
|
||||||
|
# TODO filter it out when uploading artifacts instead
|
||||||
|
rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
|
||||||
|
|
||||||
|
info "Downloaded ARTIFACTS sha report"
|
||||||
|
for file in build/logstash-*; do shasum $file;done
|
||||||
|
|
||||||
|
mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv
|
||||||
|
|
||||||
|
# set required permissions on artifacts and directory
|
||||||
|
chmod -R a+r build/*
|
||||||
|
chmod -R a+w build
|
||||||
|
|
||||||
|
chmod -R a+r $PWD/*
|
||||||
|
chmod -R a+w $PWD
|
||||||
|
|
||||||
|
info "Setup docker credentials"
|
||||||
|
# TODO disable tracing
|
||||||
|
# set +o xtrace
|
||||||
|
source ./$(dirname "$0")/docker-env-setup.sh
|
||||||
|
release_manager_login
|
||||||
|
|
||||||
|
# ensure the latest image has been pulled
|
||||||
|
docker pull docker.elastic.co/infra/release-manager:latest
|
||||||
|
|
||||||
|
info "Running the release manager ..."
|
||||||
|
|
||||||
|
# collect the artifacts for use with the unified build
|
||||||
|
docker run --rm \
|
||||||
|
--name release-manager \
|
||||||
|
-e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
|
||||||
|
-e VAULT_ROLE_ID \
|
||||||
|
-e VAULT_SECRET_ID \
|
||||||
|
--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
|
||||||
|
docker.elastic.co/infra/release-manager:latest \
|
||||||
|
cli collect \
|
||||||
|
--project logstash \
|
||||||
|
--branch ${RELEASE_BRANCH} \
|
||||||
|
--commit "$(git rev-parse HEAD)" \
|
||||||
|
--workflow "${WORKFLOW_TYPE}" \
|
||||||
|
--version "${PLAIN_STACK_VERSION}" \
|
||||||
|
--artifact-set main \
|
||||||
|
${DRA_DRY_RUN}
|
||||||
|
|
||||||
|
info "Teardown logins"
|
||||||
|
$(dirname "$0")/docker-env-teardown.sh
|
||||||
|
|
||||||
|
echo "####################################################################"
|
||||||
|
echo "##################### Finishing $0"
|
||||||
|
echo "####################################################################"
|
||||||
11
.buildkite/dra_pipeline.yml
Normal file
11
.buildkite/dra_pipeline.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- label: ":pipeline: Generate steps"
|
||||||
|
command: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
echo "--- Building [${WORKFLOW_TYPE}] artifacts"
|
||||||
|
python3 -m pip install pyyaml
|
||||||
|
echo "--- Building dynamic pipeline steps"
|
||||||
|
python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
|
||||||
Loading…
Add table
Add a link
Reference in a new issue