mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 14:47:19 -04:00
[ci] Backport Buildkite DRA pipeline+scripts to 8.10 (#15392)
This is a backport of the DRA pipeline and related scripts from: - #15366 - #15365 - #15356 - #15352 - #15344 - #15343 - #15337 - #15312 Note that it's a manual backport because some PRs (e.g. #15312) contain files (`catalog-info.yaml`) that should only live on the `main` branch.
This commit is contained in:
Dimitrios Liappis
GitHub
parent
63339bee98
commit
bdc7f26c23
8 changed files with 474 additions and 0 deletions
90
.buildkite/dra/build_docker.sh
Executable file
90
.buildkite/dra/build_docker.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
|||
#!/bin/bash -ie
|
||||
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
|
||||
echo "####################################################################"
|
||||
echo "##################### Starting $0"
|
||||
echo "####################################################################"
|
||||
|
||||
source ./$(dirname "$0")/common.sh
|
||||
|
||||
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
|
||||
case "$WORKFLOW_TYPE" in
|
||||
snapshot)
|
||||
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||
rake artifact:docker || error "artifact:docker build failed."
|
||||
rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||
rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||
fi
|
||||
else
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||
fi
|
||||
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||
# in case of alpha or beta releases:
|
||||
# e.g: 8.0.0-alpha1
|
||||
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
fi
|
||||
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||
;;
|
||||
staging)
|
||||
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||
RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
|
||||
RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||
RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||
fi
|
||||
else
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
|
||||
fi
|
||||
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||
# in case of alpha or beta releases:
|
||||
# e.g: 8.0.0-alpha1
|
||||
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
fi
|
||||
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||
;;
|
||||
*)
|
||||
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Saving tar.gz for docker images"
|
||||
save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
|
||||
|
||||
info "Generated Artifacts"
|
||||
for file in build/logstash-*; do shasum $file;done
|
||||
|
||||
info "Uploading DRA artifacts in buildkite's artifact store ..."
|
||||
# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
|
||||
images="logstash logstash-oss"
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
# No logstash-ubi8 for AARCH64
|
||||
images="logstash logstash-oss logstash-ubi8"
|
||||
fi
|
||||
for image in ${images}; do
|
||||
buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
|
||||
done
|
||||
|
||||
# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
|
||||
# overwritten when building aarch64 (or viceversa).
|
||||
if [ "$ARCH" != "aarch64" ]; then
|
||||
for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do
|
||||
buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "####################################################################"
|
||||
echo "##################### Finishing $0"
|
||||
echo "####################################################################"
|
||||
58
.buildkite/dra/build_packages.sh
Executable file
58
.buildkite/dra/build_packages.sh
Executable file
|
|
@ -0,0 +1,58 @@
|
|||
#!/bin/bash -ie
|
||||
#Note - ensure that the -e flag is set to properly set the $? status if any command fails
|
||||
echo "####################################################################"
|
||||
echo "##################### Starting $0"
|
||||
echo "####################################################################"
|
||||
|
||||
source ./$(dirname "$0")/common.sh
|
||||
|
||||
# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
|
||||
case "$WORKFLOW_TYPE" in
|
||||
snapshot)
|
||||
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||
SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||
else
|
||||
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||
# in case of alpha or beta releases:
|
||||
# e.g: 8.0.0-alpha1
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
fi
|
||||
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||
;;
|
||||
staging)
|
||||
info "Building artifacts for the $WORKFLOW_TYPE workflow..."
|
||||
if [ -z "$VERSION_QUALIFIER_OPT" ]; then
|
||||
RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||
else
|
||||
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||
# in case of alpha or beta releases:
|
||||
# e.g: 8.0.0-alpha1
|
||||
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
|
||||
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
fi
|
||||
info "Build complete, setting STACK_VERSION to $STACK_VERSION."
|
||||
;;
|
||||
*)
|
||||
error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Generated Artifacts"
|
||||
for file in build/logstash-*; do shasum $file;done
|
||||
|
||||
info "Creating dependencies report for ${STACK_VERSION}"
|
||||
mkdir -p build/distributions/dependencies-reports/
|
||||
bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
|
||||
|
||||
info "Generated dependencies report"
|
||||
shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
|
||||
|
||||
info "Uploading DRA artifacts in buildkite's artifact store ..."
|
||||
buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*"
|
||||
|
||||
echo "####################################################################"
|
||||
echo "##################### Finishing $0"
|
||||
echo "####################################################################"
|
||||
46
.buildkite/dra/common.sh
Executable file
46
.buildkite/dra/common.sh
Executable file
|
|
@ -0,0 +1,46 @@
|
|||
function info {
|
||||
echo "--- INFO: $1"
|
||||
}
|
||||
|
||||
function error {
|
||||
echo "--- ERROR: $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function save_docker_tarballs {
|
||||
local arch="${1:?architecture required}"
|
||||
local version="${2:?stack-version required}"
|
||||
local images="logstash logstash-oss"
|
||||
if [ "${arch}" != "aarch64" ]; then
|
||||
# No logstash-ubi8 for AARCH64
|
||||
images="logstash logstash-oss logstash-ubi8"
|
||||
fi
|
||||
|
||||
for image in ${images}; do
|
||||
tar_file="${image}-${version}-docker-image-${arch}.tar"
|
||||
docker save -o "build/${tar_file}" \
|
||||
"docker.elastic.co/logstash/${image}:${version}" || \
|
||||
error "Unable to save tar file ${tar_file} for ${image} image."
|
||||
# NOTE: if docker save exited with non-zero the error log already exited the script
|
||||
gzip "build/${tar_file}"
|
||||
done
|
||||
}
|
||||
|
||||
# Since we are using the system jruby, we need to make sure our jvm process
|
||||
# uses at least 1g of memory, If we don't do this we can get OOM issues when
|
||||
# installing gems. See https://github.com/elastic/logstash/issues/5179
|
||||
export JRUBY_OPTS="-J-Xmx1g"
|
||||
|
||||
# Extract the version number from the version.yml file
|
||||
# e.g.: 8.6.0
|
||||
# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
|
||||
STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
|
||||
|
||||
info "Agent is running on architecture [$(uname -i)]"
|
||||
|
||||
export VERSION_QUALIFIER_OPT=${VERSION_QUALIFIER_OPT:-""}
|
||||
export DRA_DRY_RUN=${DRA_DRY_RUN:-""}
|
||||
|
||||
if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then
|
||||
info "Release manager will run in dry-run mode [$DRA_DRY_RUN]"
|
||||
fi
|
||||
23
.buildkite/dra/docker-env-setup.sh
Executable file
23
.buildkite/dra/docker-env-setup.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DOCKER_REGISTRY="docker.elastic.co"
|
||||
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
|
||||
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
|
||||
|
||||
|
||||
function docker_login {
|
||||
DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
|
||||
DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
|
||||
docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
|
||||
unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
|
||||
}
|
||||
|
||||
function release_manager_login {
|
||||
DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
|
||||
VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
|
||||
VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
|
||||
VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
|
||||
export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID
|
||||
}
|
||||
15
.buildkite/dra/docker-env-teardown.sh
Executable file
15
.buildkite/dra/docker-env-teardown.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Unset all variables ending with _SECRET or _TOKEN
|
||||
for var in $(printenv | sed 's;=.*;;' | sort); do
|
||||
if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then
|
||||
unset "$var"
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v docker &>/dev/null; then
|
||||
DOCKER_REGISTRY="docker.elastic.co"
|
||||
docker logout $DOCKER_REGISTRY
|
||||
fi
|
||||
131
.buildkite/dra/generatesteps.py
Normal file
131
.buildkite/dra/generatesteps.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
def to_bk_key_friendly_string(key):
|
||||
"""
|
||||
Convert and return key to an acceptable format for Buildkite's key: field
|
||||
Only alphanumerics, dashes and underscores are allowed.
|
||||
"""
|
||||
|
||||
mapping_table = str.maketrans({'.': '_'})
|
||||
|
||||
return key.translate(mapping_table)
|
||||
|
||||
def package_x86_step(branch, workflow_type):
|
||||
step = f'''
|
||||
- label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||
key: "logstash_build_packages_dra"
|
||||
agents:
|
||||
provider: gcp
|
||||
imageProject: elastic-images-prod
|
||||
image: family/platform-ingest-logstash-ubuntu-2204
|
||||
machineType: "n2-standard-16"
|
||||
diskSizeGb: 200
|
||||
command: |
|
||||
export WORKFLOW_TYPE="{workflow_type}"
|
||||
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||
eval "$(rbenv init -)"
|
||||
.buildkite/scripts/dra/build_packages.sh
|
||||
'''
|
||||
|
||||
return step
|
||||
|
||||
def package_x86_docker_step(branch, workflow_type):
|
||||
step = f'''
|
||||
- label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||
key: "logstash_build_x86_64_docker_dra"
|
||||
agents:
|
||||
provider: gcp
|
||||
imageProject: elastic-images-prod
|
||||
image: family/platform-ingest-logstash-ubuntu-2204
|
||||
machineType: "n2-standard-16"
|
||||
diskSizeGb: 200
|
||||
command: |
|
||||
export WORKFLOW_TYPE="{workflow_type}"
|
||||
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||
export ARCH="x86_64"
|
||||
eval "$(rbenv init -)"
|
||||
.buildkite/scripts/dra/build_docker.sh
|
||||
'''
|
||||
|
||||
return step
|
||||
|
||||
def package_aarch64_docker_step(branch, workflow_type):
|
||||
step = f'''
|
||||
- label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||
key: "logstash_build_aarch64_docker_dra"
|
||||
agents:
|
||||
provider: aws
|
||||
imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
|
||||
instanceType: "m6g.4xlarge"
|
||||
diskSizeGb: 200
|
||||
command: |
|
||||
export WORKFLOW_TYPE="{workflow_type}"
|
||||
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
|
||||
export ARCH="aarch64"
|
||||
eval "$(rbenv init -)"
|
||||
.buildkite/scripts/dra/build_docker.sh
|
||||
'''
|
||||
|
||||
return step
|
||||
|
||||
def publish_dra_step(branch, workflow_type, depends_on):
|
||||
step = f'''
|
||||
- label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts"
|
||||
key: "logstash_publish_dra"
|
||||
depends_on: "{depends_on}"
|
||||
agents:
|
||||
provider: gcp
|
||||
imageProject: elastic-images-prod
|
||||
image: family/platform-ingest-logstash-ubuntu-2204
|
||||
machineType: "n2-standard-16"
|
||||
diskSizeGb: 200
|
||||
command: |
|
||||
echo "+++ Restoring Artifacts"
|
||||
buildkite-agent artifact download "build/logstash*" .
|
||||
buildkite-agent artifact download "build/distributions/**/*" .
|
||||
echo "+++ Changing permissions for the release manager"
|
||||
sudo chown -R :1000 build
|
||||
echo "+++ Running DRA publish step"
|
||||
export WORKFLOW_TYPE="{workflow_type}"
|
||||
.buildkite/scripts/dra/publish.sh
|
||||
'''
|
||||
|
||||
return step
|
||||
|
||||
def build_steps_to_yaml(branch, workflow_type):
|
||||
steps = []
|
||||
steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type)))
|
||||
steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type)))
|
||||
steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type)))
|
||||
|
||||
return steps
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
workflow_type = os.environ["WORKFLOW_TYPE"]
|
||||
except ImportError:
|
||||
print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=<staging|snapshot>\n.Exiting.")
|
||||
exit(1)
|
||||
|
||||
branch = os.environ["BUILDKITE_BRANCH"]
|
||||
|
||||
structure = {"steps": []}
|
||||
|
||||
# Group defining parallel steps that build and save artifacts
|
||||
group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
|
||||
|
||||
structure["steps"].append({
|
||||
"group": f":Build Artifacts - {workflow_type.upper()}",
|
||||
"key": group_key,
|
||||
"steps": build_steps_to_yaml(branch, workflow_type),
|
||||
})
|
||||
|
||||
# Final step: pull artifacts built above and publish them via the release-manager
|
||||
structure["steps"].extend(
|
||||
yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
|
||||
)
|
||||
|
||||
print(yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))
|
||||
100
.buildkite/dra/publish.sh
Executable file
100
.buildkite/dra/publish.sh
Executable file
|
|
@ -0,0 +1,100 @@
|
|||
#!/bin/bash -i
|
||||
set -euo pipefail
|
||||
|
||||
echo "####################################################################"
|
||||
echo "##################### Starting $0"
|
||||
echo "####################################################################"
|
||||
|
||||
source ./$(dirname "$0")/common.sh
|
||||
|
||||
PLAIN_STACK_VERSION=$STACK_VERSION
|
||||
|
||||
# This is the branch selector that needs to be passed to the release-manager
|
||||
# It has to be the name of the branch which originates the artifacts.
|
||||
RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'`
|
||||
if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then
|
||||
RELEASE_BRANCH=$RELEASE_VER
|
||||
else
|
||||
RELEASE_BRANCH=main
|
||||
fi
|
||||
|
||||
if [ -n "$VERSION_QUALIFIER_OPT" ]; then
|
||||
# Qualifier is passed from CI as optional field and specify the version postfix
|
||||
# in case of alpha or beta releases:
|
||||
# e.g: 8.0.0-alpha1
|
||||
STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
|
||||
fi
|
||||
|
||||
case "$WORKFLOW_TYPE" in
|
||||
snapshot)
|
||||
STACK_VERSION=${STACK_VERSION}-SNAPSHOT
|
||||
;;
|
||||
staging)
|
||||
;;
|
||||
*)
|
||||
error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
|
||||
|
||||
if [ "$RELEASE_VER" != "7.17" ]; then
|
||||
# Version 7.17.x doesn't generates ARM artifacts for Darwin
|
||||
# TODO see if we need to do anything here
|
||||
:
|
||||
fi
|
||||
|
||||
# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected
|
||||
# by the release manager.
|
||||
# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle
|
||||
# for more details.
|
||||
# TODO filter it out when uploading artifacts instead
|
||||
rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
|
||||
|
||||
info "Downloaded ARTIFACTS sha report"
|
||||
for file in build/logstash-*; do shasum $file;done
|
||||
|
||||
mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv
|
||||
|
||||
# set required permissions on artifacts and directory
|
||||
chmod -R a+r build/*
|
||||
chmod -R a+w build
|
||||
|
||||
chmod -R a+r $PWD/*
|
||||
chmod -R a+w $PWD
|
||||
|
||||
info "Setup docker credentials"
|
||||
# TODO disable tracing
|
||||
# set +o xtrace
|
||||
source ./$(dirname "$0")/docker-env-setup.sh
|
||||
release_manager_login
|
||||
|
||||
# ensure the latest image has been pulled
|
||||
docker pull docker.elastic.co/infra/release-manager:latest
|
||||
|
||||
info "Running the release manager ..."
|
||||
|
||||
# collect the artifacts for use with the unified build
|
||||
docker run --rm \
|
||||
--name release-manager \
|
||||
-e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
|
||||
-e VAULT_ROLE_ID \
|
||||
-e VAULT_SECRET_ID \
|
||||
--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
|
||||
docker.elastic.co/infra/release-manager:latest \
|
||||
cli collect \
|
||||
--project logstash \
|
||||
--branch ${RELEASE_BRANCH} \
|
||||
--commit "$(git rev-parse HEAD)" \
|
||||
--workflow "${WORKFLOW_TYPE}" \
|
||||
--version "${PLAIN_STACK_VERSION}" \
|
||||
--artifact-set main \
|
||||
${DRA_DRY_RUN}
|
||||
|
||||
info "Teardown logins"
|
||||
$(dirname "$0")/docker-env-teardown.sh
|
||||
|
||||
echo "####################################################################"
|
||||
echo "##################### Finishing $0"
|
||||
echo "####################################################################"
|
||||
11
.buildkite/dra_pipeline.yml
Normal file
11
.buildkite/dra_pipeline.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
|
||||
|
||||
steps:
|
||||
- label: ":pipeline: Generate steps"
|
||||
command: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "--- Building [${WORKFLOW_TYPE}] artifacts"
|
||||
python3 -m pip install pyyaml
|
||||
echo "--- Building dynamic pipeline steps"
|
||||
python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
|
||||
Loading…
Add table
Add a link
Reference in a new issue