mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[artifacts] Publish (#129499)
* [artifacts] Publish release artifacts
This adds the publish step to the release pipeline, which uses the
release-manager CLI to upload a list Kibana artifacts.
* cleanup
* add missing artifacts
* enable rm
* fix syntax
* fix artifact
* source utils
* fix path
* fix version
* skip steps
* re-add pipeline steps
* set permissions
* Update .buildkite/scripts/steps/artifacts/publish.sh
Co-authored-by: Chris <chris.koehnke@elastic.co>
* Revert "Update .buildkite/scripts/steps/artifacts/publish.sh"
This reverts commit a6228f52cd.
* export
* support version qualifier
* setup env
* cleanup
* consistency
* \n
* comment
* unneccessary fallback
* +x
* hoist env args
* source
* link to artifacts
* fix artifacts summary
* relax docker glob
* \n
* fix summary
* fix variable
Co-authored-by: Chris <chris.koehnke@elastic.co>
This commit is contained in:
Jonathan Budzenski
GitHub
parent
255bba6877
commit
ab1bec320b
6 changed files with 141 additions and 55 deletions
|
|
@ -8,25 +8,33 @@ steps:
|
|||
- wait
|
||||
|
||||
- command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
|
||||
label: Package testing for deb
|
||||
label: Artifact Testing
|
||||
agents:
|
||||
queue: n2-4-virt
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
|
||||
- command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
|
||||
label: Package testing for rpm
|
||||
label: Artifact Testing
|
||||
agents:
|
||||
queue: n2-4-virt
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
|
||||
- command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
|
||||
label: Package testing for docker
|
||||
label: Artifact Testing
|
||||
agents:
|
||||
queue: n2-4-virt
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
|
||||
- command: .buildkite/scripts/steps/artifacts/docker_context.sh
|
||||
label: 'Docker Build Context'
|
||||
label: 'Docker Context Verification'
|
||||
agents:
|
||||
queue: n2-2
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
|
||||
- wait
|
||||
|
||||
- command: .buildkite/scripts/steps/artifacts/publish.sh
|
||||
label: 'Publish Kibana Artifacts'
|
||||
agents:
|
||||
queue: n2-2
|
||||
timeout_in_minutes: 30
|
||||
|
|
|
|||
|
|
@ -4,44 +4,14 @@ set -euo pipefail
|
|||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
|
||||
if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
|
||||
VERSION="$(jq -r '.version' package.json)"
|
||||
RELEASE_ARG="--release"
|
||||
else
|
||||
VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
|
||||
RELEASE_ARG=""
|
||||
fi
|
||||
source .buildkite/scripts/steps/artifacts/env.sh
|
||||
|
||||
echo "--- Build Kibana Distribution"
|
||||
node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud
|
||||
|
||||
echo "--- Build dependencies report"
|
||||
node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
|
||||
|
||||
# Release verification
|
||||
if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
|
||||
echo "--- Build and push Kibana Cloud Distribution"
|
||||
# This doesn't meet the requirements for a release image, implementation TBD
|
||||
# Beats artifacts will need to match a specific commit sha that matches other stack iamges
|
||||
# For now this is a placeholder step that will allow us to run automated Cloud tests
|
||||
# against a best guess approximation of a release image
|
||||
echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
|
||||
trap 'docker logout docker.elastic.co' EXIT
|
||||
|
||||
node scripts/build \
|
||||
"$RELEASE_ARG" \
|
||||
--skip-initialize \
|
||||
--skip-generic-folders \
|
||||
--skip-platform-folders \
|
||||
--skip-archives \
|
||||
--docker-images \
|
||||
--docker-tag-qualifier="$GIT_COMMIT" \
|
||||
--docker-push \
|
||||
--skip-docker-ubi \
|
||||
--skip-docker-ubuntu \
|
||||
--skip-docker-contexts
|
||||
fi
|
||||
echo "--- Build Kibana artifacts"
|
||||
node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
|
||||
|
||||
echo "--- Build and upload dependencies report"
|
||||
node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
|
||||
cd target
|
||||
sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
|
||||
buildkite-agent artifact upload "*"
|
||||
cd -
|
||||
cd -
|
||||
|
|
|
|||
|
|
@ -4,17 +4,11 @@ set -euo pipefail
|
|||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
|
||||
if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
|
||||
VERSION="$(jq -r '.version' package.json)"
|
||||
RELEASE_ARG="--release"
|
||||
else
|
||||
VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
|
||||
RELEASE_ARG=""
|
||||
fi
|
||||
source .buildkite/scripts/steps/artifacts/env.sh
|
||||
|
||||
echo "--- Create contexts"
|
||||
mkdir -p target
|
||||
node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
|
||||
node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
|
||||
|
||||
echo "--- Setup default context"
|
||||
DOCKER_BUILD_FOLDER=$(mktemp -d)
|
||||
|
|
@ -22,7 +16,7 @@ DOCKER_BUILD_FOLDER=$(mktemp -d)
|
|||
tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
|
||||
cd $DOCKER_BUILD_FOLDER
|
||||
|
||||
buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
|
||||
echo "--- Build context"
|
||||
docker build .
|
||||
|
|
|
|||
34
.buildkite/scripts/steps/artifacts/env.sh
Executable file
34
.buildkite/scripts/steps/artifacts/env.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RELEASE_BUILD="${RELEASE_BUILD:="false"}"
|
||||
VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
|
||||
|
||||
BASE_VERSION="$(jq -r '.version' package.json)"
|
||||
|
||||
if [[ "$VERSION_QUALIFIER" == "" ]]; then
|
||||
QUALIFIER_VERSION="$BASE_VERSION"
|
||||
else
|
||||
QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER"
|
||||
fi
|
||||
|
||||
if [[ "$RELEASE_BUILD" == "true" ]]; then
|
||||
FULL_VERSION="$QUALIFIER_VERSION"
|
||||
|
||||
# Beats artifacts will need to match a specific commit sha that matches other stack images
|
||||
# for release builds. For now we are skipping Cloud builds until there's a pointer.
|
||||
BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
|
||||
WORKFLOW="staging"
|
||||
else
|
||||
FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
|
||||
BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
|
||||
WORKFLOW="snapshot"
|
||||
fi
|
||||
|
||||
export VERSION_QUALIFIER
|
||||
export BASE_VERSION
|
||||
export QUALIFIER_VERSION
|
||||
export FULL_VERSION
|
||||
export BUILD_ARGS
|
||||
export WORKFLOW
|
||||
80
.buildkite/scripts/steps/artifacts/publish.sh
Normal file
80
.buildkite/scripts/steps/artifacts/publish.sh
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
source .buildkite/scripts/steps/artifacts/env.sh
|
||||
|
||||
echo "--- Download and verify artifacts"
|
||||
function download {
|
||||
buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
sha512sum -c "$1.sha512.txt"
|
||||
rm "$1.sha512.txt"
|
||||
}
|
||||
|
||||
mkdir -p target
|
||||
cd target
|
||||
|
||||
download "kibana-$FULL_VERSION-docker-image.tar.gz"
|
||||
download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
|
||||
download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
|
||||
|
||||
download "kibana-$FULL_VERSION-arm64.deb"
|
||||
download "kibana-$FULL_VERSION-amd64.deb"
|
||||
download "kibana-$FULL_VERSION-x86_64.rpm"
|
||||
download "kibana-$FULL_VERSION-aarch64.rpm"
|
||||
|
||||
download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
|
||||
download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
|
||||
download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
|
||||
|
||||
download "kibana-$FULL_VERSION-linux-aarch64.tar.gz"
|
||||
download "kibana-$FULL_VERSION-linux-x86_64.tar.gz"
|
||||
|
||||
download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz"
|
||||
download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz"
|
||||
|
||||
download "kibana-$FULL_VERSION-windows-x86_64.zip"
|
||||
|
||||
download "dependencies-$FULL_VERSION.csv"
|
||||
|
||||
cd -
|
||||
|
||||
echo "--- Set artifact permissions"
|
||||
chmod -R a+r target/*
|
||||
chmod -R a+w target
|
||||
|
||||
echo "--- Pull latest Release Manager CLI"
|
||||
echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
|
||||
trap 'docker logout docker.elastic.co' EXIT
|
||||
docker pull docker.elastic.co/infra/release-manager:latest
|
||||
|
||||
echo "--- Publish artifacts"
|
||||
export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
|
||||
export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
|
||||
export VAULT_ADDR="https://secrets.elastic.co:8200"
|
||||
docker run --rm \
|
||||
--name release-manager \
|
||||
-e VAULT_ADDR \
|
||||
-e VAULT_ROLE_ID \
|
||||
-e VAULT_SECRET_ID \
|
||||
--mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
|
||||
docker.elastic.co/infra/release-manager:latest \
|
||||
cli collect \
|
||||
--project kibana \
|
||||
--branch "$KIBANA_BASE_BRANCH" \
|
||||
--commit "$GIT_COMMIT" \
|
||||
--workflow "$WORKFLOW" \
|
||||
--version "$BASE_VERSION" \
|
||||
--qualifier "$VERSION_QUALIFIER" \
|
||||
--artifact-set main
|
||||
|
||||
ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
|
||||
ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
|
||||
|
||||
cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
|
||||
### Artifacts Summary
|
||||
|
||||
$ARTIFACTS_SUMMARY
|
||||
EOF
|
||||
|
|
@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
|
|||
buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
KIBANA_IP_ADDRESS="192.168.56.6"
|
||||
elif [[ "$TEST_PACKAGE" == "docker" ]]; then
|
||||
buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
|
||||
KIBANA_IP_ADDRESS="192.168.56.7"
|
||||
fi
|
||||
cd ..
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue