github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

add a timeout to buildkite artifact downloads (#139046) (#139115)

Browse source 
(cherry picked from commit ab576bb6c1)

Co-authored-by: Spencer <spencer@elastic.co>
This commit is contained in:
Kibana Machine 2022-08-18 14:17:06 -04:00 • committed by GitHub
parent 207ee7ef8b
commit ae57b8a683
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 39 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.buildkite/pipeline-utils/test-failures/annotate.ts
View file

@ -150,7 +150,7 @@ export const annotateTestFailures = async () => {
}
exec(
`buildkite-agent artifact download --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
`.buildkite/scripts/common/download_artifact.sh --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
);
const failures: TestFailure[] = recursiveReadDir(failureDir)

6
.buildkite/scripts/common/download_artifact.sh Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
source "$(dirname "$0")/util.sh"
download_artifact "$@"

6
.buildkite/scripts/common/util.sh
View file

@ -168,3 +168,9 @@ npm_install_global() {
npm install -g "$toInstall" && touch "$successFlag"
fi
}
# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage
# times-out after 60 seconds and retries up to 3 times
download_artifact() {
retry 3 1 timeout 3m buildkite-agent artifact download "$@"
}

6
.buildkite/scripts/download_build_artifacts.sh
View file

@ -2,13 +2,15 @@
set -euo pipefail
source "$(dirname "$0")/common/util.sh"
if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
echo '--- Downloading Distribution and Plugin artifacts'
cd "$WORKSPACE"
buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
mkdir -p "$KIBANA_BUILD_LOCATION"
tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

3
.buildkite/scripts/steps/artifacts/cloud.sh
View file

@ -4,12 +4,13 @@ set -euo pipefail
.buildkite/scripts/bootstrap.sh
source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/artifacts/env.sh
echo "--- Build and publish Cloud image"
mkdir -p target
buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
node scripts/build \
--skip-initialize \

3
.buildkite/scripts/steps/artifacts/docker_context.sh
View file

@ -4,6 +4,7 @@ set -euo pipefail
.buildkite/scripts/bootstrap.sh
source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/artifacts/env.sh
KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
@ -28,7 +29,7 @@ fi
tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
cd $DOCKER_BUILD_FOLDER
buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
echo "--- Build context"
docker build .

6
.buildkite/scripts/steps/artifacts/publish.sh
View file

@ -7,8 +7,8 @@ source .buildkite/scripts/steps/artifacts/env.sh
echo "--- Download and verify artifacts"
function download {
buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
sha512sum -c "$1.sha512.txt"
rm "$1.sha512.txt"
}
@ -40,7 +40,7 @@ download "kibana-$FULL_VERSION-windows-x86_64.zip"
download "dependencies-$FULL_VERSION.csv"
cd -
cd -
echo "--- Set artifact permissions"
chmod -R a+r target/*

2
.buildkite/scripts/steps/cloud/build_and_deploy.sh
View file

@ -13,7 +13,7 @@ VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
echo "--- Download Kibana Distribution"
mkdir -p ./target
buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
echo "--- Build Cloud Distribution"
ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"

2
.buildkite/scripts/steps/code_coverage/ftr_configs.sh
View file

@ -28,7 +28,7 @@ configs="${FTR_CONFIG:-}"
if [[ "$configs" == "" ]]; then
echo "--- Downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
download_artifact ftr_run_order.json .
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi

4
.buildkite/scripts/steps/code_coverage/ingest.sh
View file

@ -29,7 +29,7 @@ revolveBuildHashes() {
}
collectRan() {
buildkite-agent artifact download target/ran_files/* .
download_artifact target/ran_files/* .
while read -r x; do
ran=("${ran[@]}" "$(cat "$x")")
@ -51,7 +51,7 @@ fetchArtifacts() {
local xs=("$@")
for x in "${xs[@]}"; do
buildkite-agent artifact download "target/kibana-coverage/${x}/*" .
download_artifact "target/kibana-coverage/${x}/*" .
done
}

3
.buildkite/scripts/steps/code_coverage/jest_parallel.sh
View file

@ -2,6 +2,7 @@
set -euo pipefail
source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/code_coverage/util.sh
export JOB=$BUILDKITE_PARALLEL_JOB
@ -18,7 +19,7 @@ fi
export TEST_TYPE
echo "--- downloading jest test run order"
buildkite-agent artifact download jest_run_order.json .
download_artifact jest_run_order.json .
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
echo "--- KIBANA_DIR: $KIBANA_DIR"

4
.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
View file

@ -34,8 +34,8 @@ mkdir "${BUILD_ID}"
tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
# Upload Kibana build, plugins, commit sha and traces to the bucket
buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
download_artifact kibana-default.tar.gz ./"${BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz ./"${BUILD_ID}"
echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
echo "--- Update reference to the latest CI build"

7
.buildkite/scripts/steps/package_testing/test.sh
View file

@ -2,6 +2,7 @@
set -euo pipefail
source "$(dirname "$0")/../../common/util.sh"
.buildkite/scripts/bootstrap.sh
echo "--- Package Testing for $TEST_PACKAGE"
@ -9,13 +10,13 @@ echo "--- Package Testing for $TEST_PACKAGE"
mkdir -p target
cd target
if [[ "$TEST_PACKAGE" == "deb" ]]; then
buildkite-agent artifact download 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.5"
elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
buildkite-agent artifact download 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.6"
elif [[ "$TEST_PACKAGE" == "docker" ]]; then
buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.7"
fi
cd ..

2
.buildkite/scripts/steps/test/ftr_configs.sh
View file

@ -32,7 +32,7 @@ fi
if [ "$configs" == "" ] && [ "$FTR_CONFIG_GROUP_KEY" != "" ]; then
echo "--- downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
download_artifact ftr_run_order.json .
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi

3
.buildkite/scripts/steps/test/jest_parallel.sh
View file

@ -2,6 +2,7 @@
set -euo pipefail
source "$(dirname "$0")/../../common/util.sh"
export JOB=${BUILDKITE_PARALLEL_JOB:-0}
# a jest failure will result in the script returning an exit code of 10
@ -20,7 +21,7 @@ fi
export TEST_TYPE
echo "--- downloading jest test run order"
buildkite-agent artifact download jest_run_order.json .
download_artifact jest_run_order.json .
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
while read -r config; do