From 30abc09a68730f66f7dd0eed2dac7c386ef74805 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 20 Nov 2020 08:03:07 +0000 Subject: [PATCH] Add retries to Docker build apk commands (#65194) Most of the Elasticsearch Docker build process implements retries in order to make the process more robust in the face of transient errors e.g. network failures. However, we missed out the `apk` command in `master`'s Dockerfile. This PR adds the same retry loop around `apk` as features elsewhere in the `Dockerfile`. As part of this, I implemented a helper closure that generates the same loop construct throughout the `Dockerfile`. --- distribution/docker/build.gradle | 15 +++++++++- distribution/docker/src/docker/Dockerfile | 34 ++++++++++------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 16830809de60..d926bb38c8b3 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -75,6 +75,18 @@ RUN curl --retry 8 -S -L \\ def (major,minor) = VersionProperties.elasticsearch.split("\\.") + def retry_loop = { name, command, indentSize = 4, exitKeyword = 'exit' -> + String indent = ' ' * indentSize + String commandWithRetry = """for iter in {1..10}; do +${indent} ${command} && +${indent} exit_code=0 && break || +${indent} exit_code=\$? && echo "${name} error: retry \$iter in 10s" && sleep 10; +${indent}done; +${indent}${exitKeyword} \$exit_code""" + + return commandWithRetry.replaceAll(" *\n", " \\\\\n") + } + return [ 'base_image' : base.getImage(), 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', @@ -87,7 +99,8 @@ RUN curl --retry 8 -S -L \\ 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, - 'major_minor_version' : "${major}.${minor}" + 'major_minor_version' : "${major}.${minor}", + 'retry_loop' : retry_loop ] } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 33ded0d25761..415f020e4f06 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -27,7 +27,7 @@ FROM ${base_image} AS builder # Install required packages to extract the Elasticsearch distribution -RUN ${package_manager} install -y tar gzip +RUN <%= retry_loop(package_manager, "${package_manager} install -y tar gzip") %> # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. @@ -42,8 +42,8 @@ RUN set -eux ; \\ x86_64) tini_bin='tini-amd64' ;; \\ *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\ esac ; \\ - curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\ - curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ sha256sum -c \${tini_bin}.sha256sum ; \\ rm \${tini_bin}.sha256sum ; \\ mv \${tini_bin} /bin/tini ; \\ @@ -74,7 +74,7 @@ ENV TARBALL_URL https://curl.haxx.se/download/curl-\${VERSION}.tar.xz ENV TARBALL_PATH curl-\${VERSION}.tar.xz # Install dependencies -RUN apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file +RUN <%= retry_loop('apk', 'apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file') %> RUN mkdir /work WORKDIR /work @@ -83,11 +83,7 @@ WORKDIR /work RUN function retry_wget() { \\ local URL="\$1" ; \\ local DEST="\$2" ; \\ - for iter in {1..10}; do \\ - wget "\$URL" -O "\$DEST" && \\ - exit_code=0 && break || exit_code=\$? && echo "wget error fetching \$URL: retry \$iter in 10s" && sleep 10; \\ - done; \\ - return \$exit_code ; \\ + <%= retry_loop('wget', 'wget "\$URL\" -O "\$DEST"', 6, 'return') %> ; \\ } ; \\ retry_wget "https://daniel.haxx.se/mykey.asc" "curl-gpg.pub" && \\ retry_wget "\${TARBALL_URL}.asc" "\${TARBALL_PATH}.asc" && \\ @@ -162,8 +158,8 @@ RUN set -e ; \\ ;; \\ *) echo >&2 "Unsupported architecture \$(arch)" ; exit 1 ;; \\ esac ; \\ - curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/v0.19.0/\${TINI_BIN}" ; \\ - curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/v0.19.0/\${TINI_BIN}.sha256sum" ; \\ + curl --retry 10 -S -L -O "https://github.com/krallin/tini/releases/download/v0.19.0/\${TINI_BIN}" ; \\ + curl --retry 10 -S -L -O "https://github.com/krallin/tini/releases/download/v0.19.0/\${TINI_BIN}.sha256sum" ; \\ sha256sum -c "\${TINI_BIN}.sha256sum" ; \\ rm "\${TINI_BIN}.sha256sum" ; \\ mv "\${TINI_BIN}" /rootfs/bin/tini ; \\ @@ -243,15 +239,13 @@ FROM ${base_image} <% if (docker_base == "ubi") { %> -RUN for iter in {1..10}; do \\ - ${package_manager} update --setopt=tsflags=nodocs -y && \\ - ${package_manager} install --setopt=tsflags=nodocs -y \\ - nc shadow-utils zip unzip findutils procps-ng && \\ - ${package_manager} clean all && \\ - exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && \\ - sleep 10; \\ - done; \\ - (exit \$exit_code) +RUN <%= retry_loop( + package_manager, + "${package_manager} update --setopt=tsflags=nodocs -y && \n" + + " ${package_manager} install --setopt=tsflags=nodocs -y \n" + + " nc shadow-utils zip unzip findutils procps-ng && \n" + + " ${package_manager} clean all" + ) %> <% } else { %>