[7.17] Add docker retries (#191981)

## Summary
Backport of #191824

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.buildkite/scripts/common/util.sh

@@ -124,3 +124,32 @@ set_git_merge_base() {
 
   export GITHUB_PR_MERGE_BASE
 }
+
+docker_with_retry () {
+  cmd=$1
+  shift
+  args=("$@")
+  attempt=0
+  max_retries=5
+  sleep_time=15
+
+  while true
+  do
+    attempt=$((attempt+1))
+
+    if [ $attempt -gt $max_retries ]
+    then
+      echo "Docker $cmd retries exceeded, aborting."
+      exit 1
+    fi
+
+    if docker "$cmd" "${args[@]}"
+    then
+      echo "Docker $cmd successful."
+      break
+    else
+      echo "Docker $cmd unsuccessful, attempt '$attempt'... Retrying in $sleep_time"
+      sleep $sleep_time
+    fi
+  done
+}

.buildkite/scripts/steps/artifacts/publish.sh

@@ -39,14 +39,14 @@ download "kibana-$FULL_VERSION-windows-x86_64.zip"
 
 download "dependencies-$FULL_VERSION.csv"
 
-cd - 
+cd -
 
 echo "--- Set artifact permissions"
 chmod -R a+r target/*
 chmod -R a+w target
 
 echo "--- Pull latest Release Manager CLI"
-docker pull docker.elastic.co/infra/release-manager:latest
+docker_with_retry pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then

packages/kbn-test/BUILD.bazel

@@ -75,6 +75,7 @@ RUNTIME_DEPS = [
     "@npm//strip-ansi",
     "@npm//xmlbuilder",
     "@npm//xml2js",
+    "@npm//p-retry"
 ]
 
 TYPES_DEPS = [
@@ -116,6 +117,7 @@ TYPES_DEPS = [
   "@npm//@types/semver",
   "@npm//@types/uuid",
   "@npm//@types/xml2js",
+  "@npm//p-retry"
 ]
 
 jsts_transpiler(

packages/kbn-test/src/functional_test_runner/lib/docker_servers/docker_servers_service.ts

@@ -11,6 +11,7 @@ import execa from 'execa';
 import * as Rx from 'rxjs';
 import { filter, take, map } from 'rxjs/operators';
 import { ToolingLog } from '@kbn/dev-utils';
+import pRetry from 'p-retry';
 
 import { Lifecycle } from '../lifecycle';
 import { observeContainerRunning } from './container_running';
@@ -104,8 +105,13 @@ export class DockerServersService {
     const { image, name, waitFor, waitForLogLine } = server;
 
     // pull image from registry
-    log.info(`[docker:${name}] pulling docker image "${image}"`);
-    await execa('docker', ['pull', image]);
+    await pRetry(
+      async () => {
+        log.info(`[docker:${name}] pulling docker image "${image}"`);
+        await execa('docker', ['pull', image]);
+      },
+      { retries: 5 }
+    );
 
     // run the image that we just pulled
     const containerId = await this.dockerRun(server);