[ci] simplify ci_setup scripts (#26215) (#26393)

Cleanup some of the legacy bits from our ci_setup, simplify the setup necessary in child CI scripts, and unify the more generic bits of the scripts.

.ci/run.sh

@@ -5,6 +5,8 @@ set -e
 # move to Kibana root
 cd "$(dirname "$0")/.."
 
+source src/dev/ci_setup/setup.sh
+
 case "$JOB" in
 kibana-intake)
   ./test/scripts/jenkins_unit.sh

src/dev/ci_setup/checkout_sibling_es.sh

@@ -1,29 +1,5 @@
 #!/bin/bash
 
-# Turn on semi-strict mode
-set -e
-set -o pipefail
-
-SCRIPT=${BASH_SOURCE[0]}
-# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
-while [ -h "$SCRIPT" ] ; do
-  ls=$(ls -ld "$SCRIPT")
-  # Drop everything prior to ->
-  link=$(expr "$ls" : '.*-> \(.*\)$')
-  if expr "$link" : '/.*' > /dev/null; then
-    SCRIPT="$link"
-  else
-    SCRIPT=$(dirname "$SCRIPT")/"$link"
-  fi
-  echo $SCRIPT
-done
-
-# kibana directory is the parent of x-pack
-export KIBANA_DIR="$(cd "$(dirname "$SCRIPT")/../../.."; pwd)"
-
-# PARENT_DIR is the directory where we will checkout es and x-pack-es
-export PARENT_DIR="$(cd "$KIBANA_DIR"/..; pwd)"
-
 function checkout_sibling {
   project=$1
   targetDir=$2
@@ -108,3 +84,27 @@ function checkout_sibling {
 }
 
 checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES"
+
+# Set the JAVA_HOME based on the Java property file in the ES repo
+# This assumes the naming convention used on CI (ex: ~/.java/java10)
+ES_DIR="$PARENT_DIR/elasticsearch"
+ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
+
+
+if [ ! -f $ES_JAVA_PROP_PATH ]; then
+  echo "Unable to set JAVA_HOME, $ES_JAVA_PROP_PATH does not exist"
+  exit 1
+fi
+
+# While sourcing the property file would currently work, we want
+# to support the case where whitespace surrounds the equals.
+# This has the added benefit of explicitly exporting property values
+export ES_BUILD_JAVA="$(cat "$ES_JAVA_PROP_PATH" | grep "^ES_BUILD_JAVA" | cut -d'=' -f2 | tr -d '[:space:]')"
+
+if [ -z "$ES_BUILD_JAVA" ]; then
+  echo "Unable to set JAVA_HOME, ES_BUILD_JAVA not present in $ES_JAVA_PROP_PATH"
+  exit 1
+fi
+
+echo "Setting JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA"
+export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA

src/dev/ci_setup/java_setup.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-set -e
-
-# Sets the JAVA_HOME based on the Java property file in the ES repo
-# This assumes the naming convention used on CI (ex: ~/.java/java10)
-
-ES_DIR="$(cd "$(dirname "$BASH_SOURCE")/../../../../elasticsearch"; pwd)"
-ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
-
-# While sourcing the property file would currently work, we want
-# to support the case where whitespace surrounds the equals.
-# This has the added benefit of explicitly exporting property values
-
-function exportPropertyValue {
-  propertyFile=$1
-  propertyKey=$2
-
-  propertyValue=$(cat $propertyFile | grep "^$propertyKey" | cut -d'=' -f2 | tr -d '[:space:]')
-  export $propertyKey=$propertyValue
-}
-
-if [ ! -f $ES_JAVA_PROP_PATH ]; then
-  echo "Unable to set JAVA_HOME, $ES_JAVA_PROP_PATH does not exist"
-  exit 1
-fi
-
-exportPropertyValue $ES_JAVA_PROP_PATH "ES_BUILD_JAVA"
-
-if [ -z "$ES_BUILD_JAVA" ]; then
-  echo "Unable to set JAVA_HOME, ES_BUILD_JAVA not present in $ES_JAVA_PROP_PATH"
-  exit 1
-fi
-
-echo "Setting JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA"
-export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA

src/dev/ci_setup/setup.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -e
+set -o pipefail
 
 dir="$(pwd)"
 cacheDir="${CACHE_DIR:-"$HOME/.kibana"}"
@@ -26,6 +27,24 @@ else
   exit 1
 fi
 
+export KIBANA_DIR="$dir"
+export XPACK_DIR="$KIBANA_DIR/x-pack"
+export PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
+echo "-> KIBANA_DIR $KIBANA_DIR"
+echo "-> XPACK_DIR $XPACK_DIR"
+echo "-> PARENT_DIR $PARENT_DIR"
+
+###
+### Extract the bootstrap cache that we create in the packer_cache.sh script
+###
+bootstrapCache="$cacheDir/bootstrap_cache/master.tar"
+if [ -f "$bootstrapCache" ]; then
+  echo "extracting bootstrap_cache from $bootstrapCache";
+  tar -xf "$bootstrapCache";
+else
+  echo "bootstrap_cache missing";
+  exit 1;
+fi
 
 ###
 ### download node

test/scripts/jenkins_ci_group.sh

@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
-source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
-source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+source src/dev/ci_setup/checkout_sibling_es.sh
 
 "$(FORCE_COLOR=0 yarn bin)/grunt" functionalTests:ensureAllTestsInCiGroup;
 

test/scripts/jenkins_report_failed_tests.sh

@@ -1,6 +1,3 @@
 #!/usr/bin/env bash
 
-set -e
-source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
-
 xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report;

test/scripts/jenkins_unit.sh

@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
-source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/java_setup.sh"
+source src/dev/ci_setup/checkout_sibling_es.sh
 
 export TEST_BROWSER_HEADLESS=1
 export TEST_ES_FROM=${TEST_ES_FROM:-source}

test/scripts/jenkins_xpack.sh

@@ -1,16 +1,8 @@
 #!/usr/bin/env bash
 
-set -e
-set -o pipefail
-
-source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname $0)/../../src/dev/ci_setup/java_setup.sh"
+source src/dev/ci_setup/checkout_sibling_es.sh
 
 export TEST_BROWSER_HEADLESS=1
-export XPACK_DIR="$(cd "$(dirname "$0")/../../x-pack"; pwd)"
-echo "-> XPACK_DIR ${XPACK_DIR}"
-
 
 echo " -> Running mocha tests"
 cd "$XPACK_DIR"

test/scripts/jenkins_xpack_ci_group.sh

@@ -1,15 +1,8 @@
 #!/usr/bin/env bash
 
-set -e
-set -o pipefail
-
-source "$(dirname "$0")/../../src/dev/ci_setup/setup.sh"
-source "$(dirname "$0")/../../src/dev/ci_setup/git_setup.sh"
-source "$(dirname "$0")/../../src/dev/ci_setup/java_setup.sh"
+source src/dev/ci_setup/checkout_sibling_es.sh
 
 export TEST_BROWSER_HEADLESS=1
-export XPACK_DIR="$(cd "$(dirname "$0")/../../x-pack"; pwd)"
-echo "-> XPACK_DIR ${XPACK_DIR}"
 
 echo " -> Ensuring all functional tests are in a ciGroup"
 cd "$XPACK_DIR"