[kbn-es] make custom snapshot url a feature (#45750)

* [kbn-es] make custom snapshot url a feature

* download es snapshot first

(cherry picked from commit 510566d66b)
(cherry picked from commit f85c7402f6)

packages/kbn-es/src/artifact.js

@@ -31,6 +31,7 @@ const asyncPipeline = promisify(pipeline);
 const V1_VERSIONS_API = 'https://artifacts-api.elastic.co/v1/versions';
 
 const { cache } = require('./utils');
+const { resolveCustomSnapshotUrl } = require('./custom_snapshots');
 const { createCliError, isCliError } = require('./errors');
 
 const TEST_ES_SNAPSHOT_VERSION = process.env.TEST_ES_SNAPSHOT_VERSION
@@ -94,6 +95,12 @@ exports.Artifact = class Artifact {
    */
   static async getSnapshot(license, version, log) {
     const urlVersion = `${encodeURIComponent(version)}-SNAPSHOT`;
+
+    const customSnapshotArtifactSpec = resolveCustomSnapshotUrl(urlVersion, license);
+    if (customSnapshotArtifactSpec) {
+      return new Artifact(customSnapshotArtifactSpec, log);
+    }
+
     const urlBuild = encodeURIComponent(TEST_ES_SNAPSHOT_VERSION);
     const url = `${V1_VERSIONS_API}/${urlVersion}/builds/${urlBuild}/projects/elasticsearch`;
 

packages/kbn-es/src/custom_snapshots.js

@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { basename } = require('path');
+
+function isVersionFlag(a) {
+  return a.startsWith('--version');
+}
+
+function getCustomSnapshotUrl() {
+  // force use of manually created snapshots until live ones are available
+  if (!process.env.KBN_ES_SNAPSHOT_URL && !process.argv.some(isVersionFlag)) {
+    // return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}';
+    return undefined;
+  }
+
+  if (process.env.KBN_ES_SNAPSHOT_URL && process.env.KBN_ES_SNAPSHOT_URL !== 'false') {
+    return process.env.KBN_ES_SNAPSHOT_URL;
+  }
+}
+
+function resolveCustomSnapshotUrl(urlVersion, license) {
+  const customSnapshotUrl = getCustomSnapshotUrl();
+
+  if (!customSnapshotUrl) {
+    return;
+  }
+
+  const ext = process.platform === 'win32' ? 'zip' : 'tar.gz';
+  const os = process.platform === 'win32' ? 'windows' : process.platform;
+  const name = license === 'oss' ? 'elasticsearch-oss' : 'elasticsearch';
+  const overrideUrl = customSnapshotUrl
+    .replace('{name}', name)
+    .replace('{ext}', ext)
+    .replace('{os}', os)
+    .replace('{version}', urlVersion);
+
+  return {
+    url: overrideUrl,
+    checksumUrl: overrideUrl + '.sha512',
+    checksumType: 'sha512',
+    filename: basename(overrideUrl),
+  };
+}
+
+module.exports = { getCustomSnapshotUrl, resolveCustomSnapshotUrl };

test/scripts/jenkins_build_kibana.sh

@@ -1,7 +1,13 @@
 #!/usr/bin/env bash
 
 source src/dev/ci_setup/setup_env.sh
+
+echo " -> downloading es snapshot"
+node scripts/es snapshot --license=oss --download-only;
+
+echo " -> Ensuring all functional tests are in a ciGroup"
 yarn run grunt functionalTests:ensureAllTestsInCiGroup;
+
+echo " -> building and extracting OSS Kibana distributable for use in functional tests"
 node scripts/build --debug --oss
 
-node scripts/es snapshot --license=oss --download-only;

test/scripts/jenkins_xpack_build_kibana.sh

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 
-echo " -> building and extracting default Kibana distributable for use in functional tests"
 cd "$KIBANA_DIR"
 source src/dev/ci_setup/setup_env.sh
 
+echo " -> downloading es snapshot"
+node scripts/es snapshot --download-only;
+
 echo " -> Ensuring all functional tests are in a ciGroup"
 cd "$XPACK_DIR"
 node scripts/functional_tests --assert-none-excluded \
@@ -18,11 +20,10 @@ node scripts/functional_tests --assert-none-excluded \
   --include-tag ciGroup9 \
   --include-tag ciGroup10
 
+echo " -> building and extracting default Kibana distributable for use in functional tests"
 cd "$KIBANA_DIR"
 node scripts/build --debug --no-oss
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$PARENT_DIR/install/kibana"
 mkdir -p "$installDir"
 tar -xzf "$linuxBuild" -C "$installDir" --strip=1
-
-node scripts/es snapshot --download-only;