[Ops] ES Serverless verification: tag based filtering (#168374)

## Summary
Allows for env-var controlled filtering of executed test suites in the
ES Serverless verification job, and adjusts existing flags and behaviour
to better-fit future usage.

We're changing the job to not publish the `latest-verified` tag by
default. This is to prevent external calls to the job from accidentally
promoting random tags to `latest-verified`, see changes below.

Flag changes: 
- `PUBLISH_DOCKER_TAG`: if set to 1/true, passing runs will promote the
tested ES Serverless tag to `latest-verified`.
   - (used to be default, now it requires this flag)
- `PUBLISH_MANIFEST`: if set to 1/true, passing runs will upload the
manifest attesting what (kibana + es) combination was used in the test
   - (used to be called `UPLOAD_MANIFEST`)
- `SKIP_CYPRESS`: if set to 1/true, it will skip running the cypress
tests
   - new flag
- `FTR_EXTRA_ARGS`: a string argument, if passed, it will be forwarded
verbatim to the FTR run script
- new flag, can be used to control the filtering required for #167611
(eg.: `FTR_EXTRA_ARGS="--include-tag=ml"`)

Example run:

https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/builds/64#018b1a30-3360-4995-874a-864f18e104d5

Closes: #168376

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -226,6 +226,10 @@ export async function pickTestGroupRunOrder() {
           .filter(Boolean)
       : ['build'];
 
+  const FTR_EXTRA_ARGS: Record<string, string> = process.env.FTR_EXTRA_ARGS
+    ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS }
+    : {};
+
   const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
 
   const ftrConfigsIncluded = LIMIT_CONFIG_TYPE.includes('functional');
@@ -464,6 +468,7 @@ export async function pickTestGroupRunOrder() {
                   },
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,
+                    ...FTR_EXTRA_ARGS,
                   },
                   retry: {
                     automatic: [

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -1,4 +1,14 @@
 # https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+
+### Parameters for this job:
+# PUBLISH_DOCKER_TAG: if set to 1/true, passing runs will promote the tested ES Serverless tag to latest-verified.
+# PUBLISH_MANIFEST: if set to 1/true, passing runs will upload the manifest attesting what (kibana + es) combination was used in the test
+# SKIP_VERIFICATION: if set to 1/true, it will skip running all tests
+# SKIP_CYPRESS: if set to 1/true, it will skip running the cypress tests
+# FTR_EXTRA_ARGS: a string argument, if passed, it will be forwarded verbatim to the FTR run script
+# ES_SERVERLESS_IMAGE: the tag for the docker image to test, in the form of docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:$TAG
+# BUILDKITE_COMMIT: the commit hash of the kibana branch to test
+
 agents:
   queue: kibana-default
 
@@ -24,7 +34,6 @@ steps:
           queue: n2-16-spot
         key: build
         depends_on: pre-build
-        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
         timeout_in_minutes: 60
         retry:
           automatic:
@@ -35,10 +44,12 @@ steps:
         command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
         agents:
           queue: kibana-default
+        depends_on: build
         timeout_in_minutes: 10
         env:
           FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
           FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
           LIMIT_CONFIG_TYPE: 'functional'
         retry:
           automatic:
@@ -47,6 +58,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless.sh
         label: 'Serverless Security Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
@@ -59,6 +71,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
         label: 'Serverless Explore - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
@@ -71,6 +84,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
         label: 'Serverless Investigations - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -19,7 +19,12 @@ else
   SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
 fi
 
-echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+if [[ "${PUBLISH_DOCKER_TAG:-}" =~ ^(1|true)$ ]]; then
+  echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+else
+  echo "--- Skipping ES Serverless image because PUBLISH_DOCKER_TAG is not set"
+  exit 0
+fi
 
 echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
 
@@ -63,7 +68,7 @@ echo "Image push to $TARGET_IMAGE successful."
 echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
 
 MANIFEST_UPLOAD_PATH="Skipped"
-if [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+if [[ "${PUBLISH_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Uploading latest-verified manifest to GCS"
   cat << EOT >> $MANIFEST_FILE_NAME
 {
@@ -84,7 +89,7 @@ EOT
   gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
   MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
 
-elif [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ ]]; then
+elif [[ "${PUBLISH_MANIFEST:-}" =~ ^(1|true)$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
 elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -11,6 +11,9 @@ if [ "$FTR_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; the
   exit 1
 fi
 
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
 export JOB="$FTR_CONFIG_GROUP_KEY"
 
 FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
@@ -49,7 +52,9 @@ while read -r config; do
     continue;
   fi
 
-  echo "--- $ node scripts/functional_tests --bail --config $config"
+  FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS"
+  echo "--- $ $FULL_COMMAND"
+
   start=$(date +%s)
 
   # prevent non-zero exit code from breaking the loop
@@ -57,7 +62,8 @@ while read -r config; do
   node ./scripts/functional_tests \
     --bail \
     --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-    --config="$config"
+    --config="$config" \
+    "$EXTRA_ARGS"
   lastCode=$?
   set -e;