[Ops] ES Serverless image verification pipeline (#166054)

## Summary
Prepares the serverless FTR tests to be runnable with a custom ES image.
(`--esServerlessImage` cli arg)
Creates a pipeline for testing and promoting ES Serverless docker
releases.

The job can be triggered here:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote
The three main env variables it takes:
- BUILDKITE_BRANCH: the kibana branch to test with (maybe not as
important)
 - BUILDKITE_COMMIT: the kibana commit to test with
- ES_SERVERLESS_IMAGE: the elasticsearch serverless image, or tag to use
from this repo:
`docker.elastic.co/elasticsearch-ci/elasticsearch-serverless`

## TODOS:
 - [x] set `latest_verified` with full img path as default
- [x] ~~find other CLIs that might need the `esServerlessImage` argument
(if the docker runner has multiple usages)~~ | I confused the `yarn es
docker` with this, because I thought we only run ES serverless in a
docker container, but `elasticsearch` can also be run in docker.
- [x] set `latest-compatible` or similar flag in a manifest in gcs for
Elastic's use-case
- [ ] ensure we can only verify "forward" (ie.: to avoid a
parameterization on old versions to set our pointers back) [on a second
thought, this might be kept as a feature to roll back (if we should ever
need that)]


There are two confusing things I couldn't sort out just yet:
#### Ambiguity in --esServerlessImage 
We can either have 2 CLI args: one for an image tag, one for an image
repo/image url, or we can have one (like I have it now) and interpret
that in the code, it can be either the image url, or the tag. It's more
flexible, but it's two things in one. Is it ok this way, or is it too
confusing?
e.g.:
```
node scripts/functional_tests --esFrom serverless --esServerlessImage docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:git-8fc8f941bd4d --bail --config x-pack/test_serverless/functional/test_suites/security/config.ts

# or
node scripts/functional_tests  --esFrom serverless --esServerlessImage latest --bail --config x-pack/test_serverless/functional/test_suites/security/config.ts
```

#### Ambiguity in the default image path
The published ES Serverless images will sit on this image path:
`docker.elastic.co/elasticsearch-ci/elasticsearch-serverless`, however,
our one exception is the `latest-verified` which we will be tagging
under a different path, where we have write rights:
`docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified`.

Is it okay, that by default, we're searching in the `elasticsearch-ci`
images for any tags as parameters (after all, all the new images will be
published there), however our grand default will ultimately be
`docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified`.


## Links
Buildkite:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote
eg.:
https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/builds/24
Closes: https://github.com/elastic/kibana/issues/162931

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -0,0 +1,58 @@
+# https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+agents:
+  queue: kibana-default
+
+steps:
+  - label: "Annotate runtime parameters"
+    command: |
+      buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE"
+      buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
+
+  - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
+    if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"
+    steps:
+      - label: "Pre-Build"
+        command: .buildkite/scripts/lifecycle/pre_build.sh
+        key: pre-build
+        timeout_in_minutes: 10
+        agents:
+          queue: kibana-default
+
+      - label: "Build Kibana Distribution and Plugins"
+        command: .buildkite/scripts/steps/build_kibana.sh
+        agents:
+          queue: n2-16-spot
+        key: build
+        depends_on: pre-build
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - label: "Pick Test Group Run Order"
+        command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+        agents:
+          queue: kibana-default
+        env:
+          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
+          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          LIMIT_CONFIG_TYPE: 'functional'
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+  - wait: ~
+
+  - label: ":arrow_up::elastic::arrow_up: Promote docker image"
+    command: .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh $ES_SERVERLESS_IMAGE
+
+  - wait: ~
+
+  - label: 'Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+BASE_ES_SERVERLESS_REPO=docker.elastic.co/elasticsearch-ci/elasticsearch-serverless
+TARGET_IMAGE=docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+ES_SERVERLESS_BUCKET=kibana-ci-es-serverless-images
+MANIFEST_FILE_NAME=latest-verified.json
+
+SOURCE_IMAGE_OR_TAG=$1
+if [[ $SOURCE_IMAGE_OR_TAG =~ :[a-zA-Z_-]+$ ]]; then
+  # $SOURCE_IMAGE_OR_TAG was a full image
+  SOURCE_IMAGE=$SOURCE_IMAGE_OR_TAG
+else
+  # $SOURCE_IMAGE_OR_TAG was an image tag
+  SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
+fi
+
+echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+
+echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+docker pull "$SOURCE_IMAGE"
+docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"
+docker push "$TARGET_IMAGE"
+
+ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE")
+ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]')
+
+docker logout docker.elastic.co
+
+echo "Image push to $TARGET_IMAGE successful."
+echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
+
+MANIFEST_UPLOAD_PATH="Skipped"
+if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Uploading latest-verified manifest to GCS"
+  cat << EOT >> $MANIFEST_FILE_NAME
+{
+  "build_url": "$BUILDKITE_BUILD_URL",
+  "kibana_commit": "$BUILDKITE_COMMIT",
+  "kibana_branch": "$BUILDKITE_BRANCH",
+  "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG",
+  "elasticsearch_serverless_image_url: "$SOURCE_IMAGE",
+  "elasticsearch_serverless_commit": "TODO: this currently can't be decided",
+  "elasticsearch_commit": "$ELASTIC_COMMIT_HASH",
+  "created_at": "`date`",
+  "timestamp": "`FORCE_COLOR=0 node -p 'Date.now()'`"
+}
+EOT
+
+  gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" \
+    cp $MANIFEST_FILE_NAME "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
+
+elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
+elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"
+fi
+
+echo "--- Annotating build with info"
+cat << EOT | buildkite-agent annotate --style "success"
+  <h2>Promotion successful!</h2>
+  <br/>New image: $TARGET_IMAGE
+  <br/>Source image: $SOURCE_IMAGE
+  <br/>Kibana commit: <a href="https://github.com/elastic/kibana/commit/$BUILDKITE_COMMIT">$BUILDKITE_COMMIT</a>
+  <br/>Elasticsearch commit: <a href="https://github.com/elastic/elasticsearch/commit/$ELASTIC_COMMIT_HASH">$ELASTIC_COMMIT_HASH</a>
+  <br/>Manifest file: $MANIFEST_UPLOAD_PATH
+EOT

packages/kbn-dev-cli-runner/src/flags.ts

@@ -27,6 +27,7 @@ export interface FlagOptions {
   allowUnexpected?: boolean;
   guessTypesForUnexpectedFlags?: boolean;
   help?: string;
+  examples?: string;
   alias?: { [key: string]: string | string[] };
   boolean?: string[];
   string?: string[];
@@ -47,6 +48,7 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
     },
 
     help: local.help,
+    examples: local.examples,
 
     allowUnexpected: !!(global.allowUnexpected || local.allowUnexpected),
     guessTypesForUnexpectedFlags: !!(global.allowUnexpected || local.allowUnexpected),

packages/kbn-dev-cli-runner/src/help.ts

@@ -36,11 +36,13 @@ export function getHelp({
   usage,
   flagHelp,
   defaultLogLevel,
+  examples,
 }: {
   description?: string;
   usage?: string;
   flagHelp?: string;
   defaultLogLevel?: string;
+  examples?: string;
 }) {
   const optionHelp = joinAndTrimLines(
     dedent(flagHelp || ''),
@@ -48,13 +50,17 @@ export function getHelp({
     GLOBAL_FLAGS
   );
 
+  const examplesHelp = examples ? joinAndTrimLines('Examples:', examples) : '';
+
   return `
   ${dedent(usage || '') || DEFAULT_GLOBAL_USAGE}
 
   ${indent(dedent(description || 'Runs a dev task'), 2)}
 
   Options:
-    ${indent(optionHelp, 4)}\n\n`;
+    ${indent(optionHelp, 4)}
+${examplesHelp ? `\n  ${indent(examplesHelp, 4)}` : ''}
+`;
 }
 
 export function getCommandLevelHelp({

packages/kbn-dev-cli-runner/src/run.ts

@@ -50,6 +50,7 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
     usage: options.usage,
     flagHelp: options.flags?.help,
     defaultLogLevel: options.log?.defaultLevel,
+    examples: options.flags?.examples,
   });
 
   if (flags.help) {

packages/kbn-es/src/cli_commands/serverless.ts

@@ -13,9 +13,8 @@ import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
 import { Cluster } from '../cluster';
 import {
-  SERVERLESS_REPO,
-  SERVERLESS_TAG,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_REPO_ELASTICSEARCH,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   DEFAULT_PORT,
   ServerlessOptions,
 } from '../utils';
@@ -28,9 +27,8 @@ export const serverless: Command = {
     return dedent`
     Options:
 
-      --tag               Image tag of ES serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}]
-      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}]
-      
+      --tag               Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH}
+      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}]
       --background        Start ES serverless without attaching to the first node's logs
       --basePath          Path to the directory where the ES cluster will store data
       --clean             Remove existing file system object store before running
@@ -39,14 +37,14 @@ export const serverless: Command = {
       --ssl               Enable HTTP SSL on the ES cluster
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
-      
+
       -E                  Additional key=value settings to pass to ES
       -F                  Absolute paths for files to mount into containers
 
     Examples:
 
-      es serverless --tag git-fec36430fba2-x86_64
-      es serverless --image docker.elastic.co/repo:tag
+      es serverless --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64
+      es serverless --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
     `;
   },
   run: async (defaults = {}) => {

packages/kbn-es/src/utils/docker.test.ts

@@ -23,7 +23,7 @@ import {
   runDockerContainer,
   runServerlessCluster,
   runServerlessEsNode,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   setupServerlessVolumes,
   stopServerlessCluster,
   teardownServerlessClusterSync,
@@ -451,7 +451,7 @@ describe('runServerlessEsNode()', () => {
   const node = {
     params: ['--env', 'foo=bar', '--volume', 'foo/bar'],
     name: 'es01',
-    image: SERVERLESS_IMG,
+    image: ES_SERVERLESS_DEFAULT_IMAGE,
   };
 
   test('should call the correct Docker command', async () => {
@@ -462,7 +462,7 @@ describe('runServerlessEsNode()', () => {
     expect(execa.mock.calls[0][0]).toEqual('docker');
     expect(execa.mock.calls[0][1]).toEqual(
       expect.arrayContaining([
-        SERVERLESS_IMG,
+        ES_SERVERLESS_DEFAULT_IMAGE,
         ...node.params,
         '--name',
         node.name,
@@ -530,7 +530,9 @@ describe('teardownServerlessClusterSync()', () => {
     teardownServerlessClusterSync(log, defaultOptions);
 
     expect(execa.commandSync.mock.calls).toHaveLength(2);
-    expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG));
+    expect(execa.commandSync.mock.calls[0][0]).toEqual(
+      expect.stringContaining(ES_SERVERLESS_DEFAULT_IMAGE)
+    );
     expect(execa.commandSync.mock.calls[1][0]).toEqual(`docker kill ${nodes.join(' ')}`);
   });
 

packages/kbn-es/src/utils/docker.ts

@@ -38,9 +38,12 @@ import {
 import { SYSTEM_INDICES_SUPERUSER } from './native_realm';
 import { waitUntilClusterReady } from './wait_until_cluster_ready';
 
-interface BaseOptions {
-  tag?: string;
+interface ImageOptions {
   image?: string;
+  tag?: string;
+}
+
+interface BaseOptions extends ImageOptions {
   port?: number;
   ssl?: boolean;
   /** Kill running cluster before starting a new cluster  */
@@ -106,9 +109,10 @@ export const DOCKER_REPO = `${DOCKER_REGISTRY}/elasticsearch/elasticsearch`;
 export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`;
 export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`;
 
-export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
-export const SERVERLESS_TAG = 'latest';
-export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`;
+export const ES_SERVERLESS_REPO_KIBANA = `${DOCKER_REGISTRY}/kibana-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_REPO_ELASTICSEARCH = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_LATEST_VERIFIED_TAG = 'latest-verified';
+export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SERVERLESS_LATEST_VERIFIED_TAG}`;
 
 // See for default cluster settings
 // https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts
@@ -275,7 +279,12 @@ export function resolveDockerImage({
   image,
   repo,
   defaultImg,
-}: (ServerlessOptions | DockerOptions) & { repo: string; defaultImg: string }) {
+}: {
+  tag?: string;
+  image?: string;
+  repo: string;
+  defaultImg: string;
+}) {
   if (image) {
     if (!image.includes(DOCKER_REGISTRY)) {
       throw createCliError(
@@ -525,11 +534,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
 /**
  * Resolve the Serverless ES image based on defaults and CLI options
  */
-function getServerlessImage(options: ServerlessOptions) {
+function getServerlessImage({ image, tag }: ImageOptions) {
   return resolveDockerImage({
-    ...options,
-    repo: SERVERLESS_REPO,
-    defaultImg: SERVERLESS_IMG,
+    image,
+    tag,
+    repo: ES_SERVERLESS_REPO_ELASTICSEARCH,
+    defaultImg: ES_SERVERLESS_DEFAULT_IMAGE,
   });
 }
 
@@ -573,7 +583,10 @@ function getESClient(clientOptions: ClientOptions): Client {
  * Runs an ES Serverless Cluster through Docker
  */
 export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) {
-  const image = getServerlessImage(options);
+  const image = getServerlessImage({
+    image: options.image,
+    tag: options.tag,
+  });
   await setupDocker({ log, image, options });
 
   const volumeCmd = await setupServerlessVolumes(log, options);
@@ -686,8 +699,13 @@ export function teardownServerlessClusterSync(log: ToolingLog, options: Serverle
 /**
  * Resolve the Elasticsearch image based on defaults and CLI options
  */
-function getDockerImage(options: DockerOptions) {
-  return resolveDockerImage({ ...options, repo: DOCKER_REPO, defaultImg: DOCKER_IMG });
+function getDockerImage({ image, tag }: ImageOptions) {
+  return resolveDockerImage({
+    image,
+    tag,
+    repo: DOCKER_REPO,
+    defaultImg: DOCKER_IMG,
+  });
 }
 
 /**
@@ -713,7 +731,10 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions
   let image;
 
   if (!options.dockerCmd) {
-    image = getDockerImage(options);
+    image = getDockerImage({
+      image: options.image,
+      tag: options.tag,
+    });
     await setupDocker({ log, image, options });
   }
 

packages/kbn-test/src/es/es_test_config.ts

@@ -27,6 +27,10 @@ class EsTestConfig {
     return process.env.TEST_ES_FROM || 'snapshot';
   }
 
+  getESServerlessImage() {
+    return process.env.TEST_ES_SERVERLESS_IMAGE;
+  }
+
   getTransportPort() {
     return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
   }

packages/kbn-test/src/es/test_es_cluster.ts

@@ -70,6 +70,10 @@ export interface CreateTestEsClusterOptions {
    */
   esArgs?: string[];
   esFrom?: string;
+  esServerlessOptions?: {
+    image?: string;
+    tag?: string;
+  };
   esJavaOpts?: string;
   /**
    * License to run your cluster under. Keep in mind that a `trial` license
@@ -164,6 +168,7 @@ export function createTestEsCluster<
     writeLogsToPath,
     basePath = Path.resolve(REPO_ROOT, '.es'),
     esFrom = esTestConfig.getBuildFrom(),
+    esServerlessOptions,
     dataArchive,
     nodes = [{ name: 'node-01' }],
     esArgs: customEsArgs = [],
@@ -236,9 +241,11 @@ export function createTestEsCluster<
       } else if (esFrom === 'snapshot') {
         installPath = (await firstNode.installSnapshot(config)).installPath;
       } else if (esFrom === 'serverless') {
-        return await firstNode.runServerless({
+        await firstNode.runServerless({
           basePath,
           esArgs: customEsArgs,
+          image: esServerlessOptions?.image,
+          tag: esServerlessOptions?.tag,
           port,
           clean: true,
           background: true,
@@ -247,6 +254,7 @@ export function createTestEsCluster<
           kill: true, // likely don't need this but avoids any issues where the ESS cluster wasn't cleaned up
           waitForReady: true,
         });
+        return;
       } else if (Path.isAbsolute(esFrom)) {
         installPath = esFrom;
       } else {
@@ -275,9 +283,9 @@ export function createTestEsCluster<
           });
         }
 
-        nodeStartPromises.push(async () => {
+        nodeStartPromises.push(() => {
           log.info(`[es] starting node ${node.name} on port ${nodePort}`);
-          return await this.nodes[i].start(installPath, {
+          return this.nodes[i].start(installPath, {
             password: config.password,
             esArgs: assignArgs(esArgs, overriddenArgs),
             esJavaOpts,
@@ -292,7 +300,7 @@ export function createTestEsCluster<
         });
       }
 
-      await Promise.all(extractDirectoryPromises.map(async (extract) => await extract()));
+      await Promise.all(extractDirectoryPromises.map((extract) => extract()));
       for (const start of nodeStartPromises) {
         await start();
       }

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts

@@ -12,11 +12,12 @@ import getPort from 'get-port';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ArtifactLicense } from '@kbn/es';
 import type { Config } from '../../functional_test_runner';
-import { createTestEsCluster } from '../../es';
+import { createTestEsCluster, esTestConfig } from '../../es';
 
 interface RunElasticsearchOptions {
   log: ToolingLog;
   esFrom?: string;
+  esServerlessImage?: string;
   config: Config;
   onEarlyExit?: (msg: string) => void;
   logsDir?: string;
@@ -32,6 +33,7 @@ type EsConfig = ReturnType<typeof getEsConfig>;
 function getEsConfig({
   config,
   esFrom = config.get('esTestCluster.from'),
+  esServerlessImage,
 }: RunElasticsearchOptions) {
   const ssl = !!config.get('esTestCluster.ssl');
   const license: ArtifactLicense = config.get('esTestCluster.license');
@@ -50,6 +52,8 @@ function getEsConfig({
   const serverless: boolean = config.get('serverless');
   const files: string[] | undefined = config.get('esTestCluster.files');
 
+  const esServerlessOptions = getESServerlessOptions(esServerlessImage, config);
+
   return {
     ssl,
     license,
@@ -57,6 +61,7 @@ function getEsConfig({
     esJavaOpts,
     isSecurityEnabled,
     esFrom,
+    esServerlessOptions,
     port,
     password,
     dataArchive,
@@ -129,6 +134,7 @@ async function startEsNode({
     clusterName: `cluster-${name}`,
     esArgs: config.esArgs,
     esFrom: config.esFrom,
+    esServerlessOptions: config.esServerlessOptions,
     esJavaOpts: config.esJavaOpts,
     license: config.license,
     password: config.password,
@@ -153,3 +159,23 @@ async function startEsNode({
 
   return cluster;
 }
+
+function getESServerlessOptions(esServerlessImageFromArg: string | undefined, config: Config) {
+  const esServerlessImageUrlOrTag =
+    esServerlessImageFromArg ||
+    esTestConfig.getESServerlessImage() ||
+    (config.has('esTestCluster.esServerlessImage') &&
+      config.get('esTestCluster.esServerlessImage'));
+
+  if (esServerlessImageUrlOrTag) {
+    if (esServerlessImageUrlOrTag.includes(':')) {
+      return {
+        image: esServerlessImageUrlOrTag,
+      };
+    } else {
+      return {
+        tag: esServerlessImageUrlOrTag,
+      };
+    }
+  }
+}

packages/kbn-test/src/functional_tests/run_tests/cli.ts

@@ -26,6 +26,7 @@ export function runTestsCli() {
     {
       description: `Run Functional Tests`,
       usage: `
+      Usage:
         node scripts/functional_tests --help
         node scripts/functional_tests [--config <file1> [--config <file2> ...]]
         node scripts/functional_tests [options] [-- --<other args>]

packages/kbn-test/src/functional_tests/run_tests/flags.test.ts

@@ -42,6 +42,7 @@ describe('parse runTest flags', () => {
         ],
         "dryRun": false,
         "esFrom": undefined,
+        "esServerlessImage": undefined,
         "esVersion": <EsVersion 9.9.9>,
         "grep": undefined,
         "installDir": undefined,

packages/kbn-test/src/functional_tests/run_tests/flags.ts

@@ -23,6 +23,7 @@ export const FLAG_OPTIONS: FlagOptions = {
     'config',
     'journey',
     'esFrom',
+    'esServerlessImage',
     'kibana-install-dir',
     'grep',
     'include-tag',
@@ -37,6 +38,7 @@ export const FLAG_OPTIONS: FlagOptions = {
     --config             Define a FTR config that should be executed. Can be specified multiple times
     --journey            Define a Journey that should be executed. Can be specified multiple times
     --esFrom             Build Elasticsearch from source or run snapshot or serverless. Default: $TEST_ES_FROM or "snapshot"
+    --esServerlessImage  When 'esFrom' is "serverless", this argument will be interpreted either as a tag within the ES Serverless repo, OR a full docker image path.
     --include-tag        Tags that suites must include to be run, can be included multiple times
     --exclude-tag        Tags that suites must NOT include to be run, can be included multiple times
     --include            Files that must included to be run, can be included multiple times
@@ -50,6 +52,13 @@ export const FLAG_OPTIONS: FlagOptions = {
     --updateSnapshots    Replace inline and file snapshots with whatever is generated from the test
     --updateAll, -u      Replace both baseline screenshots and snapshots
   `,
+  examples: `
+Run the latest verified, kibana-compatible ES Serverless image:
+  node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+Run with a specific ES Serverless tag from the docker.elastic.co/elasticsearch-ci/elasticsearch-serverless repo:
+  node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage git-fec36430fba2
+  `,
 };
 
 export function parseFlags(flags: FlagsReader) {
@@ -75,6 +84,7 @@ export function parseFlags(flags: FlagsReader) {
       ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
       : undefined,
     esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']),
+    esServerlessImage: flags.string('esServerlessImage'),
     installDir: flags.path('kibana-install-dir'),
     grep: flags.string('grep'),
     suiteTags: {