[scout] add dynamic ci pipeline to run tests (#211797)

## Summary

closes https://github.com/elastic/kibana/issues/211592

This PR improves the way we run scout tests by discovering all the
plugins that have the scout tests and run tests in multiple workers:

<img width="1586" alt="image"
src="https://github.com/user-attachments/assets/4936ab50-fefb-470c-af3a-21263b58143f"
/>

How it works:

1. Run search script to find _all existing_ scout playwright config
files across kibana repo
2. Save results into `.scout/scout_playwright_configs.json` file, that
will be used as source to run configs in individual jobs per plugin.
Upload it as BK artifact.
3. Spin up job for each plugin mentioned in
`scout_playwright_configs.json`
4. In each individual job use `scout_playwright_configs.json` and get
configs for specific plugin, use worker with 8 vcpus where tests are run
in parallel (`usesParallelWorkers` prop)
While running configs 1 by 1 collect command exit code with the
following rules:
- configs run passed => exit code `0` , final status remains `0`
- config has no tests => exit code `2`, put config name into
`configsWithoutTests` group to push BK annotation later, change exit
status to `0` - we accept configs without tests at current stage
- config run fails => exit code `1`, final status changed to `1` and job
will fail in the end; put config name into `failedConfigs` group to push
BK annotation later

<img width="1564" alt="Screenshot 2025-02-21 at 14 34 16"
src="https://github.com/user-attachments/assets/06e9298d-466c-46bb-8e85-3d691a40850a"
/>

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -561,3 +561,57 @@ export async function pickTestGroupRunOrder() {
     ].flat()
   );
 }
+
+export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
+  const bk = new BuildkiteClient();
+  const envFromlabels: Record<string, string> = collectEnvFromLabels();
+
+  if (!Fs.existsSync(scoutConfigsPath)) {
+    throw new Error(`Scout configs file not found at ${scoutConfigsPath}`);
+  }
+
+  const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8'));
+  const pluginsWithScoutConfigs: string[] = Object.keys(rawScoutConfigs);
+
+  if (pluginsWithScoutConfigs.length === 0) {
+    // no scout configs found, nothing to need to upload steps
+    return;
+  }
+
+  const scoutGroups = pluginsWithScoutConfigs.map((plugin) => ({
+    title: plugin,
+    key: plugin,
+    usesParallelWorkers: rawScoutConfigs[plugin].usesParallelWorkers,
+    group: rawScoutConfigs[plugin].group,
+  }));
+
+  // upload the step definitions to Buildkite
+  bk.uploadSteps(
+    [
+      {
+        group: 'Scout Configs',
+        key: 'scout-configs',
+        depends_on: ['build'],
+        steps: scoutGroups.map(
+          ({ title, key, group, usesParallelWorkers }): BuildkiteStep => ({
+            label: `Scout: [ ${group} / ${title} ] plugin`,
+            command: getRequiredEnv('SCOUT_CONFIGS_SCRIPT'),
+            timeout_in_minutes: 60,
+            agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+            env: {
+              SCOUT_CONFIG_GROUP_KEY: key,
+              SCOUT_CONFIG_GROUP_TYPE: group,
+              ...envFromlabels,
+            },
+            retry: {
+              automatic: [
+                { exit_status: '-1', limit: 1 },
+                { exit_status: '*', limit: 0 },
+              ],
+            },
+          })
+        ),
+      },
+    ].flat()
+  );
+}

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -68,21 +68,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
-    label: 'Scout UI Tests'
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-8
-      preemptible: true
-    depends_on: build
+      machineType: n2-standard-2
+      diskSizeGb: 75
+    timeout_in_minutes: 10
     env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
       PING_SLACK_TEAM: "@appex-qa-team"
-    timeout_in_minutes: 60
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 2
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/scout_tests.yml

@@ -0,0 +1,13 @@
+steps:
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
+    agents:
+      machineType: n2-standard-2
+      diskSizeGb: 75
+    timeout_in_minutes: 10
+    env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/scout_ui_tests.yml

@@ -1,18 +0,0 @@
-steps:
-  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
-    label: 'Scout UI Tests'
-    agents:
-      machineType: n2-standard-8
-      preemptible: true
-    depends_on:
-      - build
-      - quick_checks
-      - checks
-      - linting
-      - linting_with_types
-      - check_types
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 2

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -458,13 +458,18 @@ const getPipeline = (filename: string, removeSteps = true) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/platform\/plugins\/private\/discover_enhanced\/ui_tests/,
-        /^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
         /^src\/platform\/packages\/shared\/kbn-scout/,
+        /^src\/platform\/packages\/private\/kbn-scout-info/,
+        /^src\/platform\/packages\/private\/kbn-scout-reporting/,
+        /^x-pack\/platform\/plugins\/shared\/maps/,
+        /^x-pack\/platform\/plugins\/private\/discover_enhanced/,
+        /^x-pack\/solutions\/observability\/packages\/kbn-scout-oblt/,
+        /^x-pack\/solutions\/observability\/plugins\/apm/,
+        /^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
       ])) ||
       GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
     ) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
     }
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));

.buildkite/scripts/steps/functional/scout_ui_tests.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/scripts/steps/functional/common.sh
-
-export JOB=kibana-scout-ui-tests
-
-KIBANA_DIR="$KIBANA_BUILD_LOCATION"
-
-run_tests() {
-  local suit_name=$1
-  local config_path=$2
-  local run_mode=$3
-
-  echo "--- $suit_name ($run_mode) UI Tests"
-  if ! node scripts/scout run-tests "$run_mode" --config "$config_path" --kibana-install-dir "$KIBANA_DIR"; then
-    echo "$suit_name: failed"
-    EXIT_CODE=1
-  else
-    echo "$suit_name: passed"
-  fi
-}
-
-EXIT_CODE=0
-
-# Discovery Enhanced && Maps
-for run_mode in "--stateful" "--serverless=es" "--serverless=oblt" "--serverless=security"; do
-  run_tests "Discovery Enhanced: Parallel Workers" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts" "$run_mode"
-  run_tests "Discovery Enhanced" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts" "$run_mode"
-  run_tests "Maps" "x-pack/platform/plugins/shared/maps/ui_tests/playwright.config.ts" "$run_mode"
-done
-
-# Observability Onboarding
-for run_mode in "--stateful" "--serverless=oblt"; do
-  run_tests "Observability Onboarding: Parallel Workers" "x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/parallel.playwright.config.ts" "$run_mode"
-  # Disabled while we don't have any tests under the config
-  # run_tests "Observability Onboarding" "x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts" "$run_mode"
-done
-
-
-exit $EXIT_CODE

.buildkite/scripts/steps/test/scout_configs.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
+SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
+SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
+
+if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
+  echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
+  exit 1
+fi
+
+if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
+  echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
+  exit 1
+fi
+
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
+export JOB="$SCOUT_CONFIG_GROUP_KEY"
+
+FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
+
+configs=""
+group=$SCOUT_CONFIG_GROUP_TYPE
+
+# The first retry should only run the configs that failed in the previous attempt
+# Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
+if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
+  configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
+  if [[ "$configs" ]]; then
+    echo "--- Retrying only failed configs"
+    echo "$configs"
+  fi
+fi
+
+if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+  echo "--- downloading scout test configuration"
+  download_artifact scout_playwright_configs.json .
+  configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
+fi
+
+if [ "$configs" == "" ]; then
+  echo "unable to determine configs to run"
+  exit 1
+fi
+
+# Define run modes based on group
+declare -A RUN_MODES
+RUN_MODES["platform"]="--stateful --serverless=es --serverless=oblt --serverless=security"
+RUN_MODES["observability"]="--stateful --serverless=oblt"
+RUN_MODES["search"]="--stateful --serverless=es"
+RUN_MODES["security"]="--stateful --serverless=security"
+
+# Determine valid run modes for the group
+RUN_MODE_LIST=${RUN_MODES[$group]}
+
+if [[ -z "$RUN_MODE_LIST" ]]; then
+  echo "Unknown group: $group"
+  exit 1
+fi
+
+results=()
+failedConfigs=()
+configWithoutTests=()
+passedConfigs=()
+
+FINAL_EXIT_CODE=0
+
+# Run tests for each config
+while read -r config_path; do
+  if [[ -z "$config_path" ]]; then
+    continue
+  fi
+
+  for mode in $RUN_MODE_LIST; do
+    echo "--- Running tests: $config_path ($mode)"
+
+    # prevent non-zero exit code from breaking the loop
+    set +e;
+    node scripts/scout run-tests "$mode" --config "$config_path" --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+    EXIT_CODE=$?
+    set -e;
+
+    if [[ $EXIT_CODE -eq 2 ]]; then
+      configWithoutTests+=("$config_path ($mode)")
+    elif [[ $EXIT_CODE -ne 0 ]]; then
+      failedConfigs+=("$config_path ($mode) ❌")
+      FINAL_EXIT_CODE=10  # Ensure we exit with failure if any test fails with (exit code 10 to match FTR)
+    else
+      results+=("$config_path ($mode) ✅")
+    fi
+  done
+done <<< "$configs"
+
+echo "--- Scout Test Run Complete: Summary"
+echo "✅ Passed: ${#results[@]}"
+echo "⚠️ Configs without tests: ${#configWithoutTests[@]}"
+echo "❌ Failed: ${#failedConfigs[@]}"
+
+if [[ ${#results[@]} -gt 0 ]]; then
+  echo "✅ Successful tests:"
+  printf '%s\n' "${results[@]}"
+fi
+
+if [[ ${#configWithoutTests[@]} -gt 0 ]]; then
+{
+  echo "Scout Playwright configs without tests:"
+  echo ""
+  for config in "${configWithoutTests[@]}"; do
+    echo "- $config"
+  done
+} | buildkite-agent annotate --style "warning" --context "no-tests"
+fi
+
+if [[ ${#failedConfigs[@]} -gt 0 ]]; then
+  echo "❌ Failed tests:"
+  printf '%s\n' "${failedConfigs[@]}"
+  buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
+fi
+
+exit $FINAL_EXIT_CODE  # Exit with 10 only if there were config failures

.buildkite/scripts/steps/test/scout_test_run_builder.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+.buildkite/scripts/bootstrap.sh
+
+echo '--- Discover Playwright Configs and upload to Buildkite artifacts'
+node scripts/scout discover-playwright-configs --save
+cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
+buildkite-agent artifact upload "scout_playwright_configs.json"
+
+echo '--- Scout Test Run Builder'
+ts-node "$(dirname "${0}")/scout_test_run_builder.ts"

.buildkite/scripts/steps/test/scout_test_run_builder.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import path from 'path';
+import { CiStats } from '#pipeline-utils';
+
+(async () => {
+  try {
+    const scoutConfigsPath = path.resolve(
+      process.cwd(),
+      '.scout',
+      'test_configs',
+      'scout_playwright_configs.json'
+    );
+    await CiStats.pickScoutTestGroupRunOrder(scoutConfigsPath);
+  } catch (ex) {
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();

src/platform/packages/private/kbn-scout-info/src/paths.ts

@@ -17,3 +17,10 @@ export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers');
 
 // Reporting
 export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports');
+
+// Scout playwright configs
+export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
+  SCOUT_OUTPUT_ROOT,
+  'test_configs',
+  'scout_playwright_configs.json'
+);

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -9,8 +9,8 @@
 
 import fs from 'fs';
 import { Command } from '@kbn/dev-cli-runner';
-import { SCOUT_OUTPUT_ROOT } from '@kbn/scout-info';
-import { resolve } from 'path';
+import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
+import path from 'path';
 import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config';
 import { measurePerformance } from '../common';
 
@@ -44,13 +44,19 @@ export const discoverPlaywrightConfigs: Command<void> = {
         ? 'No Playwright config files found'
         : `Found Playwright config files in '${pluginsMap.size}' plugins`;
 
-    if (pluginsMap.size > 0 && flagsReader.boolean('save')) {
-      const scoutConfigsFilePath = resolve(SCOUT_OUTPUT_ROOT, 'scout_playwright_configs.json');
+    if (flagsReader.boolean('save')) {
+      const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
+
+      if (!fs.existsSync(dirPath)) {
+        fs.mkdirSync(dirPath, { recursive: true });
+      }
+
       fs.writeFileSync(
-        scoutConfigsFilePath,
+        SCOUT_PLAYWRIGHT_CONFIGS_PATH,
         JSON.stringify(Object.fromEntries(pluginsMap), null, 2)
       );
-      log.info(`${finalMessage}. Saved to '${scoutConfigsFilePath}'`);
+
+      log.info(`${finalMessage}. Saved to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`);
       return;
     }
 

src/platform/packages/shared/kbn-scout/src/playwright/page_objects/discover_app.ts

@@ -33,6 +33,7 @@ export class DiscoverApp {
     await this.page.testSubj.hover('discoverNewButton');
     await this.page.testSubj.click('discoverNewButton');
     await this.page.testSubj.hover('unifiedFieldListSidebar__toggle-collapse'); // cancel tooltips
+    await this.page.waitForLoadingIndicatorHidden();
   }
 
   async saveSearch(name: string) {