[8.x] [scout] add dynamic ci pipeline to run tests (#211797) (#214094)

# Backport This will backport the following commits from `main` to `8.x`: - [[scout] add dynamic ci pipeline to run tests (#211797)](https://github.com/elastic/kibana/pull/211797)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
2025-04-23 09:19:04 -04:00 · 2025-03-12 14:28:03 +01:00 · 2025-03-12 14:28:03 +01:00 · 2c5600bd01
commit 2c5600bd01
parent acf9ca56f6
12 changed files with 271 additions and 75 deletions
--- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
+++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
@ -562,3 +562,57 @@ export async function pickTestGroupRunOrder() {
    ].flat()
  );
 }
+
+export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
+  const bk = new BuildkiteClient();
+  const envFromlabels: Record<string, string> = collectEnvFromLabels();
+
+  if (!Fs.existsSync(scoutConfigsPath)) {
+    throw new Error(`Scout configs file not found at ${scoutConfigsPath}`);
+  }
+
+  const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8'));
+  const pluginsWithScoutConfigs: string[] = Object.keys(rawScoutConfigs);
+
+  if (pluginsWithScoutConfigs.length === 0) {
+    // no scout configs found, nothing to need to upload steps
+    return;
+  }
+
+  const scoutGroups = pluginsWithScoutConfigs.map((plugin) => ({
+    title: plugin,
+    key: plugin,
+    usesParallelWorkers: rawScoutConfigs[plugin].usesParallelWorkers,
+    group: rawScoutConfigs[plugin].group,
+  }));
+
+  // upload the step definitions to Buildkite
+  bk.uploadSteps(
+    [
+      {
+        group: 'Scout Configs',
+        key: 'scout-configs',
+        depends_on: ['build'],
+        steps: scoutGroups.map(
+          ({ title, key, group, usesParallelWorkers }): BuildkiteStep => ({
+            label: `Scout: [ ${group} / ${title} ] plugin`,
+            command: getRequiredEnv('SCOUT_CONFIGS_SCRIPT'),
+            timeout_in_minutes: 60,
+            agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+            env: {
+              SCOUT_CONFIG_GROUP_KEY: key,
+              SCOUT_CONFIG_GROUP_TYPE: group,
+              ...envFromlabels,
+            },
+            retry: {
+              automatic: [
+                { exit_status: '-1', limit: 1 },
+                { exit_status: '*', limit: 0 },
+              ],
+            },
+          })
+        ),
+      },
+    ].flat()
+  );
+}
--- a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml
+++ b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml
@ -68,21 +68,20 @@ steps:
        - exit_status: '*'
          limit: 1

-  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
-    label: 'Scout UI Tests'
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
    agents:
      image: family/kibana-ubuntu-2004
      imageProject: elastic-images-prod
      provider: gcp
-      machineType: n2-standard-8
+      machineType: n2-standard-2
      preemptible: true
    depends_on: build
+    timeout_in_minutes: 10
    env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
      PING_SLACK_TEAM: "@appex-qa-team"
-    timeout_in_minutes: 60
    retry:
      automatic:
-        - exit_status: '-1'
-          limit: 2
        - exit_status: '*'
          limit: 1
--- a/.buildkite/pipelines/pull_request/scout_tests.yml
+++ b/.buildkite/pipelines/pull_request/scout_tests.yml
@ -0,0 +1,12 @@
+steps:
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
+    agents:
+      machineType: n2-standard-2
+    timeout_in_minutes: 10
+    env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
--- a/.buildkite/pipelines/pull_request/scout_ui_tests.yml
+++ b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
@ -1,18 +0,0 @@
-steps:
-  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
-    label: 'Scout UI Tests'
-    agents:
-      machineType: n2-standard-8
-      preemptible: true
-    depends_on:
-      - build
-      - quick_checks
-      - checks
-      - linting
-      - linting_with_types
-      - check_types
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 2
--- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts
+++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
@ -395,13 +395,18 @@ const getPipeline = (filename: string, removeSteps = true) => {

    if (
      (await doAnyChangesMatch([
-        /^x-pack\/platform\/plugins\/private\/discover_enhanced\/ui_tests/,
-        /^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
        /^src\/platform\/packages\/shared\/kbn-scout/,
+        /^src\/platform\/packages\/private\/kbn-scout-info/,
+        /^src\/platform\/packages\/private\/kbn-scout-reporting/,
+        /^x-pack\/platform\/plugins\/shared\/maps/,
+        /^x-pack\/platform\/plugins\/private\/discover_enhanced/,
+        /^x-pack\/solutions\/observability\/packages\/kbn-scout-oblt/,
+        /^x-pack\/solutions\/observability\/plugins\/apm/,
+        /^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
      ])) ||
      GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
    ) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
    }

    pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/scripts/steps/functional/common.sh
-
-export JOB=kibana-scout-ui-tests
-
-KIBANA_DIR="$KIBANA_BUILD_LOCATION"
-
-run_tests() {
-  local suit_name=$1
-  local config_path=$2
-  local run_mode=$3
-
-  echo "--- $suit_name ($run_mode) UI Tests"
-  if ! node scripts/scout run-tests "$run_mode" --config "$config_path" --kibana-install-dir "$KIBANA_DIR"; then
-    echo "$suit_name: failed"
-    EXIT_CODE=1
-  else
-    echo "$suit_name: passed"
-  fi
-}
-
-EXIT_CODE=0
-
-# Discovery Enhanced && Maps
-for run_mode in "--stateful"; do
-  run_tests "Discovery Enhanced: Parallel Workers" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts" "$run_mode"
-  run_tests "Discovery Enhanced" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts" "$run_mode"
-  run_tests "Maps" "x-pack/platform/plugins/shared/maps/ui_tests/playwright.config.ts" "$run_mode"
-done
-
-# Observability Onboarding
-for run_mode in "--stateful"; do
-  run_tests "Observability Onboarding: Parallel Workers" "x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/parallel.playwright.config.ts" "$run_mode"
-  # Disabled while we don't have any tests under the config
-  # run_tests "Observability Onboarding" "x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts" "$run_mode"
-done
-
-
-exit $EXIT_CODE
--- a/.buildkite/scripts/steps/test/scout_configs.sh
+++ b/.buildkite/scripts/steps/test/scout_configs.sh
@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
+SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
+SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
+
+if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
+  echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
+  exit 1
+fi
+
+if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
+  echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
+  exit 1
+fi
+
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
+export JOB="$SCOUT_CONFIG_GROUP_KEY"
+
+FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
+
+configs=""
+group=$SCOUT_CONFIG_GROUP_TYPE
+
+# The first retry should only run the configs that failed in the previous attempt
+# Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
+if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
+  configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
+  if [[ "$configs" ]]; then
+    echo "--- Retrying only failed configs"
+    echo "$configs"
+  fi
+fi
+
+if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+  echo "--- downloading scout test configuration"
+  download_artifact scout_playwright_configs.json .
+  configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
+fi
+
+if [ "$configs" == "" ]; then
+  echo "unable to determine configs to run"
+  exit 1
+fi
+
+# Define run modes based on group
+declare -A RUN_MODES
+RUN_MODES["platform"]="--stateful"
+RUN_MODES["observability"]="--stateful"
+RUN_MODES["search"]="--stateful"
+RUN_MODES["security"]="--stateful"
+
+# Determine valid run modes for the group
+RUN_MODE_LIST=${RUN_MODES[$group]}
+
+if [[ -z "$RUN_MODE_LIST" ]]; then
+  echo "Unknown group: $group"
+  exit 1
+fi
+
+results=()
+failedConfigs=()
+configWithoutTests=()
+passedConfigs=()
+
+FINAL_EXIT_CODE=0
+
+# Run tests for each config
+while read -r config_path; do
+  if [[ -z "$config_path" ]]; then
+    continue
+  fi
+
+  for mode in $RUN_MODE_LIST; do
+    echo "--- Running tests: $config_path ($mode)"
+
+    # prevent non-zero exit code from breaking the loop
+    set +e;
+    node scripts/scout run-tests "$mode" --config "$config_path" --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+    EXIT_CODE=$?
+    set -e;
+
+    if [[ $EXIT_CODE -eq 2 ]]; then
+      configWithoutTests+=("$config_path ($mode)")
+    elif [[ $EXIT_CODE -ne 0 ]]; then
+      failedConfigs+=("$config_path ($mode) ❌")
+      FINAL_EXIT_CODE=10  # Ensure we exit with failure if any test fails with (exit code 10 to match FTR)
+    else
+      results+=("$config_path ($mode) ✅")
+    fi
+  done
+done <<< "$configs"
+
+echo "--- Scout Test Run Complete: Summary"
+echo "✅ Passed: ${#results[@]}"
+echo "⚠️ Configs without tests: ${#configWithoutTests[@]}"
+echo "❌ Failed: ${#failedConfigs[@]}"
+
+if [[ ${#results[@]} -gt 0 ]]; then
+  echo "✅ Successful tests:"
+  printf '%s\n' "${results[@]}"
+fi
+
+if [[ ${#configWithoutTests[@]} -gt 0 ]]; then
+{
+  echo "Scout Playwright configs without tests:"
+  echo ""
+  for config in "${configWithoutTests[@]}"; do
+    echo "- $config"
+  done
+} | buildkite-agent annotate --style "warning" --context "no-tests"
+fi
+
+if [[ ${#failedConfigs[@]} -gt 0 ]]; then
+  echo "❌ Failed tests:"
+  printf '%s\n' "${failedConfigs[@]}"
+  buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
+fi
+
+exit $FINAL_EXIT_CODE  # Exit with 10 only if there were config failures
--- a/.buildkite/scripts/steps/test/scout_test_run_builder.sh
+++ b/.buildkite/scripts/steps/test/scout_test_run_builder.sh
@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
+
+echo '--- Discover Playwright Configs and upload to Buildkite artifacts'
+node scripts/scout discover-playwright-configs --save
+cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
+buildkite-agent artifact upload "scout_playwright_configs.json"
+
+echo '--- Scout Test Run Builder'
+ts-node "$(dirname "${0}")/scout_test_run_builder.ts"
--- a/.buildkite/scripts/steps/test/scout_test_run_builder.ts
+++ b/.buildkite/scripts/steps/test/scout_test_run_builder.ts
@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import path from 'path';
+import { CiStats } from '#pipeline-utils';
+
+(async () => {
+  try {
+    const scoutConfigsPath = path.resolve(
+      process.cwd(),
+      '.scout',
+      'test_configs',
+      'scout_playwright_configs.json'
+    );
+    await CiStats.pickScoutTestGroupRunOrder(scoutConfigsPath);
+  } catch (ex) {
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();
--- a/src/platform/packages/private/kbn-scout-info/src/paths.ts
+++ b/src/platform/packages/private/kbn-scout-info/src/paths.ts
@ -17,3 +17,10 @@ export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers');

 // Reporting
 export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports');
+
+// Scout playwright configs
+export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
+  SCOUT_OUTPUT_ROOT,
+  'test_configs',
+  'scout_playwright_configs.json'
+);
--- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts
+++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts
@ -9,8 +9,8 @@

 import fs from 'fs';
 import { Command } from '@kbn/dev-cli-runner';
-import { SCOUT_OUTPUT_ROOT } from '@kbn/scout-info';
-import { resolve } from 'path';
+import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
+import path from 'path';
 import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config';
 import { measurePerformance } from '../common';

@ -44,13 +44,19 @@ export const discoverPlaywrightConfigs: Command<void> = {
        ? 'No Playwright config files found'
        : `Found Playwright config files in '${pluginsMap.size}' plugins`;

-    if (pluginsMap.size > 0 && flagsReader.boolean('save')) {
-      const scoutConfigsFilePath = resolve(SCOUT_OUTPUT_ROOT, 'scout_playwright_configs.json');
+    if (flagsReader.boolean('save')) {
+      const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
+
+      if (!fs.existsSync(dirPath)) {
+        fs.mkdirSync(dirPath, { recursive: true });
+      }
+
      fs.writeFileSync(
-        scoutConfigsFilePath,
+        SCOUT_PLAYWRIGHT_CONFIGS_PATH,
        JSON.stringify(Object.fromEntries(pluginsMap), null, 2)
      );
-      log.info(`${finalMessage}. Saved to '${scoutConfigsFilePath}'`);
+
+      log.info(`${finalMessage}. Saved to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`);
      return;
    }

--- a/src/platform/packages/shared/kbn-scout/src/playwright/page_objects/discover_app.ts
+++ b/src/platform/packages/shared/kbn-scout/src/playwright/page_objects/discover_app.ts
@ -33,6 +33,7 @@ export class DiscoverApp {
    await this.page.testSubj.hover('discoverNewButton');
    await this.page.testSubj.click('discoverNewButton');
    await this.page.testSubj.hover('unifiedFieldListSidebar__toggle-collapse'); // cancel tooltips
+    await this.page.waitForLoadingIndicatorHidden();
  }

  async saveSearch(name: string) {