[scout] extend config-discovery with CI validator (#214403)

## Summary

More teams are adding Scout tests in their plugins, often as a PoC and
not stable yet for continuous execution.
We don't want to block it, but need a way to manage the scope of Scout
pipeline and be able to disable it quickly to unblock the Scout
development.

Since Scout is in active development and we need it to be simple and
quick as possible (we can iterate and improve later), we agreed with
Robert to disable tests by plugin:

```
ui_tests:
  enabled:
    - apm
    - discover_enhanced
    - maps
    - observability_onboarding
  disabled:
    - *skipped_plugin*
```

When scout configuration is added to the new plugin, it will require to
update `.buildkite/scout_ci_config.yml` that is owned by `appex-qa`
team. If there is no intention to run Scout tests on CI, plugin name
should be added under `disabled` section.

**How to test locally:**

- Scout tests were added in `observability_onboarding` plugin, pipeline
will throw error

modify locally `.buildkite/scout_ci_config.yml`
```
ui_tests:
  enabled:
    - apm
    - discover_enhanced
    - maps
  disabled:
```

run `node scripts/scout discover-playwright-configs --validate --save`

```
ERROR The following plugins are not registered in Scout CI config '.buildkite/scout_ci_config.yml'
      - observability_onboarding
```

~~On CI annotation will be added to clarify the failure:~~

we decided to move validation to "Quick Checks", no need to annotate.

<img width="1583" alt="image"
src="https://github.com/user-attachments/assets/ed6b5778-74cb-4473-8218-b96239aab067"
/>


- `observability_onboarding` plugin is disabled, pipeline won't include
it (excluded in `scout_playwright_configs.json`)

modify locally `.buildkite/scout_ci_config.yml`
```
ui_tests:
  enabled:
    - apm
    - discover_enhanced
    - maps
  disabled:
   - observability_onboarding
```

run `node scripts/scout discover-playwright-configs --validate --save`

```
 warn The following plugins are disabled in '.buildkite/scout_ci_config.yml' and will be excluded from CI run
      - observability_onboarding
 info Found Playwright config files in '4' plugins.
      Saved '3' plugins to '/Users/dmle/github/kibana/.scout/test_configs/scout_playwright_configs.json'
```

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -83,5 +83,7 @@ steps:
       PING_SLACK_TEAM: "@appex-qa-team"
     retry:
       automatic:
+        - exit_status: 10
+          limit: 0
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/scout_tests.yml

@@ -8,5 +8,8 @@ steps:
       SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
     retry:
       automatic:
+        # No retry when Scout CI config validation fails
+        - exit_status: 10
+          limit: 0
         - exit_status: '*'
           limit: 1

.buildkite/scout_ci_config.yml

@@ -0,0 +1,8 @@
+# Define which plugins should be run or skipped in Scout CI pipeline
+ui_tests:
+  enabled:
+    - apm
+    - discover_enhanced
+    - maps
+    - observability_onboarding
+  disabled:

.buildkite/scripts/steps/checks/check_scout_config.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check for unregistered Scout Playwright configs
+node scripts/scout discover-playwright-configs --validate 

.buildkite/scripts/steps/checks/quick_checks.txt

@@ -20,3 +20,4 @@
 .buildkite/scripts/steps/checks/styled_components_mapping.sh
 .buildkite/scripts/steps/checks/dependencies_missing_owner.sh
 .buildkite/scripts/steps/checks/validate_pipelines.sh
+.buildkite/scripts/steps/checks/check_scout_config.sh

.github/CODEOWNERS

@@ -1625,6 +1625,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /.ci/.storybook @elastic/kibana-operations
 
 # QA - Appex QA
+.buildkite/scout_ci_config.yml @elastic/appex-qa
 /x-pack/test/.gitignore @elastic/appex-qa
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/es/roles.yml @elastic/appex-qa
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/oblt/roles.yml @elastic/appex-qa

src/platform/packages/shared/kbn-scout/README.md

@@ -9,6 +9,7 @@
 3. Key Components
 4. How to Use
 5. Contributing
+6. Running tests on CI
 
 ### Overview
 
@@ -306,3 +307,20 @@ export const scoutTestFixtures = mergeTests(
   - `Fixtures` can combine browser interactions with API requests, but they should be used wisely, especially with the `test` scope: a new instance of the fixture is created for **every test block**. If a fixture performs expensive operations (API setup, data ingestion), excessive usage can **slow down** the test suite runtime. Consider using `worker` scope when appropriate to reuse instances across tests within a worker.
 - **Add Unit Tests:** Include tests for new logic where applicable, ensuring it works as expected.
 - **Playwright documentation:** [Official best practices](https://playwright.dev/docs/best-practices)
+
+
+### Running tests on CI
+Scout is still in active development, which means frequent code changes may sometimes cause test failures. To maintain stability, we currently do not run Scout tests for every PR and encourage teams to limit the number of tests they add for now.
+
+If a test is difficult to stabilize within a reasonable timeframe, we reserve the right to disable it or even all tests for particular plugin.
+
+To manage Scout test execution, we use the `.buildkite/scout_ci_config.yml` file, where Kibana plugins with Scout tests are registered. If you're unsure about the stability of your tests, please add your plugin under the `disabled` section.
+
+You can check whether your plugin is already registered by running:
+```bash
+node scripts/scout discover-playwright-configs --validate
+```
+On CI we run Scout tests only for `enabled` plugins:
+
+For PRs, Scout tests run only if there are changes to registered plugins or Scout-related packages.
+On merge commits, Scout tests run in a non-blocking mode.

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -13,6 +13,7 @@ import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
 import path from 'path';
 import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config';
 import { measurePerformance } from '../common';
+import { validateWithScoutCiConfig } from '../config/discovery';
 
 /**
  * Discover Playwright configuration files with Scout tests
@@ -24,18 +25,19 @@ export const discoverPlaywrightConfigs: Command<void> = {
 
   Common usage:
     node scripts/scout discover-playwright-configs --searchPaths <search_paths>
-    node scripts/scout discover-playwright-configs --save
+    node scripts/scout discover-playwright-configs --validate // validate if all plugins are registered in the Scout CI config
+    node scripts/scout discover-playwright-configs --save // save the discovered Playwright config files to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'
     node scripts/scout discover-playwright-configs
   `,
   flags: {
     string: ['searchPaths'],
-    boolean: ['save'],
-    default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false },
+    boolean: ['save', 'validate'],
+    default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false, validate: false },
   },
   run: ({ flagsReader, log }) => {
     const searchPaths = flagsReader.arrayOfStrings('searchPaths')!;
 
-    const pluginsMap = measurePerformance(log, 'Discovering playwright config files', () => {
+    let pluginsMap = measurePerformance(log, 'Discovering Playwright config files', () => {
       return getScoutPlaywrightConfigs(searchPaths, log);
     });
 
@@ -44,6 +46,21 @@ export const discoverPlaywrightConfigs: Command<void> = {
         ? 'No Playwright config files found'
         : `Found Playwright config files in '${pluginsMap.size}' plugins`;
 
+    if (!flagsReader.boolean('save')) {
+      log.info(finalMessage);
+
+      pluginsMap.forEach((data, plugin) => {
+        log.info(`${data.group} / [${plugin}] plugin:`);
+        data.configs.map((file) => {
+          log.info(`- ${file}`);
+        });
+      });
+    }
+
+    if (flagsReader.boolean('validate')) {
+      pluginsMap = validateWithScoutCiConfig(log, pluginsMap);
+    }
+
     if (flagsReader.boolean('save')) {
       const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
 
@@ -56,17 +73,9 @@ export const discoverPlaywrightConfigs: Command<void> = {
         JSON.stringify(Object.fromEntries(pluginsMap), null, 2)
       );
 
-      log.info(`${finalMessage}. Saved to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`);
-      return;
+      log.info(
+        `${finalMessage}.\nSaved '${pluginsMap.size}' plugins to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`
+      );
     }
-
-    log.info(finalMessage);
-
-    pluginsMap.forEach((data, plugin) => {
-      log.info(`${data.group} / [${plugin}] plugin:`);
-      data.configs.map((file) => {
-        log.info(`- ${file}`);
-      });
-    });
   },
 };

src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts

@@ -7,11 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { ToolingLog } from '@kbn/tooling-log';
 import fastGlob from 'fast-glob';
-import { getScoutPlaywrightConfigs } from './search_configs';
+import yaml from 'js-yaml';
+import { ToolingLog } from '@kbn/tooling-log';
+import { getScoutPlaywrightConfigs, validateWithScoutCiConfig } from './search_configs';
 
 jest.mock('fast-glob');
+jest.mock('js-yaml');
 
 describe('getScoutPlaywrightConfigs', () => {
   let mockLog: ToolingLog;
@@ -80,3 +82,99 @@ describe('getScoutPlaywrightConfigs', () => {
     );
   });
 });
+
+describe('validateWithScoutCiConfig', () => {
+  let mockLog: ToolingLog;
+  const mockScoutCiConfig = {
+    ui_tests: {
+      enabled: ['pluginA', 'pluginB'],
+      disabled: ['pluginC'],
+    },
+  };
+
+  beforeEach(() => {
+    mockLog = new ToolingLog({ level: 'verbose', writeTo: process.stdout });
+    jest.spyOn(mockLog, 'warning').mockImplementation(jest.fn());
+    (yaml.load as jest.Mock).mockReturnValue(mockScoutCiConfig);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should return only enabled plugins', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginA',
+        {
+          group: 'group1',
+          pluginPath: 'pluginPathA',
+          usesParallelWorkers: true,
+          configs: ['configA'],
+        },
+      ],
+      [
+        'pluginB',
+        {
+          group: 'group1',
+          pluginPath: 'pluginPathB',
+          usesParallelWorkers: false,
+          configs: ['configB1', 'configB2'],
+        },
+      ],
+      [
+        'pluginC',
+        {
+          group: 'group2',
+          pluginPath: 'pluginPathC',
+          usesParallelWorkers: true,
+          configs: ['configC'],
+        },
+      ],
+    ]);
+
+    const result = validateWithScoutCiConfig(mockLog, pluginsWithConfigs);
+    expect(result.size).toBe(2);
+    expect(result.has('pluginA')).toBe(true);
+    expect(result.has('pluginB')).toBe(true);
+    expect(result.has('pluginC')).toBe(false);
+  });
+
+  it('should throw an error if plugins are not registered in Scout CI config', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginX',
+        {
+          group: 'groupX',
+          pluginPath: 'pluginPathX',
+          usesParallelWorkers: true,
+          configs: ['configX'],
+        },
+      ],
+    ]);
+
+    expect(() => validateWithScoutCiConfig(mockLog, pluginsWithConfigs)).toThrow(
+      "The following plugins are not registered in Scout CI config '.buildkite/scout_ci_config.yml':\n- pluginX"
+    );
+  });
+
+  it('should log a warning for disabled plugins', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginC',
+        {
+          group: 'group2',
+          pluginPath: 'pluginPathC',
+          usesParallelWorkers: true,
+          configs: ['configC'],
+        },
+      ],
+    ]);
+
+    validateWithScoutCiConfig(mockLog, pluginsWithConfigs);
+
+    expect(mockLog.warning).toHaveBeenCalledWith(
+      `The following plugins are disabled in '.buildkite/scout_ci_config.yml' and will be excluded from CI run\n- pluginC`
+    );
+  });
+});

src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts

@@ -10,6 +10,10 @@
 import fastGlob from 'fast-glob';
 import path from 'path';
 import { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/repo-info';
+import fs from 'fs';
+import yaml from 'js-yaml';
+import { createFailError } from '@kbn/dev-cli-errors';
 
 export const DEFAULT_TEST_PATH_PATTERNS = ['src/platform/plugins', 'x-pack/**/plugins'];
 
@@ -88,3 +92,56 @@ export const getScoutPlaywrightConfigs = (searchPaths: string[], log: ToolingLog
 
   return pluginsWithConfigs;
 };
+
+export const validateWithScoutCiConfig = (
+  log: ToolingLog,
+  pluginsWithConfigs: Map<string, PluginScoutConfig>
+) => {
+  const scoutCiConfigRelPath = path.join('.buildkite', 'scout_ci_config.yml');
+  const scoutCiConfigPath = path.resolve(REPO_ROOT, scoutCiConfigRelPath);
+  const ciConfig = yaml.load(fs.readFileSync(scoutCiConfigPath, 'utf8')) as {
+    ui_tests: {
+      enabled?: string[];
+      disabled?: string[];
+    };
+  };
+
+  const enabledPlugins = new Set(ciConfig.ui_tests.enabled || []);
+  const disabledPlugins = new Set(ciConfig.ui_tests.disabled || []);
+  const allRegisteredPlugins = new Set([...enabledPlugins, ...disabledPlugins]);
+
+  const unregisteredPlugins: string[] = [];
+  const filteredPlugins = new Map<string, PluginScoutConfig>();
+
+  for (const [pluginName, config] of pluginsWithConfigs.entries()) {
+    if (!allRegisteredPlugins.has(pluginName)) {
+      unregisteredPlugins.push(pluginName);
+    } else if (enabledPlugins.has(pluginName)) {
+      filteredPlugins.set(pluginName, config);
+    }
+  }
+
+  if (unregisteredPlugins.length > 0) {
+    throw createFailError(
+      `The following plugins are not registered in Scout CI config '${scoutCiConfigRelPath}':\n${unregisteredPlugins
+        .map((plugin) => {
+          return `- ${plugin}`;
+        })
+        .join('\n')}\nRead more: src/platform/packages/shared/kbn-scout/README.md`
+    );
+  }
+
+  if (disabledPlugins.size > 0) {
+    log.warning(
+      `The following plugins are disabled in '${scoutCiConfigRelPath}' and will be excluded from CI run\n${[
+        ...disabledPlugins,
+      ]
+        .map((plugin) => {
+          return `- ${plugin}`;
+        })
+        .join('\n')}`
+    );
+  }
+
+  return filteredPlugins;
+};