[8.x] [scout] validate config has tests before starting servers (#211918) (#211940)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[scout] validate config has tests before starting servers
(#211918)](https://github.com/elastic/kibana/pull/211918)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Dzmitry
Lemechko","email":"dzmitry.lemechko@elastic.co"},"sourceCommit":{"committedDate":"2025-02-20T17:20:21Z","message":"[scout]
validate config has tests before starting servers (#211918)\n\n##
Summary\n\nThere is no need to start servers (~1.5 min run time) if
there are no\ntests matching filters or maybe config itself has all
tests skipped.\n\nThis PR uses Playwright cli with `--list` flag to
quickly validate\nplaywright config and exit with status code `2` (`1`
is reserved for\nerrors during servers start or test failures). it also
useful to know in\nadvance how many tests were about to run:\n\ncase 1:
tests found\n\n```\n$ node scripts/scout.js run-tests --config
x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts
--serverless=security\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\n info scout: Total: 5 tests in 2
files\n info Verifying Docker is installed.\n │ info Docker version
20.10.14, build a224086349\n...\n```\n\ncase 2: no tests found\n```\n$
node scripts/scout.js run-tests --config
x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts
--stateful\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\nERROR scout: No tests found in
[x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts]\n```","sha":"30c4275462bfb975f4eb86d45fc608971ebfcfbc","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:QA","release_note:skip","v9.0.0","backport:version","test:scout","v9.1.0","v8.19.0"],"title":"[scout]
validate config has tests before starting
servers","number":211918,"url":"https://github.com/elastic/kibana/pull/211918","mergeCommit":{"message":"[scout]
validate config has tests before starting servers (#211918)\n\n##
Summary\n\nThere is no need to start servers (~1.5 min run time) if
there are no\ntests matching filters or maybe config itself has all
tests skipped.\n\nThis PR uses Playwright cli with `--list` flag to
quickly validate\nplaywright config and exit with status code `2` (`1`
is reserved for\nerrors during servers start or test failures). it also
useful to know in\nadvance how many tests were about to run:\n\ncase 1:
tests found\n\n```\n$ node scripts/scout.js run-tests --config
x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts
--serverless=security\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\n info scout: Total: 5 tests in 2
files\n info Verifying Docker is installed.\n │ info Docker version
20.10.14, build a224086349\n...\n```\n\ncase 2: no tests found\n```\n$
node scripts/scout.js run-tests --config
x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts
--stateful\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\nERROR scout: No tests found in
[x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts]\n```","sha":"30c4275462bfb975f4eb86d45fc608971ebfcfbc"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.x"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/211918","number":211918,"mergeCommit":{"message":"[scout]
validate config has tests before starting servers (#211918)\n\n##
Summary\n\nThere is no need to start servers (~1.5 min run time) if
there are no\ntests matching filters or maybe config itself has all
tests skipped.\n\nThis PR uses Playwright cli with `--list` flag to
quickly validate\nplaywright config and exit with status code `2` (`1`
is reserved for\nerrors during servers start or test failures). it also
useful to know in\nadvance how many tests were about to run:\n\ncase 1:
tests found\n\n```\n$ node scripts/scout.js run-tests --config
x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts
--serverless=security\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\n info scout: Total: 5 tests in 2
files\n info Verifying Docker is installed.\n │ info Docker version
20.10.14, build a224086349\n...\n```\n\ncase 2: no tests found\n```\n$
node scripts/scout.js run-tests --config
x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts
--stateful\n info scout: Test server configuration saved at
/Users/dmle/github/kibana/.scout/servers/local.json\n info scout:
Validate Playwright config has tests\nERROR scout: No tests found in
[x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts]\n```","sha":"30c4275462bfb975f4eb86d45fc608971ebfcfbc"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>

packages/kbn-scout/src/playwright/runner/run_tests.ts

@@ -8,6 +8,9 @@
  */
 
 import { resolve } from 'path';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+const execPromise = promisify(exec);
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { withProcRunner } from '@kbn/dev-proc-runner';
@@ -28,7 +31,21 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const playwrightGrepTag = getPlaywrightGrepTag(config);
   const playwrightConfigPath = options.configPath;
 
+  const cmd = resolve(REPO_ROOT, './node_modules/.bin/playwright');
+  const cmdArgs = ['test', `--config=${playwrightConfigPath}`, `--grep=${playwrightGrepTag}`];
+
   await withProcRunner(log, async (procs) => {
+    log.info(`scout: Validate Playwright config has tests`);
+    try {
+      // '--list' flag tells Playwright to collect all the tests, but do not run it
+      const result = await execPromise(`${cmd} ${cmdArgs.join(' ')} --list`);
+      const lastLine = result.stdout.trim().split('\n').pop();
+      log.info(`scout: ${lastLine}`);
+    } catch (err) {
+      log.error(`scout: No tests found in [${playwrightConfigPath}]`);
+      process.exit(2); // code "2" means no tests found
+    }
+
     const abortCtrl = new AbortController();
 
     const onEarlyExit = (msg: string) => {
@@ -60,13 +77,8 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
 
       // Running 'npx playwright test --config=${playwrightConfigPath}'
       await procs.run(`playwright`, {
-        cmd: resolve(REPO_ROOT, './node_modules/.bin/playwright'),
-        args: [
-          'test',
-          `--config=${playwrightConfigPath}`,
-          `--grep=${playwrightGrepTag}`,
-          ...(options.headed ? ['--headed'] : []),
-        ],
+        cmd,
+        args: [...cmdArgs, ...(options.headed ? ['--headed'] : [])],
         cwd: resolve(REPO_ROOT),
         env: {
           ...process.env,