[scout] Add tests to Scout CLI commands (#216272)

Depends on https://github.com/elastic/kibana/pull/216052 being merged. 

---

This is a follow-up PR based on @dmlemeshko's
[comment](https://github.com/elastic/kibana/pull/216052/files#r2015868889).

This PR introduces the following changes:

* Adds tests to all Scout CLI commands: `discover-playwright-configs`,
`run-tests`, `start-server`
* Some of the tests verify that this change works correctly:
https://github.com/elastic/kibana/pull/216052
* For each command file it separates the `run` function from the
`Command` itself to make the `run` handler easier to test.

### Bonus: code coverage

The `Command`s itself aren't tested (which explains the uncovered lines
below) - happy to receive your feedback on this.

| File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s

|----------------------------------------------------------------------------------------|---------|----------|---------|---------|------------------------------------------------------------------------------------
| platform/packages/shared/kbn-scout/src/cli | 91.89 | 78.57 | 66.66 |
91.89 |
| config_discovery.ts | 95.65 | 90 | 80 | 95.65 | 88
| run_tests.ts | 85.71 | 50 | 50 | 85.71 | 44
| start_server.ts | 85.71 | 50 | 50 | 85.71 | 34

### Try it out locally

```shell
yarn test:jest --config src/platform/packages/shared/kbn-scout/jest.config.js --coverage
```

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts

@@ -0,0 +1,182 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import fs from 'fs';
+import { FlagsReader } from '@kbn/dev-cli-runner';
+import { ToolingLog } from '@kbn/tooling-log';
+import { getScoutPlaywrightConfigs } from '../config';
+import { runDiscoverPlaywrightConfigs } from './config_discovery';
+import { measurePerformance } from '../common';
+import { validateWithScoutCiConfig } from '../config/discovery';
+
+jest.mock('fs');
+
+jest.mock('@kbn/scout-info', () => ({
+  SCOUT_PLAYWRIGHT_CONFIGS_PATH: '/path/to/scout_playwright_configs.json',
+}));
+
+jest.mock('../common', () => ({
+  measurePerformance: jest.fn(),
+}));
+
+jest.mock('../config', () => ({
+  getScoutPlaywrightConfigs: jest.fn(),
+}));
+
+jest.mock('../config/discovery', () => ({
+  validateWithScoutCiConfig: jest.fn(),
+}));
+
+describe('runDiscoverPlaywrightConfigs', () => {
+  let flagsReader: jest.Mocked<FlagsReader>;
+  let log: jest.Mocked<ToolingLog>;
+
+  // 'enabled' plugins
+  const mockPluginsWithRunnableConfigs: Map<string, any> = new Map([
+    [
+      'pluginA',
+      {
+        group: 'groupA',
+        pluginPath: 'plugin/path',
+        configs: ['config1.ts', 'config2.ts'],
+        usesParallelWorkers: true,
+      },
+    ],
+  ]);
+
+  // these are all the plugins found, but only some of them will be enabled (validateWithScoutCiConfig will find out which ones)
+  const mockPluginsWithConfigs = new Map([
+    ['pluginA', { group: 'groupA', configs: ['config1.ts', 'config2.ts'] }], // enabled
+    ['pluginB', { group: 'groupB', configs: ['config3.ts'] }], // disabled
+  ]);
+
+  beforeAll(() => {
+    flagsReader = {
+      arrayOfStrings: jest.fn(),
+      boolean: jest.fn(),
+    } as any;
+
+    log = {
+      info: jest.fn(),
+      error: jest.fn(),
+      warn: jest.fn(),
+    } as any;
+  });
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    (fs.existsSync as jest.Mock).mockReturnValue(false);
+    (fs.mkdirSync as jest.Mock).mockImplementation(jest.fn());
+    (fs.writeFileSync as jest.Mock).mockImplementation(jest.fn());
+
+    (measurePerformance as jest.Mock).mockImplementation((_log, _msg, fn) => fn());
+
+    (getScoutPlaywrightConfigs as jest.Mock).mockReturnValue(mockPluginsWithConfigs);
+    (validateWithScoutCiConfig as jest.Mock).mockReturnValue(mockPluginsWithRunnableConfigs);
+  });
+
+  it('validates configs when "validate" is true', () => {
+    // force --validate
+    flagsReader.boolean.mockImplementation((flag) => {
+      if (flag === 'save') {
+        return false;
+      }
+
+      // force --validate
+      if (flag === 'validate') {
+        return true;
+      }
+
+      return false;
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    // setting --validate will trigger a validation
+    expect(validateWithScoutCiConfig).toHaveBeenCalledWith(log, mockPluginsWithConfigs);
+  });
+
+  it('should correctly parse custom config search paths', () => {
+    flagsReader.arrayOfStrings.mockReturnValue(['customConfigSearchPaths']);
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+    expect(getScoutPlaywrightConfigs).toHaveBeenCalledWith(['customConfigSearchPaths'], log);
+  });
+
+  it('logs found configs when they exist and "save" flag is false', () => {
+    flagsReader.boolean.mockImplementation((flag) => {
+      // never --save
+      if (flag === 'save') {
+        return false;
+      }
+
+      if (flag === 'validate') {
+        return false;
+      }
+
+      return false;
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    expect(log.info.mock.calls).toEqual([
+      ["Found Playwright config files in '2' plugins"],
+      ['groupA / [pluginA] plugin:'],
+      ['- config1.ts'],
+      ['- config2.ts'],
+      ['groupB / [pluginB] plugin:'],
+      ['- config3.ts'],
+    ]);
+  });
+
+  it('logs "No Playwright config files found" when no configs are found', () => {
+    flagsReader.arrayOfStrings.mockReturnValue([]);
+    flagsReader.boolean.mockReturnValue(false);
+
+    (getScoutPlaywrightConfigs as jest.Mock).mockReturnValue(new Map());
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    expect(log.info).toHaveBeenCalledWith('No Playwright config files found');
+  });
+
+  it('validates and saves enabled plugins with their config files when --save is set', () => {
+    // force --save
+    flagsReader.boolean.mockImplementation((flag) => {
+      if (flag === 'save') {
+        return true;
+      }
+
+      if (flag === 'validate') {
+        return false;
+      }
+
+      return false;
+    });
+
+    flagsReader.arrayOfStrings.mockReturnValue(['searchPaths']);
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    // setting --save will trigger a validation
+    expect(validateWithScoutCiConfig).toHaveBeenCalledWith(log, mockPluginsWithConfigs);
+
+    // create directory if it doesn't exist
+    expect(fs.mkdirSync).toHaveBeenCalledWith('/path/to', { recursive: true });
+
+    // we should only write the configs of the plugins that are actually enabled
+    expect(fs.writeFileSync).toHaveBeenCalledWith(
+      '/path/to/scout_playwright_configs.json',
+      JSON.stringify(Object.fromEntries(mockPluginsWithRunnableConfigs), null, 2)
+    );
+
+    expect(log.info).toHaveBeenCalledWith(
+      `Found Playwright config files in '2' plugins.\nSaved '1' plugins to '/path/to/scout_playwright_configs.json'`
+    );
+  });
+});

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -8,17 +8,67 @@
  */
 
 import fs from 'fs';
-import { Command } from '@kbn/dev-cli-runner';
+import { Command, FlagsReader } from '@kbn/dev-cli-runner';
 import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
 import path from 'path';
+import { ToolingLog } from '@kbn/tooling-log';
 import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config';
 import { measurePerformance } from '../common';
 import { validateWithScoutCiConfig } from '../config/discovery';
 
+export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: ToolingLog) => {
+  const searchPaths = flagsReader.arrayOfStrings('searchPaths')!;
+
+  const pluginsWithConfigs = measurePerformance(log, 'Discovering Playwright config files', () =>
+    getScoutPlaywrightConfigs(searchPaths, log)
+  );
+
+  const finalMessage =
+    pluginsWithConfigs.size === 0
+      ? 'No Playwright config files found'
+      : `Found Playwright config files in '${pluginsWithConfigs.size}' plugins`;
+
+  if (!flagsReader.boolean('save')) {
+    log.info(finalMessage);
+
+    pluginsWithConfigs.forEach((data, plugin) => {
+      log.info(`${data.group} / [${plugin}] plugin:`);
+      data.configs.map((file) => {
+        log.info(`- ${file}`);
+      });
+    });
+  }
+
+  if (flagsReader.boolean('save')) {
+    const pluginsWithRunnableConfigs = validateWithScoutCiConfig(log, pluginsWithConfigs);
+
+    const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
+
+    if (!fs.existsSync(dirPath)) {
+      fs.mkdirSync(dirPath, { recursive: true });
+    }
+
+    fs.writeFileSync(
+      SCOUT_PLAYWRIGHT_CONFIGS_PATH,
+      JSON.stringify(Object.fromEntries(pluginsWithRunnableConfigs), null, 2)
+    );
+
+    log.info(
+      `${finalMessage}.\nSaved '${pluginsWithRunnableConfigs.size}' plugins to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`
+    );
+
+    return;
+  }
+
+  if (flagsReader.boolean('validate')) {
+    validateWithScoutCiConfig(log, pluginsWithConfigs);
+  }
+};
+
 /**
  * Discover Playwright configuration files with Scout tests
  */
-export const discoverPlaywrightConfigs: Command<void> = {
+export const discoverPlaywrightConfigsCmd: Command<void> = {
   name: 'discover-playwright-configs',
   description: `
   Discover Playwright configuration files with Scout tests.
@@ -35,55 +85,6 @@ export const discoverPlaywrightConfigs: Command<void> = {
     default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false, validate: false },
   },
   run: ({ flagsReader, log }) => {
-    const searchPaths = flagsReader.arrayOfStrings('searchPaths')!;
-
-    const pluginsWithConfigs = measurePerformance(
-      log,
-      'Discovering Playwright config files',
-      () => {
-        return getScoutPlaywrightConfigs(searchPaths, log);
-      }
-    );
-
-    const finalMessage =
-      pluginsWithConfigs.size === 0
-        ? 'No Playwright config files found'
-        : `Found Playwright config files in '${pluginsWithConfigs.size}' plugins`;
-
-    if (!flagsReader.boolean('save')) {
-      log.info(finalMessage);
-
-      pluginsWithConfigs.forEach((data, plugin) => {
-        log.info(`${data.group} / [${plugin}] plugin:`);
-        data.configs.map((file) => {
-          log.info(`- ${file}`);
-        });
-      });
-    }
-
-    if (flagsReader.boolean('save')) {
-      const pluginsWithRunnableConfigs = validateWithScoutCiConfig(log, pluginsWithConfigs);
-
-      const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
-
-      if (!fs.existsSync(dirPath)) {
-        fs.mkdirSync(dirPath, { recursive: true });
-      }
-
-      fs.writeFileSync(
-        SCOUT_PLAYWRIGHT_CONFIGS_PATH,
-        JSON.stringify(Object.fromEntries(pluginsWithRunnableConfigs), null, 2)
-      );
-
-      log.info(
-        `${finalMessage}.\nSaved '${pluginsWithRunnableConfigs.size}' plugins to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`
-      );
-
-      return;
-    }
-
-    if (flagsReader.boolean('validate')) {
-      validateWithScoutCiConfig(log, pluginsWithConfigs);
-    }
+    runDiscoverPlaywrightConfigs(flagsReader, log);
   },
 };

src/platform/packages/shared/kbn-scout/src/cli/index.ts

@@ -8,9 +8,9 @@
  */
 import { RunWithCommands } from '@kbn/dev-cli-runner';
 import { cli as reportingCLI } from '@kbn/scout-reporting';
-import { startServer } from './start_server';
-import { runTests } from './run_tests';
-import { discoverPlaywrightConfigs } from './config_discovery';
+import { startServerCmd } from './start_server';
+import { runTestsCmd } from './run_tests';
+import { discoverPlaywrightConfigsCmd } from './config_discovery';
 
 export async function run() {
   await new RunWithCommands(
@@ -18,9 +18,9 @@ export async function run() {
       description: 'Scout CLI',
     },
     [
-      startServer,
-      runTests,
-      discoverPlaywrightConfigs,
+      startServerCmd,
+      runTestsCmd,
+      discoverPlaywrightConfigsCmd,
       reportingCLI.initializeReportDatastream,
       reportingCLI.uploadEvents,
     ]

src/platform/packages/shared/kbn-scout/src/cli/run_tests.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { FlagsReader } from '@kbn/dev-cli-runner';
+import { ToolingLog } from '@kbn/tooling-log';
+import { runScoutPlaywrightConfig } from './run_tests';
+import { initLogsDir } from '@kbn/test';
+import { parseTestFlags, runTests } from '../playwright/runner';
+
+jest.mock('@kbn/test', () => ({
+  initLogsDir: jest.fn(),
+}));
+
+jest.mock('../playwright/runner', () => ({
+  parseTestFlags: jest.fn().mockResolvedValue({ logsDir: 'path/to/logs/directory' }),
+  runTests: jest.fn().mockResolvedValue(undefined),
+}));
+
+describe('runScoutPlaywrightConfig', () => {
+  let flagsReader: jest.Mocked<FlagsReader>;
+  let log: jest.Mocked<ToolingLog>;
+
+  beforeAll(() => {
+    flagsReader = {
+      arrayOfStrings: jest.fn(),
+      boolean: jest.fn(),
+    } as any;
+
+    log = {
+      info: jest.fn(),
+      error: jest.fn(),
+      warn: jest.fn(),
+    } as any;
+  });
+
+  it('calls parseTestFlags with the correct flagsReader', async () => {
+    await runScoutPlaywrightConfig(flagsReader, log);
+    expect(parseTestFlags).toHaveBeenCalledWith(flagsReader);
+  });
+
+  it('writes the log output to files instead of to stdout if --logToFile is set', async () => {
+    await runScoutPlaywrightConfig(flagsReader, log);
+    expect(initLogsDir).toHaveBeenCalledWith(log, 'path/to/logs/directory');
+  });
+
+  it('runs the tests', async () => {
+    await runScoutPlaywrightConfig(flagsReader, log);
+    expect(runTests).toHaveBeenCalledWith(log, { logsDir: 'path/to/logs/directory' });
+  });
+});

src/platform/packages/shared/kbn-scout/src/cli/run_tests.ts

@@ -9,13 +9,25 @@
 
 import { Command } from '@kbn/dev-cli-runner';
 import { initLogsDir } from '@kbn/test';
+import { FlagsReader } from '@kbn/dev-cli-runner';
+import { ToolingLog } from '@kbn/tooling-log';
 import { TEST_FLAG_OPTIONS } from '../playwright/runner';
-import { parseTestFlags, runTests as runTestsFn } from '../playwright/runner';
+import { parseTestFlags, runTests } from '../playwright/runner';
+
+export const runScoutPlaywrightConfig = async (flagsReader: FlagsReader, log: ToolingLog) => {
+  const options = await parseTestFlags(flagsReader);
+
+  if (options.logsDir) {
+    await initLogsDir(log, options.logsDir);
+  }
+
+  await runTests(log, options);
+};
 
 /**
  * Start servers and run the tests
  */
-export const runTests: Command<void> = {
+export const runTestsCmd: Command<void> = {
   name: 'run-tests',
   description: `
   Run a Scout Playwright config.
@@ -29,12 +41,6 @@ export const runTests: Command<void> = {
   `,
   flags: TEST_FLAG_OPTIONS,
   run: async ({ flagsReader, log }) => {
-    const options = await parseTestFlags(flagsReader);
-
-    if (options.logsDir) {
-      await initLogsDir(log, options.logsDir);
-    }
-
-    await runTestsFn(log, options);
+    await runScoutPlaywrightConfig(flagsReader, log);
   },
 };

src/platform/packages/shared/kbn-scout/src/cli/start_server.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { runStartServer } from './start_server';
+import { initLogsDir } from '@kbn/test';
+import { FlagsReader } from '@kbn/dev-cli-runner';
+import { ToolingLog } from '@kbn/tooling-log';
+import { startServers, parseServerFlags } from '../servers';
+
+jest.mock('@kbn/test', () => ({
+  initLogsDir: jest.fn(),
+}));
+
+jest.mock('../servers', () => ({
+  parseServerFlags: jest.fn().mockReturnValue({ logsDir: 'path/to/logs/directory' }),
+  startServers: jest.fn().mockResolvedValue(undefined),
+}));
+
+describe('runStartServer', () => {
+  let flagsReader: jest.Mocked<FlagsReader>;
+  let log: jest.Mocked<ToolingLog>;
+
+  beforeEach(() => {
+    flagsReader = {
+      arrayOfStrings: jest.fn(),
+      boolean: jest.fn(),
+    } as any;
+
+    log = {
+      info: jest.fn(),
+      error: jest.fn(),
+      warn: jest.fn(),
+    } as any;
+  });
+
+  it('calls parseServerFlags with the correct flagsReader', async () => {
+    await runStartServer(flagsReader, log);
+    expect(parseServerFlags).toHaveBeenCalledWith(flagsReader);
+  });
+
+  it('initializes log directory if logsDir is provided', async () => {
+    await runStartServer(flagsReader, log);
+    expect(initLogsDir).toHaveBeenCalledWith(log, 'path/to/logs/directory');
+  });
+
+  it('starts the servers with the correct options', async () => {
+    await runStartServer(flagsReader, log);
+    expect(startServers).toHaveBeenCalledWith(log, { logsDir: 'path/to/logs/directory' });
+  });
+});

src/platform/packages/shared/kbn-scout/src/cli/start_server.ts

@@ -9,22 +9,28 @@
 
 import { Command } from '@kbn/dev-cli-runner';
 import { initLogsDir } from '@kbn/test';
+import { FlagsReader } from '@kbn/dev-cli-runner';
+import { ToolingLog } from '@kbn/tooling-log';
 
 import { startServers, parseServerFlags, SERVER_FLAG_OPTIONS } from '../servers';
 
+export const runStartServer = async (flagsReader: FlagsReader, log: ToolingLog) => {
+  const options = parseServerFlags(flagsReader);
+
+  if (options.logsDir) {
+    await initLogsDir(log, options.logsDir);
+  }
+  await startServers(log, options);
+};
+
 /**
  * Start servers
  */
-export const startServer: Command<void> = {
+export const startServerCmd: Command<void> = {
   name: 'start-server',
   description: 'Start Elasticsearch & Kibana for testing purposes',
   flags: SERVER_FLAG_OPTIONS,
   run: async ({ flagsReader, log }) => {
-    const options = parseServerFlags(flagsReader);
-
-    if (options.logsDir) {
-      await initLogsDir(log, options.logsDir);
-    }
-    await startServers(log, options);
+    await runStartServer(flagsReader, log);
   },
 };