[kbn-scout] add test config category to reporting (#210167)

## Summary

This PR adds a test config category to the scout reporting. This allows
us to distinguish between UI and API FTR tests.
A new property `testConfigCategory` has been added to all FTR configs
that don't already inherit it from a higher level config.

packages/kbn-scout-info/src/reporting.ts

@@ -26,3 +26,11 @@ export const SCOUT_TEST_EVENTS_INDEX_PATTERN =
   process.env.SCOUT_TEST_EVENTS_INDEX_PATTERN || `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-*`;
 export const SCOUT_TEST_EVENTS_DATA_STREAM_NAME =
   process.env.SCOUT_TEST_EVENTS_DATA_STREAM_NAME || `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-kibana`;
+
+export enum ScoutTestRunConfigCategory {
+  UI_TEST = 'ui-test',
+  API_TEST = 'api-test',
+  UNIT_TEST = 'unit-test',
+  UNIT_INTEGRATION_TEST = 'unit-integration-test',
+  UNKNOWN = 'unknown',
+}

packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts

@@ -20,7 +20,7 @@ import type {
 
 import path from 'node:path';
 import { ToolingLog } from '@kbn/tooling-log';
-import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info';
+import { SCOUT_REPORT_OUTPUT_ROOT, ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import stripANSI from 'strip-ansi';
 import { REPO_ROOT } from '@kbn/repo-info';
 import {
@@ -107,6 +107,7 @@ export class ScoutPlaywrightReporter implements Reporter {
     if (config.configFile !== undefined) {
       configInfo = {
         file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.configFile)),
+        category: ScoutTestRunConfigCategory.UI_TEST,
       };
     }
 

packages/kbn-scout-reporting/src/reporting/report/events/event.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { BuildkiteMetadata, HostMetadata } from '../../../datasources';
 
 /**
@@ -61,6 +62,7 @@ export interface ScoutTestRunInfo {
   id: string;
   config?: {
     file?: ScoutFileInfo;
+    category?: ScoutTestRunConfigCategory;
   };
   status?: string;
   duration?: number;

packages/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts

@@ -133,6 +133,9 @@ export const testRunProperties: Record<PropertyName, MappingProperty> = {
         type: 'object',
         properties: fileInfoProperties,
       },
+      category: {
+        type: 'keyword',
+      },
     },
   },
 };

packages/kbn-test/src/functional_test_runner/integration_tests/basic.test.js

@@ -17,7 +17,10 @@ const BASIC_CONFIG = require.resolve('./__fixtures__/simple_project/config.js');
 
 describe('basic config file with a single app and test', function () {
   it('runs and prints expected output', () => {
-    const proc = spawnSync(process.execPath, [SCRIPT, '--config', BASIC_CONFIG]);
+    const proc = spawnSync(process.execPath, [SCRIPT, '--config', BASIC_CONFIG], {
+      // this FTR run should not produce a scout report
+      env: { ...process.env, SCOUT_REPORTER_ENABLED: '0' },
+    });
     const stdout = proc.stdout.toString('utf8');
     expect(stdout).toContain('$BEFORE$');
     expect(stdout).toContain('$TESTNAME$');

packages/kbn-test/src/functional_test_runner/integration_tests/failure_hooks.test.js

@@ -18,7 +18,10 @@ const FAILURE_HOOKS_CONFIG = require.resolve('./__fixtures__/failure_hooks/confi
 
 describe('failure hooks', function () {
   it('runs and prints expected output', () => {
-    const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG]);
+    const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG], {
+      // this FTR run should not produce a scout report
+      env: { ...process.env, SCOUT_REPORTER_ENABLED: '0' },
+    });
     const lines = stripAnsi(proc.stdout.toString('utf8')).split(/\r?\n/);
     const linesCopy = [...lines];
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -11,7 +11,7 @@ import { dirname, resolve } from 'path';
 
 import Joi from 'joi';
 import type { CustomHelpers } from 'joi';
-import { SCOUT_REPORTER_ENABLED } from '@kbn/scout-info';
+import { SCOUT_REPORTER_ENABLED, ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 // valid pattern for ID
 // enforced camel-case identifiers for consistency
@@ -92,6 +92,9 @@ export const schema = Joi.object()
     testFiles: Joi.array().items(Joi.string()),
     testRunner: Joi.func(),
     serverless: Joi.boolean().default(false),
+    testConfigCategory: Joi.string()
+      .valid(...Object.values(ScoutTestRunConfigCategory))
+      .default(ScoutTestRunConfigCategory.UNKNOWN),
 
     suiteFiles: Joi.object()
       .keys({

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts

@@ -68,7 +68,10 @@ export class ScoutFTRReporter {
     this.codeOwnersEntries = getCodeOwnersEntries();
     this.baseTestRunInfo = {
       id: this.runId,
-      config: { file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.path)) },
+      config: {
+        file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.path)),
+        category: config.get('testConfigCategory'),
+      },
     };
 
     // Register event listeners

test/analytics/config.ts

@@ -9,6 +9,7 @@
 
 import path from 'path';
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 /*
@@ -23,6 +24,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     services,
     pageObjects: functionalConfig.get('pageObjects'),

test/api_integration/config.js

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 export default async function ({ readConfigFile }) {
@@ -18,6 +19,7 @@ export default async function ({ readConfigFile }) {
     testFiles: [require.resolve('./apis')],
     services,
     servers: commonConfig.get('servers'),
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     junit: {
       reportName: 'API Integration Tests',
     },

test/examples/config.js

@@ -10,12 +10,14 @@
 import { resolve } from 'path';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from '../plugin_functional/services';
 
 export default async function ({ readConfigFile }) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./hello_world'),

test/functional/config.base.js

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { pageObjects } from './page_objects';
 import { services } from './services';
 
@@ -19,6 +20,8 @@ export default async function ({ readConfigFile }) {
 
     servers: commonConfig.get('servers'),
 
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
+
     esTestCluster: {
       ...commonConfig.get('esTestCluster'),
       serverArgs: [`xpack.security.enabled=${process.env.ES_SECURITY_ENABLED ? 'true' : 'false'}`],

test/health_gateway/config.ts

@@ -8,6 +8,7 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     services,
     rootTags: ['runOutsideOfCiGroups'],
     esTestCluster: functionalConfig.get('esTestCluster'),

test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts

@@ -10,6 +10,7 @@
 import fs from 'fs/promises';
 import { join, resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 import { getDataPath } from '@kbn/utils';
 
@@ -24,6 +25,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   await fs.writeFile(tempKibanaYamlFile, '');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/manual_configuration_flow_without_tls')],
     servers: xPackAPITestsConfig.get('servers'),
     services,

test/interpreter_functional/config.ts

@@ -8,12 +8,14 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/run_pipeline')],
     services: functionalConfig.get('services'),

test/node_roles_functional/all.config.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
@@ -14,6 +15,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/all')],
     services: {

test/node_roles_functional/background_tasks.config.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
@@ -14,6 +15,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/background_tasks')],
     services: {

test/node_roles_functional/ui.config.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
@@ -14,6 +15,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/ui')],
     services: {

test/plugin_functional/config.ts

@@ -8,12 +8,14 @@
  */
 
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import path from 'path';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./test_suites/usage_collection'),

test/server_integration/http/platform/config.status.ts

@@ -8,6 +8,7 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
 
 /*
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [
       // Status test should be first to resolve manually created "unavailable" plugin
       require.resolve('./status'),

test/server_integration/http/platform/config.ts

@@ -7,12 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./cache'), require.resolve('./headers')],
     services: httpConfig.get('services'),
     servers: httpConfig.get('servers'),

test/server_integration/http/ssl/config.js

@@ -9,6 +9,7 @@
 
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { createKibanaSupertestProvider } from '../../services';
 
 export default async function ({ readConfigFile }) {
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }) {
   const certificateAuthorities = [readFileSync(CA_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

test/server_integration/http/ssl_redirect/config.ts

@@ -10,6 +10,7 @@
 import Url from 'url';
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 import { createKibanaSupertestProvider } from '../../services';
@@ -21,6 +22,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const redirectPort = httpConfig.get('servers.kibana.port') + 1234;
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

test/server_integration/http/ssl_with_p12/config.js

@@ -9,6 +9,7 @@
 
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_P12_PATH, KBN_P12_PASSWORD } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { createKibanaSupertestProvider } from '../../services';
 
 export default async function ({ readConfigFile }) {
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }) {
   const certificateAuthorities = [readFileSync(CA_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

test/server_integration/http/ssl_with_p12_intermediate/config.js

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { readFileSync } from 'fs';
 import { CA1_CERT_PATH, CA2_CERT_PATH, EE_P12_PATH, EE_P12_PASSWORD } from '../../__fixtures__';
 import { createKibanaSupertestProvider } from '../../services';
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }) {
   const certificateAuthorities = [readFileSync(CA1_CERT_PATH), readFileSync(CA2_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

test/tsconfig.json

@@ -77,5 +77,6 @@
     "@kbn/core-saved-objects-import-export-server-internal",
     "@kbn/core-deprecations-common",
     "@kbn/data-grid-in-table-search",
+    "@kbn/scout-info",
   ]
 }

x-pack/test/alerting_api_integration/common/config.ts

@@ -9,6 +9,7 @@ import path from 'path';
 import getPort from 'get-port';
 import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { getAllExternalServiceSimulatorPaths } from '@kbn/actions-simulators-plugin/server/plugin';
 import { ExperimentalConfigKeys } from '@kbn/stack-connectors-plugin/common/experimental_features';
 import { SENTINELONE_CONNECTOR_ID } from '@kbn/stack-connectors-plugin/common/sentinelone/constants';
@@ -171,6 +172,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
         : [];
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: testFiles ? testFiles : [require.resolve(`../${name}/tests/`)],
       servers,
       services,

x-pack/test/api_integration/apis/cloud/saml.config.ts

@@ -8,6 +8,7 @@
 import { readFileSync } from 'fs';
 
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@@ -35,6 +36,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   };
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/relay_state')],
     servers,
     security: { disableTestUser: true },

x-pack/test/api_integration/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 
@@ -15,6 +16,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi
 
   return {
     services,
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     servers: xPackFunctionalTestsConfig.get('servers'),
     security: xPackFunctionalTestsConfig.get('security'),
     junit: {

x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts

@@ -11,6 +11,7 @@ import {
   defineDockerServersConfig,
 } from '@kbn/test';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { ServerlessProjectType } from '@kbn/es';
 import path from 'path';
 import { DeploymentAgnosticCommonServices, services } from '../services';
@@ -85,6 +86,7 @@ export function createServerlessTestConfig<T extends DeploymentAgnosticCommonSer
     return {
       ...svlSharedConfig.getAll(),
 
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       services: {
         // services can be customized, but must extend DeploymentAgnosticCommonServices
         ...(options.services || services),

x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts

@@ -20,6 +20,7 @@ import {
   FtrConfigProviderContext,
   defineDockerServersConfig,
 } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import path from 'path';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { STATEFUL_ROLES_ROOT_PATH } from '@kbn/es';
@@ -85,6 +86,7 @@ export function createStatefulTestConfig<T extends DeploymentAgnosticCommonServi
 
     return {
       servers,
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       dockerServers: defineDockerServersConfig({
         registry: {
           enabled: !!dockerRegistryPort,

x-pack/test/apm_api_integration/common/config.ts

@@ -16,6 +16,7 @@ import {
   LogLevel,
 } from '@kbn/apm-synthtrace';
 import { FtrConfigProviderContext, kbnTestConfig } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
 import { MachineLearningAPIProvider } from '../../functional/services/ml/api';
@@ -116,6 +117,7 @@ export function createTestConfig(
     const synthtraceKibanaClient = getApmSynthtraceKibanaClient(kibanaServerUrl);
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: [require.resolve('../tests')],
       servers,
       servicesRequiredForTestAnalysis: ['apmFtrConfig', 'registry'],

x-pack/test/automatic_import_api_integration/common/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 
@@ -36,6 +37,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers,
       services,

x-pack/test/banners_functional/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services, pageObjects } from './ftr_provider_context';
 
@@ -14,6 +15,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     servers: {
       ...kibanaFunctionalConfig.get('servers'),

x-pack/test/cases_api_integration/common/config.ts

@@ -9,6 +9,7 @@ import path from 'path';
 
 import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 import { getAllExternalServiceSimulatorPaths } from '@kbn/actions-simulators-plugin/server/plugin';
 import { services } from './services';
@@ -60,6 +61,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers,
       services,

x-pack/test/custom_branding/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services, pageObjects } from './ftr_provider_context';
 
@@ -14,6 +15,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     servers: {
       ...kibanaFunctionalConfig.get('servers'),

x-pack/test/dataset_quality_api_integration/common/config.ts

@@ -16,6 +16,7 @@ import {
   DATASET_QUALITY_TEST_PASSWORD,
   DatasetQualityUsername,
 } from '@kbn/dataset-quality-plugin/server/test_helpers/create_dataset_quality_users/authentication';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import {
   fleetPackageRegistryDockerImage,
   FtrConfigProviderContext,
@@ -117,6 +118,7 @@ export function createTestConfig(
     const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: [require.resolve('../tests')],
       servers,
       dockerServers: defineDockerServersConfig({

x-pack/test/encrypted_saved_objects_api_integration/config.ts

@@ -7,6 +7,7 @@
 
 import path from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests')],
     servers: xPackAPITestsConfig.get('servers'),
     services,

x-pack/test/fleet_api_integration/config.base.ts

@@ -13,6 +13,7 @@ import {
   defineDockerServersConfig,
   getKibanaCliLoggers,
 } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 const getFullPath = (relativePath: string) => path.join(path.dirname(__filename), relativePath);
 
@@ -60,6 +61,7 @@ export default async function ({ readConfigFile, log }: FtrConfigProviderContext
   }
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     servers: xPackAPITestsConfig.get('servers'),
     dockerServers,
     services: xPackAPITestsConfig.get('services'),

x-pack/test/ftr_apis/security_and_spaces/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 // eslint-disable-next-line import/no-default-export
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./apis')],
     servers: apiIntegrationConfig.get('servers'),
     services,

x-pack/test/functional/config.base.js

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { resolve } from 'path';
 import { services } from './services';
 import { pageObjects } from './page_objects';
@@ -23,6 +24,8 @@ export default async function ({ readConfigFile }) {
     services,
     pageObjects,
 
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
+
     servers: kibanaFunctionalConfig.get('servers'),
 
     esTestCluster: {

x-pack/test/functional/config_security_basic.ts

@@ -9,6 +9,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 import { pageObjects } from './page_objects';
@@ -24,6 +25,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     // list paths to the files that contain your plugins tests
     testFiles: [resolve(__dirname, './apps/security/basic_license')],
 

x-pack/test/functional_cloud/saml.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -32,6 +33,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, '../security_functional/plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/onboarding.ts')],
 
     services,

x-pack/test/functional_cors/config.ts

@@ -7,6 +7,7 @@
 
 import Url from 'url';
 import Path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 import { kbnTestConfig } from '@kbn/test';
 import { pageObjects } from '../functional/page_objects';
@@ -40,6 +41,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   });
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     servers,
     services: kibanaFunctionalConfig.get('services'),

x-pack/test/functional_embedded/config.ts

@@ -8,6 +8,7 @@
 import Fs from 'fs';
 import { resolve } from 'path';
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { pageObjects } from '../functional/page_objects';
 
@@ -31,6 +32,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   };
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     servers,
     services: kibanaFunctionalConfig.get('services'),

x-pack/test/licensing_plugin/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services, pageObjects } from './services';
 
@@ -26,6 +27,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   };
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./server')],
     servers,
     services,

x-pack/test/monitoring_api_integration/config.ts

@@ -6,11 +6,13 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./apis')],
     servers: xPackAPITestsConfig.get('servers'),
     services: xPackAPITestsConfig.get('services'),

x-pack/test/observability_api_integration/common/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { Config, FtrConfigProviderContext, kbnTestConfig } from '@kbn/test';
 import { format, UrlObject } from 'url';
 import { LogsSynthtraceEsClient, createLogger, LogLevel } from '@kbn/apm-synthtrace';
@@ -65,6 +66,7 @@ export function createTestConfig(settings: Settings) {
     const customTestServices = getCustomApiTestServices(xPackAPITestsConfig);
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers: xPackAPITestsConfig.get('servers'),
       services: {

x-pack/test/observability_onboarding_api_integration/common/config.ts

@@ -10,6 +10,7 @@ import {
   OBSERVABILITY_ONBOARDING_TEST_PASSWORD,
 } from '@kbn/observability-onboarding-plugin/server/test_helpers/create_observability_onboarding_users/authentication';
 import { createObservabilityOnboardingUsers } from '@kbn/observability-onboarding-plugin/server/test_helpers/create_observability_onboarding_users';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
@@ -94,6 +95,7 @@ export function createTestConfig(
     const esServer = servers.elasticsearch as UrlObject;
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: [require.resolve('../tests')],
       servers,
       servicesRequiredForTestAnalysis: ['observabilityOnboardingFtrConfig', 'registry'],

x-pack/test/plugin_api_integration/config.ts

@@ -7,12 +7,14 @@
 
 import path from 'path';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const integrationConfig = await readConfigFile(require.resolve('../api_integration/config'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [
       require.resolve('./test_suites/platform'),
       require.resolve('./test_suites/task_manager'),

x-pack/test/plugin_functional/config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 import { REPO_ROOT as KIBANA_ROOT } from '@kbn/repo-info';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
 import { services } from './services';
 import { pageObjects } from './page_objects';
@@ -20,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     // list paths to the files that contain your plugins tests
     testFiles: [
       resolve(__dirname, './test_suites/resolver'),

x-pack/test/profiling_api_integration/common/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
@@ -83,6 +84,7 @@ export function createTestConfig(
     const esServer = servers.elasticsearch as UrlObject;
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: [require.resolve('../tests')],
       servers,
       servicesRequiredForTestAnalysis: ['profilingFtrConfig', 'registry'],

x-pack/test/rule_registry/common/config.ts

@@ -8,6 +8,7 @@
 import path from 'path';
 import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 import { getAllExternalServiceSimulatorPaths } from '@kbn/actions-simulators-plugin/server/plugin';
 import { services } from './services';
@@ -56,6 +57,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers,
       services,

x-pack/test/saved_object_api_integration/common/config.ts

@@ -8,6 +8,7 @@
 import path from 'path';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 import { services } from './services';
 
@@ -33,6 +34,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: [require.resolve(`../${name}/apis/`)],
       servers: config.xpack.api.get('servers'),
       services,

x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 // eslint-disable-next-line import/no-default-export
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./apis')],
     servers: apiIntegrationConfig.get('servers'),
     services,

x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 // eslint-disable-next-line import/no-default-export
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./apis')],
     servers: apiIntegrationConfig.get('servers'),
     services,

x-pack/test/saved_object_tagging/api_integration/tagging_usage_collection/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 // eslint-disable-next-line import/no-default-export
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests')],
     servers: apiIntegrationConfig.get('servers'),
     services,

x-pack/test/saved_object_tagging/functional/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services, pageObjects } from './ftr_provider_context';
 
 // eslint-disable-next-line import/no-default-export
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     servers: {
       ...kibanaFunctionalConfig.get('servers'),

x-pack/test/saved_objects_field_count/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
 
@@ -15,6 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
 
   return {
     ...kibanaCommonTestsConfig.getAll(),
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
 
     services: {
       ...commonFunctionalServices,

x-pack/test/security_api_integration/anonymous.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@@ -18,6 +19,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './plugins/audit_log/anonymous.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/anonymous')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/api_keys.config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -13,6 +14,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const httpBearerAPITestsConfig = await readConfigFile(require.resolve('./http_bearer.config.ts'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/api_keys')],
     servers: httpBearerAPITestsConfig.get('servers'),
     security: httpBearerAPITestsConfig.get('security'),

x-pack/test/security_api_integration/audit.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './plugins/audit_log/audit.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/audit')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/chips.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@@ -18,6 +19,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './plugins/audit_log/anonymous.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/chips')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/features.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -17,6 +18,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const featuresProviderPlugin = resolve(__dirname, './plugins/features_provider');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/features')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/http_bearer.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -18,6 +19,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const jwksPath = require.resolve('@kbn/security-api-integration-helpers/oidc/jwks.json');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/http_bearer')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/http_no_auth_providers.config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -13,6 +14,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/http_no_auth_providers')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/kerberos.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './plugins/audit_log/kerberos.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/kerberos')],
     servers: xPackAPITestsConfig.get('servers'),
     services,

x-pack/test/security_api_integration/login_selector.config.ts

@@ -9,6 +9,7 @@ import { readFileSync } from 'fs';
 import { resolve } from 'path';
 
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@@ -53,6 +54,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   };
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/login_selector')],
     servers,
     security: { disableTestUser: true },

x-pack/test/security_api_integration/oidc.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './packages/helpers/audit/oidc.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/oidc/authorization_code_flow')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/pki.config.ts

@@ -8,6 +8,7 @@
 import { resolve } from 'path';
 
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -32,6 +33,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './packages/helpers/audit/pki.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/pki')],
     servers,
     security: { disableTestUser: true },

x-pack/test/security_api_integration/saml.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './packages/helpers/audit/saml.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/saml')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/saml_cloud.config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const idpPath = require.resolve('@kbn/security-api-integration-helpers/saml/idp_metadata.xml');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/saml_cloud')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/session_concurrent_limit.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, '../security_functional/plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [resolve(__dirname, './tests/session_concurrent_limit')],
     services,
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_api_integration/session_idle.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, '../security_functional/plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [resolve(__dirname, './tests/session_idle')],
     services,
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_api_integration/session_invalidate.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -20,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const idpPath = require.resolve('@kbn/security-api-integration-helpers/saml/idp_metadata.xml');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [resolve(__dirname, './tests/session_invalidate')],
     services,
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_api_integration/session_lifespan.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -20,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const idpPath = require.resolve('@kbn/security-api-integration-helpers/saml/idp_metadata.xml');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [resolve(__dirname, './tests/session_lifespan')],
     services,
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_api_integration/token.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -19,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const auditLogPath = resolve(__dirname, './packages/helpers/audit/token.log');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/token')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_api_integration/user_profiles.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -17,6 +18,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const userProfilesConsumerPlugin = resolve(__dirname, './plugins/user_profiles_consumer');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/user_profiles')],
     servers: xPackAPITestsConfig.get('servers'),
     security: { disableTestUser: true },

x-pack/test/security_functional/expired_session.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, './plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/expired_session')],
     services,
     pageObjects,

x-pack/test/security_functional/insecure_cluster_warning.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -27,6 +28,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     .filter((arg: string) => !arg.startsWith('--security.showInsecureClusterWarning'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/insecure_cluster_warning')],
 
     services,

x-pack/test/security_functional/login_selector.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -35,6 +36,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, './plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/login_selector')],
 
     services,

x-pack/test/security_functional/oidc.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -29,6 +30,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, './plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/oidc')],
 
     services,

x-pack/test/security_functional/saml.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -32,6 +33,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, './plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/saml')],
 
     services,

x-pack/test/security_functional/user_profiles.config.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from '../functional/page_objects';
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const testEndpointsPlugin = resolve(__dirname, './plugins/test_endpoints');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [resolve(__dirname, './tests/user_profiles')],
 
     services,

x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { Config } from '@kbn/test';
 import { SecuritySolutionEndpointRegistryHelpers } from '../../../common/services/security_solution';
 import { SUITE_TAGS } from '../../../security_solution_endpoint/configs/config.base';
@@ -27,6 +28,7 @@ export const generateConfig = async ({
     SecuritySolutionEndpointRegistryHelpers();
   return {
     ...baseConfig.getAll(),
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     dockerServers: createEndpointDockerConfig(),
     services,
     junit: {

x-pack/test/security_solution_api_integration/config/ess/config.base.ts

@@ -9,6 +9,7 @@ import path from 'path';
 
 import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { FtrConfigProviderContext, kbnTestConfig, kibanaTestUser } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services as baseServices } from './services';
 import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared';
 
@@ -52,6 +53,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers,
       services,

x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 export interface CreateTestConfigOptions {
   testFiles: string[];
@@ -20,6 +22,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
     );
     return {
       ...svlSharedConfig.getAll(),
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       services: {
         ...services,
       },

x-pack/test/security_solution_api_integration/config/serverless/config.base.ts

@@ -7,6 +7,7 @@
 import path from 'path';
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared';
 
@@ -25,6 +26,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
     );
     return {
       ...svlSharedConfig.getAll(),
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       suiteTags: options.suiteTags,
       services: {
         ...services,

x-pack/test/security_solution_api_integration/tsconfig.json

@@ -54,5 +54,6 @@
     "@kbn/elastic-assistant-plugin",
     "@kbn/test-suites-src",
     "@kbn/openapi-common",
+    "@kbn/scout-info",
   ]
 }

x-pack/test/security_solution_endpoint/configs/config.base.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { Config } from '@kbn/test';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { SecuritySolutionEndpointRegistryHelpers } from '../../common/services/security_solution';
@@ -55,6 +56,7 @@ export const generateConfig = async ({
 
   return {
     ...baseConfig.getAll(),
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     pageObjects,
     testFiles,
     dockerServers: createEndpointDockerConfig(),

x-pack/test/security_solution_endpoint/tsconfig.json

@@ -29,5 +29,6 @@
     "@kbn/test-subj-selector",
     "@kbn/ftr-common-functional-services",
     "@kbn/spaces-plugin",
+    "@kbn/scout-info",
   ]
 }

x-pack/test/session_view/common/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 interface Settings {
@@ -22,6 +23,7 @@ export function createTestConfig(settings: Settings) {
     );
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles,
       servers: xPackAPITestsConfig.get('servers'),
       services: xPackAPITestsConfig.get('services'),

x-pack/test/spaces_api_integration/common/config.ts

@@ -8,6 +8,7 @@
 import path from 'path';
 
 import { REPO_ROOT } from '@kbn/repo-info';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -35,6 +36,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     };
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       testFiles: testFiles ?? [require.resolve(`../${name}/apis/`)],
       servers: config.xpack.api.get('servers'),
       services: {

x-pack/test/task_manager_claimer_update_by_query/config.ts

@@ -7,12 +7,14 @@
 
 import path from 'path';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const integrationConfig = await readConfigFile(require.resolve('../api_integration/config'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./test_suites/task_manager')],
     services,
     servers: integrationConfig.get('servers'),

x-pack/test/tsconfig.json

@@ -190,5 +190,6 @@
     "@kbn/server-route-repository-utils",
     "@kbn/streams-plugin",
     "@kbn/response-ops-rule-params",
+    "@kbn/scout-info",
   ]
 }

x-pack/test/ui_capabilities/common/config.ts

@@ -7,6 +7,7 @@
 
 import path from 'path';
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 import { services } from './services';
 
@@ -24,6 +25,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
     );
 
     return {
+      testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
       testFiles: [require.resolve(`../${name}/tests/`)],
       servers: xPackFunctionalTestsConfig.get('servers'),
       services,

x-pack/test/upgrade_assistant_integration/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'node:path';
 
@@ -19,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   );
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./upgrade_assistant')],
     servers: xPackFunctionalTestsConfig.get('servers'),
     services: {

x-pack/test_serverless/api_integration/config.base.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 import { services } from './services';
 import type { CreateTestConfigOptions } from '../shared/types';
@@ -16,6 +17,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
     return {
       ...svlSharedConfig.getAll(),
 
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       services: {
         ...services,
         ...options.services,

x-pack/test_serverless/functional/config.base.ts

@@ -6,6 +6,7 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { resolve } from 'path';
 import { pageObjects } from './page_objects';
 import { services } from './services';
@@ -20,6 +21,7 @@ export function createTestConfig<TServices extends {} = typeof services>(
     return {
       ...svlSharedConfig.getAll(),
 
+      testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
       pageObjects,
       services: { ...services, ...options.services },
       esTestCluster: {

x-pack/test_serverless/tsconfig.json

@@ -89,5 +89,6 @@
     "@kbn/core-saved-objects-import-export-server-internal",
     "@kbn/security-plugin-types-common",
     "@kbn/data-usage-plugin",
+    "@kbn/scout-info",
   ]
 }