[ftr] add support for launching a dedicated task runner Kibana node (#135875)

* [ftr] add support for launching a dedicated task runner Kibana node

* Update run_kibana_server.ts

* disable the optimizer in kbn-tasks proc when running locally

* remove paths module

* include decicated task proc in promises array

* add getSupertest() helper to DedicatesTaskRunner service

* avoid caching a supertest instance, just create one on request

* remove surprise dependents on KIBANA_ROOT const

* remove modifications to test/analytics/config.ts

packages/kbn-test/BUILD.bazel

@@ -76,6 +76,7 @@ RUNTIME_DEPS = [
     "@npm//rxjs",
     "@npm//semver",
     "@npm//strip-ansi",
+    "@npm//supertest",
     "@npm//xmlbuilder",
     "@npm//xml2js",
 ]
@@ -123,6 +124,7 @@ TYPES_DEPS = [
   "@npm//@types/react-redux",
   "@npm//@types/react-router-dom",
   "@npm//@types/semver",
+  "@npm//@types/supertest",
   "@npm//@types/uuid",
   "@npm//@types/xml2js",
 ]

packages/kbn-test/src/es/test_es_cluster.ts

@@ -19,11 +19,11 @@ import type { ChildProcess } from 'child_process';
 import { Cluster } from '@kbn/es';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/utils';
+
 import { CI_PARALLEL_PROCESS_PREFIX } from '../ci_parallel_process_prefix';
 import { esTestConfig } from './es_test_config';
 
-import { KIBANA_ROOT } from '..';
-
 interface TestEsClusterNodesOptions {
   name: string;
   /**
@@ -168,7 +168,7 @@ export function createTestEsCluster<
     password = 'changeme',
     license = 'basic',
     log,
-    basePath = Path.resolve(KIBANA_ROOT, '.es'),
+    basePath = Path.resolve(REPO_ROOT, '.es'),
     esFrom = esTestConfig.getBuildFrom(),
     dataArchive,
     nodes = [{ name: 'node-01' }],
@@ -196,7 +196,7 @@ export function createTestEsCluster<
   const config = {
     version: esTestConfig.getVersion(),
     installPath: Path.resolve(basePath, clusterName),
-    sourcePath: Path.resolve(KIBANA_ROOT, '../elasticsearch'),
+    sourcePath: Path.resolve(REPO_ROOT, '../elasticsearch'),
     password,
     license,
     basePath,
@@ -321,7 +321,7 @@ export function createTestEsCluster<
       }
 
       const uuid = Uuid.v4();
-      const debugPath = Path.resolve(KIBANA_ROOT, `data/es_debug_${uuid}.tar.gz`);
+      const debugPath = Path.resolve(REPO_ROOT, `data/es_debug_${uuid}.tar.gz`);
       log.error(`[es] debug files found, archiving install to ${debugPath}`);
       const archiver = createArchiver('tar', { gzip: true });
       const promise = pipeline(archiver, Fs.createWriteStream(debugPath));

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -24,6 +24,7 @@ import {
   Config,
   SuiteTracker,
   EsVersion,
+  DedicatedTaskRunner,
 } from './lib';
 import { createEsClientForFtrConfig } from '../es';
 
@@ -242,6 +243,7 @@ export class FunctionalTestRunner {
         config: () => config,
         dockerServers: () => dockerServers,
         esVersion: () => this.esVersion,
+        dedicatedTaskRunner: () => new DedicatedTaskRunner(config, this.log),
       });
 
       return await handler(config, lifecycle, coreProviders);

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -216,6 +216,7 @@ export const schema = Joi.object()
         sourceArgs: Joi.array(),
         serverArgs: Joi.array(),
         installDir: Joi.string(),
+        useDedicatedTaskRunner: Joi.boolean().default(false),
         /** Options for how FTR should execute and interact with Kibana */
         runOptions: Joi.object()
           .keys({

packages/kbn-test/src/functional_test_runner/lib/dedicated_task_runner.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Url from 'url';
+
+import { ToolingLog } from '@kbn/tooling-log';
+import Supertest from 'supertest';
+
+import { KbnClient } from '../../kbn_client';
+import { Config } from './config';
+import { getKibanaCliArg } from '../../functional_tests/lib/kibana_cli_args';
+
+export class DedicatedTaskRunner {
+  static getPort(uiPort: number) {
+    return uiPort + 13;
+  }
+
+  static getUuid(mainUuid: string) {
+    if (mainUuid.length !== 36) {
+      throw new Error(`invalid mainUuid: ${mainUuid}`);
+    }
+
+    return `00000000-${mainUuid.slice(9)}`;
+  }
+
+  /**
+   * True when the FTR config indicates that Kibana has a dedicated task runner process, otherwise false. If this
+   * property is false then all other methods on this class will throw when they are called, so if you're not
+   * certain where your code will be run make sure to check `dedicatedTaskRunner.enabled` before calling
+   * other methods.
+   */
+  public readonly enabled: boolean;
+
+  private readonly enabledProps?: {
+    readonly port: number;
+    readonly url: string;
+    readonly client: KbnClient;
+    readonly uuid?: string;
+    readonly supertest?: Supertest.SuperTest<Supertest.Test>;
+  };
+
+  constructor(config: Config, log: ToolingLog) {
+    if (!config.get('kbnTestServer.useDedicatedTaskRunner')) {
+      this.enabled = false;
+      return;
+    }
+
+    this.enabled = true;
+
+    const port = DedicatedTaskRunner.getPort(config.get('servers.kibana.port'));
+    const url = Url.format({
+      ...config.get('servers.kibana'),
+      port,
+    });
+    const client = new KbnClient({
+      log,
+      url,
+      certificateAuthorities: config.get('servers.kibana.certificateAuthorities'),
+      uiSettingDefaults: config.get('uiSettings.defaults'),
+    });
+
+    const mainUuid = getKibanaCliArg(config.get('kbnTestServer.serverArgs'), 'server.uuid');
+    const uuid = typeof mainUuid === 'string' ? DedicatedTaskRunner.getUuid(mainUuid) : undefined;
+
+    this.enabledProps = { port, url, client, uuid };
+  }
+
+  private getEnabledProps() {
+    if (!this.enabledProps) {
+      throw new Error(
+        `DedicatedTaskRunner is not enabled, check the "enabled" property before calling getters assuming it is enabled.`
+      );
+    }
+
+    return this.enabledProps;
+  }
+
+  /**
+   * The port number that the dedicated task runner is running on
+   */
+  getPort() {
+    return this.getEnabledProps().port;
+  }
+
+  /**
+   * The full URL for the dedicated task runner process
+   */
+  getUrl() {
+    return this.getEnabledProps().url;
+  }
+
+  /**
+   * Returns true if the `--server.uuid` setting was passed to the Kibana server, allowing the UUID to
+   * be deterministic and ensuring that `dedicatedTaskRunner.getUuid()` won't throw.
+   */
+  hasUuid() {
+    return !!this.getEnabledProps().uuid;
+  }
+
+  /**
+   * If `--server.uuid` is passed to Kibana in the FTR config file then the dedicated task runner will
+   * use a UUID derived from that and it will be synchronously available to users via this function.
+   * Otherwise this function will through.
+   */
+  getUuid() {
+    const uuid = this.getEnabledProps().uuid;
+    if (!uuid) {
+      throw new Error(
+        'Pass `--server.uuid` the the Kibana server in your FTR config in order to make the UUID of the dedicated task runner deterministic.'
+      );
+    }
+
+    return uuid;
+  }
+
+  /**
+   * @returns a `KbnClient` instance that is configured to talk directly to the dedicated task runner. Not really sure how useful this is.
+   */
+  getClient() {
+    return this.getEnabledProps().client;
+  }
+
+  /**
+   * @returns a Supertest instance that will send requests to the dedicated task runner.
+   *
+   * @example
+   *  const supertest = dedicatedTaskRunner.getSupertest();
+   *  const response = await supertest.get('/status');
+   */
+  getSupertest() {
+    return Supertest(this.getUrl());
+  }
+}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -14,6 +14,7 @@ export * from './providers';
 export { runTests, setupMocha } from './mocha';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
+export { DedicatedTaskRunner } from './dedicated_task_runner';
 
 export type { Provider } from './providers';
 export * from './es_version';

packages/kbn-test/src/functional_test_runner/public_types.ts

@@ -8,7 +8,13 @@
 
 import type { ToolingLog } from '@kbn/tooling-log';
 
-import type { Config, Lifecycle, DockerServersService, EsVersion } from './lib';
+import type {
+  Config,
+  Lifecycle,
+  DockerServersService,
+  EsVersion,
+  DedicatedTaskRunner,
+} from './lib';
 import type { Test, Suite } from './fake_mocha_types';
 
 export { Lifecycle, Config };
@@ -56,7 +62,15 @@ export interface GenericFtrProviderContext<
    * Determine if a service is avaliable
    * @param serviceName
    */
-  hasService(serviceName: 'config' | 'log' | 'lifecycle' | 'dockerServers' | 'esVersion'): true;
+  hasService(
+    serviceName:
+      | 'config'
+      | 'log'
+      | 'lifecycle'
+      | 'dockerServers'
+      | 'esVersion'
+      | 'dedicatedTaskRunner'
+  ): true;
   hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
   hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
 
@@ -70,6 +84,7 @@ export interface GenericFtrProviderContext<
   getService(serviceName: 'lifecycle'): Lifecycle;
   getService(serviceName: 'dockerServers'): DockerServersService;
   getService(serviceName: 'esVersion'): EsVersion;
+  getService(serviceName: 'dedicatedTaskRunner'): DedicatedTaskRunner;
   getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
 
   /**

packages/kbn-test/src/functional_tests/lib/index.ts

@@ -10,5 +10,4 @@ export { runKibanaServer } from './run_kibana_server';
 export { runElasticsearch } from './run_elasticsearch';
 export type { CreateFtrOptions, CreateFtrParams } from './run_ftr';
 export { runFtr, hasTests, assertNoneExcluded } from './run_ftr';
-export { KIBANA_ROOT, KIBANA_FTR_SCRIPT } from './paths';
 export { runCli } from './run_cli';

packages/kbn-test/src/functional_tests/lib/paths.ts

@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { REPO_ROOT } from '@kbn/utils';
-import { resolve, relative } from 'path';
-
-// resolve() treats relative paths as relative to process.cwd(),
-// so to return a relative path we use relative()
-function resolveRelative(path: string) {
-  return relative(process.cwd(), resolve(path));
-}
-
-export const KIBANA_EXEC = 'node';
-export const KIBANA_SCRIPT_PATH = resolveRelative('scripts/kibana');
-export const KIBANA_ROOT = REPO_ROOT;
-export const KIBANA_FTR_SCRIPT = resolve(KIBANA_ROOT, 'scripts/functional_test_runner');

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts

@@ -9,7 +9,7 @@
 import { resolve } from 'path';
 import type { ToolingLog } from '@kbn/tooling-log';
 import getPort from 'get-port';
-import { KIBANA_ROOT } from './paths';
+import { REPO_ROOT } from '@kbn/utils';
 import type { Config } from '../../functional_test_runner';
 import { createTestEsCluster } from '../../es';
 
@@ -106,7 +106,7 @@ async function startEsNode(
     port: config.port,
     ssl: config.ssl,
     log,
-    basePath: resolve(KIBANA_ROOT, '.es'),
+    basePath: resolve(REPO_ROOT, '.es'),
     nodes: [
       {
         name,

packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts

@@ -7,12 +7,15 @@
  */
 
 import Path from 'path';
+import Os from 'os';
 
+import Uuid from 'uuid';
 import type { ProcRunner } from '@kbn/dev-proc-runner';
+import { REPO_ROOT } from '@kbn/utils';
 
-import { KIBANA_ROOT, KIBANA_EXEC, KIBANA_SCRIPT_PATH } from './paths';
 import type { Config } from '../../functional_test_runner';
-import { parseRawFlags } from './kibana_cli_args';
+import { DedicatedTaskRunner } from '../../functional_test_runner/lib';
+import { parseRawFlags, getArgValue } from './kibana_cli_args';
 
 function extendNodeOptions(installDir?: string) {
   if (!installDir) {
@@ -44,13 +47,35 @@ export async function runKibanaServer({
 }) {
   const runOptions = config.get('kbnTestServer.runOptions');
   const installDir = runOptions.alwaysUseSource ? undefined : options.installDir;
-  const extraArgs = options.extraKbnOpts ?? [];
+  const devMode = !installDir;
+  const useTaskRunner = config.get('kbnTestServer.useDedicatedTaskRunner');
+
+  const procRunnerOpts = {
+    cwd: installDir || REPO_ROOT,
+    cmd: installDir
+      ? process.platform.startsWith('win')
+        ? Path.resolve(installDir, 'bin/kibana.bat')
+        : Path.resolve(installDir, 'bin/kibana')
+      : process.execPath,
+    env: {
+      FORCE_COLOR: 1,
+      ...process.env,
+      ...config.get('kbnTestServer.env'),
+      ...extendNodeOptions(installDir),
+    },
+    wait: runOptions.wait,
+    onEarlyExit,
+  };
+
+  const prefixArgs = devMode
+    ? [Path.relative(process.cwd(), Path.resolve(REPO_ROOT, 'scripts/kibana'))]
+    : [];
 
   const buildArgs: string[] = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs: string[] = config.get('kbnTestServer.sourceArgs') || [];
   const serverArgs: string[] = config.get('kbnTestServer.serverArgs') || [];
 
-  const args = parseRawFlags([
+  const kbnFlags = parseRawFlags([
     // When installDir is passed, we run from a built version of Kibana which uses different command line
     // arguments. If installDir is not passed, we run from source code.
     ...(installDir
@@ -58,31 +83,51 @@ export async function runKibanaServer({
       : [...sourceArgs, ...serverArgs]),
 
     // We also allow passing in extra Kibana server options, tack those on here so they always take precedence
-    ...extraArgs,
+    ...(options.extraKbnOpts ?? []),
   ]);
 
-  // main process
-  await procs.run('kibana', {
-    cmd: getKibanaCmd(installDir),
-    args: installDir ? args : [KIBANA_SCRIPT_PATH, ...args],
-    env: {
-      FORCE_COLOR: 1,
-      ...process.env,
-      ...config.get('kbnTestServer.env'),
-      ...extendNodeOptions(installDir),
-    },
-    cwd: installDir || KIBANA_ROOT,
-    wait: runOptions.wait,
-    onEarlyExit,
-  });
-}
+  const promises = [
+    // main process
+    procs.run(useTaskRunner ? 'kbn-ui' : 'kibana', {
+      ...procRunnerOpts,
+      args: [
+        ...prefixArgs,
+        ...parseRawFlags([
+          ...kbnFlags,
+          ...(!useTaskRunner
+            ? []
+            : [
+                '--node.roles=["ui"]',
+                `--path.data=${Path.resolve(Os.tmpdir(), `ftr-ui-${Uuid.v4()}`)}`,
+              ]),
+        ]),
+      ],
+    }),
+  ];
 
-function getKibanaCmd(installDir?: string) {
-  if (installDir) {
-    return process.platform.startsWith('win')
-      ? Path.resolve(installDir, 'bin/kibana.bat')
-      : Path.resolve(installDir, 'bin/kibana');
+  if (useTaskRunner) {
+    const mainUuid = getArgValue(kbnFlags, 'server.uuid');
+
+    // dedicated task runner
+    promises.push(
+      procs.run('kbn-tasks', {
+        ...procRunnerOpts,
+        args: [
+          ...prefixArgs,
+          ...parseRawFlags([
+            ...kbnFlags,
+            `--server.port=${DedicatedTaskRunner.getPort(config.get('servers.kibana.port'))}`,
+            '--node.roles=["background_tasks"]',
+            `--path.data=${Path.resolve(Os.tmpdir(), `ftr-task-runner-${Uuid.v4()}`)}`,
+            ...(typeof mainUuid === 'string' && mainUuid
+              ? [`--server.uuid=${DedicatedTaskRunner.getUuid(mainUuid)}`]
+              : []),
+            ...(devMode ? ['--no-optimizer'] : []),
+          ]),
+        ],
+      })
+    );
   }
 
-  return KIBANA_EXEC;
+  await Promise.all(promises);
 }

packages/kbn-test/src/functional_tests/tasks.ts

@@ -6,9 +6,10 @@
  * Side Public License, v 1.
  */
 
-import { relative } from 'path';
-import * as Rx from 'rxjs';
+import Path from 'path';
 import { setTimeout } from 'timers/promises';
+
+import * as Rx from 'rxjs';
 import { startWith, switchMap, take } from 'rxjs/operators';
 import { withProcRunner } from '@kbn/dev-proc-runner';
 import { ToolingLog } from '@kbn/tooling-log';
@@ -16,17 +17,17 @@ import { getTimeReporter } from '@kbn/ci-stats-reporter';
 import { REPO_ROOT } from '@kbn/utils';
 import dedent from 'dedent';
 
+import { readConfigFile, EsVersion } from '../functional_test_runner/lib';
 import {
   runElasticsearch,
   runKibanaServer,
   runFtr,
   assertNoneExcluded,
   hasTests,
-  KIBANA_FTR_SCRIPT,
   CreateFtrOptions,
 } from './lib';
 
-import { readConfigFile, EsVersion } from '../functional_test_runner/lib';
+const FTR_SCRIPT_PATH = Path.resolve(REPO_ROOT, 'scripts/functional_test_runner');
 
 const makeSuccessMessage = (options: StartServerOptions) => {
   const installDirFlag = options.installDir ? ` --kibana-install-dir=${options.installDir}` : '';
@@ -34,7 +35,7 @@ const makeSuccessMessage = (options: StartServerOptions) => {
   const pathsMessage = options.useDefaultConfig
     ? ''
     : configPaths
-        .map((path) => relative(process.cwd(), path))
+        .map((path) => Path.relative(process.cwd(), path))
         .map((path) => ` --config ${path}`)
         .join('');
 
@@ -44,7 +45,7 @@ const makeSuccessMessage = (options: StartServerOptions) => {
       Elasticsearch and Kibana are ready for functional testing. Start the functional tests
       in another terminal session by running this command from this directory:
 
-          node ${relative(process.cwd(), KIBANA_FTR_SCRIPT)}${installDirFlag}${pathsMessage}
+          node ${Path.relative(process.cwd(), FTR_SCRIPT_PATH)}${installDirFlag}${pathsMessage}
     ` +
     '\n\n'
   );
@@ -96,7 +97,7 @@ export async function runTests(options: RunTestsParams) {
     await log.indent(0, async () => {
       if (options.configs.length > 1) {
         const progress = `${i + 1}/${options.configs.length}`;
-        log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`);
+        log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, configPath)}`);
       }
 
       if (!(await hasTests({ configPath, options: { ...options, log } }))) {

packages/kbn-test/src/index.ts

@@ -24,9 +24,6 @@ export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartSe
 // @internal
 export { runTests, startServers } from './functional_tests/tasks';
 
-// @internal
-export { KIBANA_ROOT } from './functional_tests/lib/paths';
-
 export { getKibanaCliArg, getKibanaCliLoggers } from './functional_tests/lib/kibana_cli_args';
 
 export type {

test/examples/config.js

@@ -9,15 +9,15 @@
 import { resolve } from 'path';
 import { services } from '../plugin_functional/services';
 import fs from 'fs';
-import { KIBANA_ROOT } from '@kbn/test';
+import { REPO_ROOT } from '@kbn/utils';
 
 export default async function ({ readConfigFile }) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   // Find all folders in /examples and /x-pack/examples since we treat all them as plugin folder
-  const examplesFiles = fs.readdirSync(resolve(KIBANA_ROOT, 'examples'));
+  const examplesFiles = fs.readdirSync(resolve(REPO_ROOT, 'examples'));
   const examples = examplesFiles.filter((file) =>
-    fs.statSync(resolve(KIBANA_ROOT, 'examples', file)).isDirectory()
+    fs.statSync(resolve(REPO_ROOT, 'examples', file)).isDirectory()
   );
 
   return {
@@ -63,7 +63,7 @@ export default async function ({ readConfigFile }) {
         '--env.name=development',
         '--telemetry.optIn=false',
         ...examples.map(
-          (exampleDir) => `--plugin-path=${resolve(KIBANA_ROOT, 'examples', exampleDir)}`
+          (exampleDir) => `--plugin-path=${resolve(REPO_ROOT, 'examples', exampleDir)}`
         ),
       ],
     },

x-pack/test/examples/config.ts

@@ -8,8 +8,7 @@
 import { FtrConfigProviderContext } from '@kbn/test';
 import { resolve } from 'path';
 import fs from 'fs';
-// @ts-expect-error https://github.com/elastic/kibana/issues/95679
-import { KIBANA_ROOT } from '@kbn/test';
+import { REPO_ROOT as KIBANA_ROOT } from '@kbn/utils';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xpackFunctionalConfig = await readConfigFile(

x-pack/test/licensing_plugin/config.public.ts

@@ -6,8 +6,7 @@
  */
 
 import path from 'path';
-// @ts-expect-error https://github.com/elastic/kibana/issues/95679
-import { KIBANA_ROOT } from '@kbn/test';
+import { REPO_ROOT as KIBANA_ROOT } from '@kbn/utils';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {

x-pack/test/plugin_functional/config.ts

@@ -7,8 +7,7 @@
 
 import { resolve } from 'path';
 import fs from 'fs';
-// @ts-expect-error https://github.com/elastic/kibana/issues/95679
-import { KIBANA_ROOT } from '@kbn/test';
+import { REPO_ROOT as KIBANA_ROOT } from '@kbn/utils';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 import { pageObjects } from './page_objects';

x-pack/test/security_solution_endpoint/services/endpoint_telemetry.ts

@@ -7,8 +7,7 @@
 
 import fs from 'fs';
 import Path from 'path';
-// @ts-expect-error https://github.com/elastic/kibana/issues/95679
-import { KIBANA_ROOT } from '@kbn/test';
+import { REPO_ROOT as KIBANA_ROOT } from '@kbn/utils';
 import { FtrProviderContext } from '../ftr_provider_context';
 
 const TELEMETRY_API_ROOT = '/api/stats?extended=true';