Option to run kibana from build for CI (#19125) (#19225) (#19256)

* Option to run kibana from build for CI (#19125)

* Support --dev option for servers only

* try skipping dashboard time zones test

packages/kbn-test/src/functional_tests/cli/run_tests_cli.js

@@ -12,7 +12,9 @@ import { runTests } from '../../';
  *                                       if no config option is passed
  */
 export async function runTestsCli(defaultConfigPaths) {
-  const { configs, help, bail, log } = processArgs(defaultConfigPaths);
+  const { configs, help, bail, log, installDir } = processArgs(
+    defaultConfigPaths
+  );
 
   if (help) return displayHelp();
 
@@ -24,7 +26,7 @@ export async function runTestsCli(defaultConfigPaths) {
   }
 
   try {
-    await runTests(configs, { bail, log });
+    await runTests(configs, { bail, log, installDir });
   } catch (err) {
     log.error('FATAL ERROR');
     log.error(err);
@@ -50,6 +52,7 @@ function processArgs(defaultConfigPaths) {
     log,
     help: options.help,
     bail: options.bail,
+    installDir: options['kibana-install-dir'],
     rest: options._,
   };
 }
@@ -64,6 +67,9 @@ function displayHelp() {
     --config      Option to pass in a config
                   Can pass in multiple configs with
                   --config file1 --config file2 --config file3
+    --kibana-install-dir
+                  Run Kibana from an existing install directory
+                  Default: run from source
     --bail        Stop the test run at the first failure
     --help        Display this menu and exit
 

packages/kbn-test/src/functional_tests/cli/start_servers_cli.js

@@ -9,7 +9,7 @@ import { startServers } from '../../';
  * @param {string} configPath path to config
  */
 export async function startServersCli(defaultConfigPath) {
-  const { config, log, help } = processArgv(defaultConfigPath);
+  const { config, log, help, installDir } = processArgv(defaultConfigPath);
 
   if (help) return displayHelp();
 
@@ -21,7 +21,7 @@ export async function startServersCli(defaultConfigPath) {
   }
 
   try {
-    await startServers(config, { log });
+    await startServers(config, { log, installDir });
   } catch (err) {
     log.error('FATAL ERROR');
     log.error(err);
@@ -50,6 +50,7 @@ function processArgv(defaultConfigPath) {
   return {
     config,
     log,
+    installDir: options.kibanaInstallDir,
     help: options.help,
     rest: options._,
   };
@@ -63,6 +64,9 @@ function displayHelp() {
     Usage:  node scripts/functional_tests_server [options]
 
     --config      Option to pass in a config
+    --kibana-install-dir
+                  Run Kibana from an existing install directory
+                  Default: run from source
     --help        Display this menu and exit
 
     Log level options:

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -1,17 +1,39 @@
+import { resolve } from 'path';
 import { KIBANA_ROOT, KIBANA_EXEC, KIBANA_EXEC_PATH } from './paths';
 
-export async function runKibanaServer({ procs, config }) {
-  const cliArgs = config.get('kibanaServerArgs') || [];
+export async function runKibanaServer({ procs, config, options }) {
+  const { installDir } = options;
 
-  // start the kibana server and wait for it to log "Server running" before resolving
   await procs.run('kibana', {
-    cmd: KIBANA_EXEC,
-    args: [KIBANA_EXEC_PATH, ...cliArgs],
+    cmd: getKibanaCmd(installDir),
+    args: getCliArgs(config, options),
     env: {
       FORCE_COLOR: 1,
       ...process.env,
     },
-    cwd: KIBANA_ROOT,
+    cwd: installDir || KIBANA_ROOT,
     wait: /Server running/,
   });
 }
+
+function getKibanaCmd(installDir) {
+  if (installDir) {
+    return process.platform.startsWith('win')
+      ? resolve(installDir, 'bin/kibana.bat')
+      : resolve(installDir, 'bin/kibana');
+  }
+
+  return KIBANA_EXEC;
+}
+
+function getCliArgs(config, { devMode, installDir }) {
+  const buildArgs = config.get('kbnTestServer.buildArgs') || [];
+  const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
+  const serverArgs = config.get('kbnTestServer.serverArgs') || [];
+
+  if (devMode) serverArgs.push('--dev');
+
+  return installDir
+    ? [...serverArgs, ...buildArgs]
+    : [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs];
+}

packages/kbn-test/src/functional_tests/tasks.js

@@ -22,29 +22,40 @@ in another terminal session by running this command from this directory:
 
 /**
  * Run servers and tests for each config
- * @param {string[]} configPaths   Array of paths to configs
- * @param {boolean}  bail          Whether to exit test run at the first failure
- * @param {Log}      log           Optional logger
+ * @param {string[]} configPaths         Array of paths to configs
+ * @param {object}   options             Optional
+ * @param {Log}      options.log         Optional logger
+ * @param {string}   options.installDir  Optional installation dir
+ *                                       from which to run Kibana
+ * @param {boolean}  options.bail  Whether to exit test run at the first failure
  */
-export async function runTests(configPaths, { bail, log }) {
+export async function runTests(configPaths, options) {
   for (const configPath of configPaths) {
-    await runSingleConfig(resolve(process.cwd(), configPath), { bail, log });
+    await runSingleConfig(resolve(process.cwd(), configPath), options);
   }
 }
 
 /**
  * Start only servers using single config
- * @param {string}  configPath   Path to a config file
- * @param {Log}     log          Optional logger
+ * @param {string}  configPath          Path to a config file
+ * @param {object}  options             Optional
+ * @param {Log}     options.log         Optional logger
+ * @param {string}  options.installDir  Optional installation dir
+ *                                      from which to run Kibana
  */
-export async function startServers(configPath, { log }) {
+export async function startServers(configPath, options) {
+  const { log } = options;
   configPath = resolve(process.cwd(), configPath);
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
     const es = await runElasticsearch({ config, log });
-    await runKibanaServer({ procs, config, log });
+    await runKibanaServer({
+      procs,
+      config,
+      options: { ...options, devMode: true },
+    });
 
     // wait for 5 seconds of silence before logging the
     // success message so that it doesn't get buried
@@ -67,12 +78,14 @@ async function silence(milliseconds, { log }) {
 /*
  * Start servers and run tests for single config
  */
-async function runSingleConfig(configPath, { bail, log }) {
+async function runSingleConfig(configPath, options) {
+  const { bail, log } = options;
+
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
     const es = await runElasticsearch({ config, log });
-    await runKibanaServer({ procs, config });
+    await runKibanaServer({ procs, config, options });
 
     // Note: When solving how to incorporate functional_test_runner
     // clean this up

src/functional_test_runner/lib/config/schema.js

@@ -94,7 +94,11 @@ export const schema = Joi.object().keys({
     serverArgs: Joi.array(),
   }).default(),
 
-  kibanaServerArgs: Joi.array(),
+  kbnTestServer: Joi.object().keys({
+    buildArgs: Joi.array(),
+    sourceArgs: Joi.array(),
+    serverArgs: Joi.array(),
+  }).default(),
 
   // env allows generic data, but should be removed
   env: Joi.object().default(),

test/api_integration/config.js

@@ -26,11 +26,14 @@ export default async function ({ readConfigFile }) {
     },
     env: commonConfig.get('env'),
     esTestCluster: commonConfig.get('esTestCluster'),
-    kibanaServerArgs: [
-      ...functionalConfig.get('kibanaServerArgs'),
-      '--optimize.enabled=false',
-      '--elasticsearch.healthCheck.delay=3600000',
-      '--server.xsrf.disableProtection=true',
-    ],
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        '--optimize.enabled=false',
+        '--elasticsearch.healthCheck.delay=3600000',
+        '--server.xsrf.disableProtection=true',
+      ],
+    },
   };
 }

test/common/config.js

@@ -23,20 +23,25 @@ export default function () {
       ],
     },
 
-    kibanaServerArgs: [
-      '--env=development',
-      '--logging.json=false',
-      '--no-base-path',
-      `--server.port=${kbnTestConfig.getPort()}`,
-      `--optimize.watchPort=${kbnTestConfig.getPort()}`,
-      '--optimize.watchPrebuild=true',
-      '--status.allowAnonymous=true',
-      '--optimize.enabled=true',
-      `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
-      `--elasticsearch.url=${formatUrl(servers.elasticsearch)}`,
-      `--elasticsearch.username=${servers.elasticsearch.username}`,
-      `--elasticsearch.password=${servers.elasticsearch.password}`,
-    ],
+    kbnTestServer: {
+      buildArgs: [ '--optimize.useBundleCache=true' ],
+      sourceArgs: [
+        '--no-base-path',
+        `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
+      ],
+      serverArgs: [
+        '--env=development',
+        '--logging.json=false',
+        `--server.port=${kbnTestConfig.getPort()}`,
+        `--optimize.watchPort=${kbnTestConfig.getPort()}`,
+        '--optimize.watchPrebuild=true',
+        '--status.allowAnonymous=true',
+        '--optimize.enabled=true',
+        `--elasticsearch.url=${formatUrl(servers.elasticsearch)}`,
+        `--elasticsearch.username=${servers.elasticsearch.username}`,
+        `--elasticsearch.password=${servers.elasticsearch.password}`,
+      ],
+    },
 
     services: {
       kibanaServer: KibanaServerProvider,

test/functional/config.js

@@ -87,10 +87,13 @@ export default async function ({ readConfigFile }) {
 
     esTestCluster: commonConfig.get('esTestCluster'),
 
-    kibanaServerArgs: [
-      ...commonConfig.get('kibanaServerArgs'),
-      '--oss',
-    ],
+    kbnTestServer: {
+      ...commonConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...commonConfig.get('kbnTestServer.serverArgs'),
+        '--oss',
+      ],
+    },
 
     apps: {
       status_page: {

x-pack/test/api_integration/config.js

@@ -28,7 +28,7 @@ export default async function ({ readConfigFile }) {
       reportName: 'X-Pack API Integration Tests',
     },
     env: xPackFunctionalTestsConfig.get('env'),
-    kibanaServerArgs: xPackFunctionalTestsConfig.get('kibanaServerArgs'),
+    kbnTestServer: xPackFunctionalTestsConfig.get('kbnTestServer'),
     esTestCluster: xPackFunctionalTestsConfig.get('esTestCluster'),
   };
 }

x-pack/test/functional/config.js

@@ -157,15 +157,18 @@ export default async function ({ readConfigFile }) {
       ],
     },
 
-    kibanaServerArgs: [
-      ...kibanaCommonConfig.get('kibanaServerArgs'),
-      `--server.uuid=${env.kibana.server.uuid}`,
-      `--server.port=${servers.kibana.port}`,
-      `--elasticsearch.url=${formatUrl(servers.elasticsearch)}`,
-      '--xpack.monitoring.kibana.collection.enabled=false',
-      '--xpack.xpack_main.telemetry.enabled=false',
-      '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions
-    ],
+    kbnTestServer: {
+      ...kibanaCommonConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...kibanaCommonConfig.get('kbnTestServer.serverArgs'),
+        `--server.uuid=${env.kibana.server.uuid}`,
+        `--server.port=${servers.kibana.port}`,
+        `--elasticsearch.url=${formatUrl(servers.elasticsearch)}`,
+        '--xpack.monitoring.kibana.collection.enabled=false',
+        '--xpack.xpack_main.telemetry.enabled=false',
+        '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions
+      ],
+    },
 
     // the apps section defines the urls that
     // `PageObjects.common.navigateTo(appKey)` will use.

x-pack/test/saml_api_integration/config.js

@@ -42,11 +42,14 @@ export default async function ({ readConfigFile }) {
       ],
     },
 
-    kibanaServerArgs: [
-      ...xPackAPITestsConfig.get('kibanaServerArgs'),
-      '--optimize.enabled=false',
-      '--server.xsrf.whitelist=[\"/api/security/v1/saml\"]',
-      '--xpack.security.authProviders=[\"saml\"]',
-    ],
+    kbnTestServer: {
+      ...xPackAPITestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+        '--optimize.enabled=false',
+        '--server.xsrf.whitelist=[\"/api/security/v1/saml\"]',
+        '--xpack.security.authProviders=[\"saml\"]',
+      ],
+    },
   };
 }