[serverless] add support for loading serverless specific config locally (#149878)

This adds the `--serverless` CLI arg (only available in dev mode, you
can also use `yarn start-serverless`), which will load the new
`config/kibana.serverless.yml` file. For now, this file is not included
in the build artifact, though we might include a `--serverless` flag
which replaces the `kibana.yml` file with `kibana.serverless.yml`.
@jbudz will follow up after this PR with the build related changes to
get this working with PR cloud deploys, which will be enough changes for
us to start iterating on UI specific changes based on running in a
serverless environment.

Additionally, support for the undocumented `KBN_CONFIG_PATHS` env var is
added, which should contain a comma-separated list of paths to kibana
config files. These files are loaded, in the specified order, before any
of the config files listed in the CLI but after the kibana.yml config
file.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.gitignore

@@ -50,6 +50,7 @@ disabledPlugins
 webpackstats.json
 /config/*
 !/config/kibana.yml
+!/config/kibana.serverless.yml
 !/config/node.options
 coverage
 selenium

config/kibana.serverless.yml

@@ -0,0 +1,9 @@
+# as work on serverless picks up we will add config values to this file that
+# define how Kibana will run in "serverless" mode. To start Kibana locally with
+# this configuration, pass `--serverless` or run `yarn start-serverless`
+
+# configuration is applied in the following order, later values override
+#  1. kibana.yml
+#  2. kibana.serverless.yml (when --serverless is passed)
+#  3. kibana.dev.yml
+#  4. kibana.serverless.dev.yml (when --serverless is passed)

package.json

@@ -56,6 +56,7 @@
     "makelogs": "node scripts/makelogs",
     "spec_to_console": "node scripts/spec_to_console",
     "start": "node scripts/kibana --dev",
+    "start-serverless": "node scripts/kibana --dev --serverless",
     "storybook": "node scripts/storybook",
     "test:ftr": "node scripts/functional_tests",
     "test:ftr:runner": "node scripts/functional_test_runner",

src/cli/serve/serve.js

@@ -12,8 +12,8 @@ import { statSync } from 'fs';
 import { resolve } from 'path';
 import url from 'url';
 
-import { getConfigPath } from '@kbn/utils';
-import { fromRoot, isKibanaDistributable } from '@kbn/repo-info';
+import { getConfigPath, getConfigDirectory } from '@kbn/utils';
+import { isKibanaDistributable } from '@kbn/repo-info';
 import { readKeystore } from '../keystore/read_keystore';
 
 function canRequire(path) {
@@ -55,6 +55,40 @@ const pathCollector = function () {
 const configPathCollector = pathCollector();
 const pluginPathCollector = pathCollector();
 
+/**
+ * @param {string} name
+ * @param {string[]} configs
+ * @param {'push' | 'unshift'} method
+ */
+function maybeAddConfig(name, configs, method) {
+  const path = resolve(getConfigDirectory(), name);
+  try {
+    if (statSync(path).isFile()) {
+      configs[method](path);
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return;
+    }
+
+    throw err;
+  }
+}
+
+/**
+ * @returns {string[]}
+ */
+function getEnvConfigs() {
+  const val = process.env.KBN_CONFIG_PATHS;
+  if (typeof val === 'string') {
+    return val
+      .split(',')
+      .filter((v) => !!v)
+      .map((p) => resolve(p.trim()));
+  }
+  return [];
+}
+
 function applyConfigOverrides(rawConfig, opts, extraCliOptions) {
   const set = _.partial(lodashSet, rawConfig);
   const get = _.partial(_.get, rawConfig);
@@ -152,7 +186,7 @@ export default function (program) {
       '-c, --config <path>',
       'Path to the config file, use multiple --config args to include multiple config files',
       configPathCollector,
-      [getConfigPath()]
+      []
     )
     .option('-p, --port <port>', 'The port to bind to', parseInt)
     .option('-Q, --silent', 'Set the root logger level to off')
@@ -177,7 +211,8 @@ export default function (program) {
       .option(
         '--run-examples',
         'Adds plugin paths for all the Kibana example plugins and runs with no base path'
-      );
+      )
+      .option('--serverless', 'Start Kibana with serverless configuration overrides');
   }
 
   if (DEV_MODE_SUPPORTED) {
@@ -200,19 +235,22 @@ export default function (program) {
   }
 
   command.action(async function (opts) {
+    const unknownOptions = this.getUnknownOptions();
+    const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])];
+
+    // we "unshift" .serverless. config so that it only overrides defaults
+    if (opts.serverless) {
+      maybeAddConfig('kibana.serverless.yml', configs, 'unshift');
+    }
+
+    // .dev. configs are "pushed" so that they override all other config files
     if (opts.dev && opts.devConfig !== false) {
-      try {
-        const kbnDevConfig = fromRoot('config/kibana.dev.yml');
-        if (statSync(kbnDevConfig).isFile()) {
-          opts.config.push(kbnDevConfig);
-        }
-      } catch (err) {
-        // ignore, kibana.dev.yml does not exist
+      maybeAddConfig('kibana.dev.yml', configs, 'push');
+      if (opts.serverless) {
+        maybeAddConfig('kibana.serverless.dev.yml', configs, 'push');
       }
     }
 
-    const unknownOptions = this.getUnknownOptions();
-    const configs = [].concat(opts.config || []);
     const cliArgs = {
       dev: !!opts.dev,
       envName: unknownOptions.env ? unknownOptions.env.name : undefined,
@@ -231,6 +269,7 @@ export default function (program) {
       oss: !!opts.oss,
       cache: !!opts.cache,
       dist: !!opts.dist,
+      serverless: !!opts.serverless,
     };
 
     // In development mode, the main process uses the @kbn/dev-cli-mode