[Serverless] Select project type via config (#155754)

config/serverless.es.yml

@@ -1 +1,2 @@
+uiSettings.overrides.defaultRoute: /app/enterprise_search/content/search_indices
 xpack.serverless.plugin.developer.projectSwitcher.currentType: 'search'

config/serverless.oblt.yml

@@ -1,2 +1,3 @@
+uiSettings.overrides.defaultRoute: /app/observability/overview
 xpack.infra.logs.app_target: discover
 xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability'

config/serverless.security.yml

@@ -1 +1,2 @@
+uiSettings.overrides.defaultRoute: /app/security/get_started
 xpack.serverless.plugin.developer.projectSwitcher.currentType: 'security'

config/serverless.yml

@@ -15,5 +15,5 @@ xpack.snapshot_restore.enabled: false
 xpack.license_management.enabled: false
 
 # Other disabled plugins
-xpack.canvas.enabled: false
+#xpack.canvas.enabled: false #only disabable in dev-mode
 xpack.reporting.enabled: false

packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Env } from '@kbn/config';
+import { rawConfigServiceMock, configServiceMock } from '@kbn/config-mocks';
+
+export const mockConfigService = configServiceMock.create();
+export const mockRawConfigService = rawConfigServiceMock.create();
+export const mockRawConfigServiceConstructor = jest.fn(() => mockRawConfigService);
+jest.doMock('@kbn/config', () => ({
+  ConfigService: jest.fn(() => mockConfigService),
+  Env,
+  RawConfigService: jest.fn(mockRawConfigServiceConstructor),
+}));
+
+jest.doMock('./root', () => ({
+  Root: jest.fn(() => ({
+    shutdown: jest.fn(),
+  })),
+}));

packages/core/root/core-root-server-internal/src/bootstrap.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { of } from 'rxjs';
+import type { CliArgs } from '@kbn/config';
+
+import { mockRawConfigService, mockRawConfigServiceConstructor } from './bootstrap.test.mocks';
+
+jest.mock('@kbn/core-logging-server-internal');
+
+import { bootstrap } from './bootstrap';
+
+const bootstrapCfg = {
+  configs: ['config/kibana.yml'],
+  cliArgs: {} as unknown as CliArgs,
+  applyConfigOverrides: () => ({}),
+};
+
+describe('bootstrap', () => {
+  describe('serverless', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    test('should load additional serverless files for a valid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'es' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(2);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        2,
+        [
+          expect.stringContaining('config/serverless.yml'),
+          expect.stringContaining('config/serverless.es.yml'),
+          ...bootstrapCfg.configs,
+        ],
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+
+    test('should skip loading the serverless files for an invalid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'not-valid' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(1);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+  });
+});

packages/core/root/core-root-server-internal/src/bootstrap.ts

@@ -7,9 +7,14 @@
  */
 
 import chalk from 'chalk';
+import { firstValueFrom } from 'rxjs';
 import { getPackages } from '@kbn/repo-packages';
 import { CliArgs, Env, RawConfigService } from '@kbn/config';
 import { CriticalError } from '@kbn/core-base-server-internal';
+import { resolve } from 'path';
+import { getConfigDirectory } from '@kbn/utils';
+import { statSync } from 'fs';
+import { VALID_SERVERLESS_PROJECT_TYPES } from './root/serverless_config';
 import { Root } from './root';
 import { MIGRATION_EXCEPTION_CODE } from './constants';
 
@@ -38,15 +43,40 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   const { REPO_ROOT } = require('@kbn/repo-info');
 
-  const env = Env.createDefault(REPO_ROOT, {
+  let env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
     repoPackages: getPackages(REPO_ROOT),
   });
 
-  const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+  let rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
   rawConfigService.loadConfig();
 
+  // Hack to load the extra serverless config files if `serverless: {projectType}` is found in it.
+  const rawConfig = await firstValueFrom(rawConfigService.getConfig$());
+  const serverlessProjectType = rawConfig?.serverless;
+  if (
+    typeof serverlessProjectType === 'string' &&
+    VALID_SERVERLESS_PROJECT_TYPES.includes(serverlessProjectType)
+  ) {
+    const extendedConfigs = [
+      ...['serverless.yml', `serverless.${serverlessProjectType}.yml`]
+        .map((name) => resolve(getConfigDirectory(), name))
+        .filter(configFileExists),
+      ...configs,
+    ];
+
+    env = Env.createDefault(REPO_ROOT, {
+      configs: extendedConfigs,
+      cliArgs: { ...cliArgs, serverless: true },
+      repoPackages: getPackages(REPO_ROOT),
+    });
+
+    rawConfigService.stop();
+    rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+    rawConfigService.loadConfig();
+  }
+
   const root = new Root(rawConfigService, env, onRootShutdown);
 
   process.on('SIGHUP', () => reloadConfiguration());
@@ -128,3 +158,15 @@ function onRootShutdown(reason?: any) {
 
   process.exit(0);
 }
+
+function configFileExists(path: string) {
+  try {
+    return statSync(path).isFile();
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return false;
+    }
+
+    throw err;
+  }
+}

packages/core/root/core-root-server-internal/src/register_service_config.ts

@@ -28,6 +28,7 @@ import { uiSettingsConfig } from '@kbn/core-ui-settings-server-internal';
 
 import { config as pluginsConfig } from '@kbn/core-plugins-server-internal';
 import { elasticApmConfig } from './root/elastic_config';
+import { serverlessConfig } from './root/serverless_config';
 
 const rootConfigPath = '';
 
@@ -49,6 +50,7 @@ export function registerServiceConfig(configService: ConfigService) {
     pluginsConfig,
     savedObjectsConfig,
     savedObjectsMigrationConfig,
+    serverlessConfig,
     statusConfig,
     uiSettingsConfig,
   ];

packages/core/root/core-root-server-internal/src/root/serverless_config.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf, Type } from '@kbn/config-schema';
+import { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
+
+// Config validation for how to run Kibana in Serverless mode.
+// Clients need to specify the project type to run in.
+// Going for a simple `serverless` string because it serves as
+// a direct replacement to the legacy --serverless CLI flag.
+// If we even decide to extend this further, and converting it into an object,
+// BWC can be ensured by adding the object definition as another alternative to `schema.oneOf`.
+
+export const VALID_SERVERLESS_PROJECT_TYPES = ['es', 'oblt', 'security'];
+
+const serverlessConfigSchema = schema.maybe(
+  schema.oneOf(
+    VALID_SERVERLESS_PROJECT_TYPES.map((projectName) => schema.literal(projectName)) as [
+      Type<typeof VALID_SERVERLESS_PROJECT_TYPES[number]> // This cast is needed because it's different to Type<T>[] :sight:
+    ]
+  )
+);
+
+export type ServerlessConfigType = TypeOf<typeof serverlessConfigSchema>;
+
+export const serverlessConfig: ServiceConfigDescriptor<ServerlessConfigType> = {
+  path: 'serverless',
+  schema: serverlessConfigSchema,
+};

src/cli/serve/integration_tests/serverless_config_flag.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { spawn, spawnSync } from 'child_process';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { filter, firstValueFrom, from, take, concatMap } from 'rxjs';
+
+import { REPO_ROOT } from '@kbn/repo-info';
+import { getConfigDirectory } from '@kbn/utils';
+
+describe('cli serverless project type', () => {
+  it(
+    'exits with statusCode 1 and logs an error when serverless project type is invalid',
+    () => {
+      const { error, status, stdout } = spawnSync(
+        process.execPath,
+        ['scripts/kibana', '--serverless=non-existing-project-type'],
+        {
+          cwd: REPO_ROOT,
+        }
+      );
+      expect(error).toBe(undefined);
+
+      expect(stdout.toString('utf8')).toContain(
+        'FATAL CLI ERROR Error: invalid --serverless value, must be one of es, oblt, security'
+      );
+
+      expect(status).toBe(1);
+    },
+    20 * 1000
+  );
+
+  // Skipping this one because on CI it fails to read the config file
+  it.skip.each(['es', 'oblt', 'security'])(
+    'writes the serverless project type %s in config/serverless.recent.yml',
+    async (mode) => {
+      // Making sure `--serverless` translates into the `serverless` config entry, and validates against the accepted values
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait for 5 lines in the logs
+      await firstValueFrom(from(child.stdout).pipe(take(5)));
+
+      expect(
+        readFileSync(resolve(getConfigDirectory(), 'serverless.recent.yml'), 'utf-8')
+      ).toContain(`serverless: ${mode}\n`);
+
+      child.kill('SIGKILL');
+    }
+  );
+
+  it.each(['es', 'oblt', 'security'])(
+    'Kibana does not crash when running project type %s',
+    async (mode) => {
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait until Kibana starts listening to the port
+      let leftover = '';
+      const found = await firstValueFrom(
+        from(child.stdout).pipe(
+          concatMap((chunk: Buffer) => {
+            const data = leftover + chunk.toString('utf-8');
+            const msgs = data.split('\n');
+            leftover = msgs.pop() ?? '';
+            return msgs;
+          }),
+          filter(
+            (msg) =>
+              msg.includes('http server running at http://localhost:5601') || msg.includes('FATAL')
+          )
+        )
+      );
+
+      child.kill('SIGKILL');
+
+      expect(found).not.toContain('FATAL');
+    }
+  );
+});

src/cli/serve/serve.js

@@ -8,7 +8,7 @@
 
 import { set as lodashSet } from '@kbn/safer-lodash-set';
 import _ from 'lodash';
-import { statSync, copyFileSync, existsSync, readFileSync, writeFileSync } from 'fs';
+import { statSync, existsSync, readFileSync, writeFileSync } from 'fs';
 import { resolve } from 'path';
 import url from 'url';
 
@@ -22,7 +22,7 @@ const VALID_SERVERLESS_PROJECT_MODE = ['es', 'oblt', 'security'];
 
 /**
  * @param {Record<string, unknown>} opts
- * @returns {ServerlessProjectMode | null}
+ * @returns {ServerlessProjectMode | true | null}
  */
 function getServerlessProjectMode(opts) {
   if (!opts.serverless) {
@@ -94,16 +94,6 @@ function configFileExists(name) {
   }
 }
 
-/**
- * @returns {boolean} Whether the distribution can run in Serverless mode
- */
-function isServerlessCapableDistribution() {
-  // For now, checking if the `serverless.yml` config file exists should be enough
-  // We could also check the following as well, but I don't think it's necessary:
-  // VALID_SERVERLESS_PROJECT_MODE.some((projectType) => configFileExists(`serverless.${projectType}.yml`))
-  return configFileExists('serverless.yml');
-}
-
 /**
  * @param {string} name
  * @param {string[]} configs
@@ -117,24 +107,34 @@ function maybeAddConfig(name, configs, method) {
 
 /**
  * @param {string} file
- * @param {'es' | 'security' | 'oblt' | true} mode
+ * @param {'es' | 'security' | 'oblt' | true} projectType
+ * @param {boolean} isDevMode
  * @param {string[]} configs
  * @param {'push' | 'unshift'} method
  */
-function maybeSetRecentConfig(file, mode, configs, method) {
+function maybeSetRecentConfig(file, projectType, isDevMode, configs, method) {
   const path = resolve(getConfigDirectory(), file);
 
+  function writeMode(selectedProjectType) {
+    writeFileSync(
+      path,
+      `${
+        isDevMode ? 'xpack.serverless.plugin.developer.projectSwitcher.enabled: true\n' : ''
+      }serverless: ${selectedProjectType}\n`
+    );
+  }
+
   try {
-    if (mode === true) {
+    if (projectType === true) {
       if (!existsSync(path)) {
-        const data = readFileSync(path.replace('recent', 'es'), 'utf-8');
-        writeFileSync(
-          path,
-          `${data}\nxpack.serverless.plugin.developer.projectSwitcher.enabled: true\n`
-        );
+        writeMode('es');
       }
     } else {
-      copyFileSync(path.replace('recent', mode), path);
+      const data = readFileSync(path, 'utf-8');
+      const match = data.match(/serverless: (\w+)\n/);
+      if (!match || match[1] !== projectType) {
+        writeMode(projectType);
+      }
     }
 
     configs[method](path);
@@ -283,18 +283,11 @@ export default function (program) {
       .option(
         '--run-examples',
         'Adds plugin paths for all the Kibana example plugins and runs with no base path'
-      );
-  }
-
-  if (isServerlessCapableDistribution()) {
-    command
-      .option(
-        '--serverless',
-        'Start Kibana in the most recent serverless project mode, (default is es)'
       )
       .option(
-        '--serverless <oblt|security|es>',
-        'Start Kibana in a specific serverless project mode'
+        '--serverless [oblt|security|es]',
+        'Start Kibana in a specific serverless project mode. ' +
+          'If no mode is provided, it starts Kibana in the most recent serverless project mode (default is es)'
       );
   }
 
@@ -322,10 +315,8 @@ export default function (program) {
     const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])];
     const serverlessMode = getServerlessProjectMode(opts);
 
-    // we "unshift" .serverless. config so that it only overrides defaults
     if (serverlessMode) {
-      maybeAddConfig(`serverless.yml`, configs, 'push');
-      maybeSetRecentConfig('serverless.recent.yml', serverlessMode, configs, 'unshift');
+      maybeSetRecentConfig('serverless.recent.yml', serverlessMode, opts.dev, configs, 'push');
     }
 
     // .dev. configs are "pushed" so that they override all other config files
@@ -333,7 +324,7 @@ export default function (program) {
       maybeAddConfig('kibana.dev.yml', configs, 'push');
       if (serverlessMode) {
         maybeAddConfig(`serverless.dev.yml`, configs, 'push');
-        maybeSetRecentConfig('serverless.recent.dev.yml', serverlessMode, configs, 'unshift');
+        maybeAddConfig('serverless.recent.dev.yml', configs, 'push');
       }
     }
 
@@ -355,7 +346,6 @@ export default function (program) {
       oss: !!opts.oss,
       cache: !!opts.cache,
       dist: !!opts.dist,
-      serverless: !!opts.serverless,
     };
 
     // In development mode, the main process uses the @kbn/dev-cli-mode

x-pack/plugins/serverless/server/plugin.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { writeFileSync } from 'fs';
 import { resolve } from 'path';
 
 import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from '@kbn/core/server';
@@ -56,27 +56,21 @@ export class ServerlessPlugin implements Plugin<ServerlessPluginSetup, Serverles
         },
         async (_context, request, response) => {
           const { id } = request.body;
-          const path = resolve(getConfigDirectory(), `serverless.${typeToIdMap[id]}.yml`);
+          const selectedProjectType = typeToIdMap[id];
 
           try {
-            if (existsSync(path)) {
-              const data = readFileSync(path, 'utf8');
+            // The switcher is not enabled by default, in cases where one has started Serverless
+            // with a specific config.  So in this case, to ensure the switcher remains enabled,
+            // write the selected config to `recent` and tack on the setting to enable the switcher.
+            writeFileSync(
+              resolve(getConfigDirectory(), 'serverless.recent.yml'),
+              `xpack.serverless.plugin.developer.projectSwitcher.enabled: true\nserverless: ${selectedProjectType}\n`
+            );
 
-              // The switcher is not enabled by default, in cases where one has started Serverless
-              // with a specific config.  So in this case, to ensure the switcher remains enabled,
-              // erite the selected config to `recent` and tack on the setting to enable the switcher.
-              writeFileSync(
-                resolve(getConfigDirectory(), 'serverless.recent.yml'),
-                `${data}\nxpack.serverless.plugin.developer.projectSwitcher.enabled: true\n`
-              );
-
-              return response.ok({ body: id });
-            }
+            return response.ok({ body: id });
           } catch (e) {
             return response.badRequest({ body: e });
           }
-
-          return response.badRequest();
         }
       );
     }