[8.8] [Serverless] Select project type via config (#155754) (#155997)

# Backport

This will backport the following commits from `main` to `8.8`:
- [[Serverless] Select project type via config
(#155754)](https://github.com/elastic/kibana/pull/155754)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Alejandro Fernández
Haro","email":"alejandro.haro@elastic.co"},"sourceCommit":{"committedDate":"2023-04-27T04:49:44Z","message":"[Serverless]
Select project type via config
(#155754)","sha":"de64ff5edfc2637c042d800f7c4c62d104f35320","branchLabelMapping":{"^v8.9.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Core","technical
debt","release_note:skip","backport:prev-minor","v8.9.0","Project:Serverless
MVP"],"number":155754,"url":"https://github.com/elastic/kibana/pull/155754","mergeCommit":{"message":"[Serverless]
Select project type via config
(#155754)","sha":"de64ff5edfc2637c042d800f7c4c62d104f35320"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/155754","number":155754,"mergeCommit":{"message":"[Serverless]
Select project type via config
(#155754)","sha":"de64ff5edfc2637c042d800f7c4c62d104f35320"}}]}]
BACKPORT-->

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

config/serverless.oblt.yml

@@ -1 +1,2 @@
+uiSettings.overrides.defaultRoute: /app/observability/overview
 xpack.infra.logs.app_target: discover

packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Env } from '@kbn/config';
+import { rawConfigServiceMock, configServiceMock } from '@kbn/config-mocks';
+
+export const mockConfigService = configServiceMock.create();
+export const mockRawConfigService = rawConfigServiceMock.create();
+export const mockRawConfigServiceConstructor = jest.fn(() => mockRawConfigService);
+jest.doMock('@kbn/config', () => ({
+  ConfigService: jest.fn(() => mockConfigService),
+  Env,
+  RawConfigService: jest.fn(mockRawConfigServiceConstructor),
+}));
+
+jest.doMock('./root', () => ({
+  Root: jest.fn(() => ({
+    shutdown: jest.fn(),
+  })),
+}));

packages/core/root/core-root-server-internal/src/bootstrap.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { of } from 'rxjs';
+import type { CliArgs } from '@kbn/config';
+
+import { mockRawConfigService, mockRawConfigServiceConstructor } from './bootstrap.test.mocks';
+
+jest.mock('@kbn/core-logging-server-internal');
+
+import { bootstrap } from './bootstrap';
+
+const bootstrapCfg = {
+  configs: ['config/kibana.yml'],
+  cliArgs: {} as unknown as CliArgs,
+  applyConfigOverrides: () => ({}),
+};
+
+describe('bootstrap', () => {
+  describe('serverless', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    test('should load additional serverless files for a valid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'es' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(2);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        2,
+        [
+          expect.stringContaining('config/serverless.yml'),
+          expect.stringContaining('config/serverless.es.yml'),
+          ...bootstrapCfg.configs,
+        ],
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+
+    test('should skip loading the serverless files for an invalid project', async () => {
+      mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'not-valid' }));
+      await bootstrap(bootstrapCfg);
+      expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(1);
+      expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith(
+        1,
+        bootstrapCfg.configs,
+        bootstrapCfg.applyConfigOverrides
+      );
+    });
+  });
+});

packages/core/root/core-root-server-internal/src/bootstrap.ts

@@ -7,9 +7,14 @@
  */
 
 import chalk from 'chalk';
+import { firstValueFrom } from 'rxjs';
 import { getPackages } from '@kbn/repo-packages';
 import { CliArgs, Env, RawConfigService } from '@kbn/config';
 import { CriticalError } from '@kbn/core-base-server-internal';
+import { resolve } from 'path';
+import { getConfigDirectory } from '@kbn/utils';
+import { statSync } from 'fs';
+import { VALID_SERVERLESS_PROJECT_TYPES } from './root/serverless_config';
 import { Root } from './root';
 import { MIGRATION_EXCEPTION_CODE } from './constants';
 
@@ -38,15 +43,40 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   const { REPO_ROOT } = require('@kbn/repo-info');
 
-  const env = Env.createDefault(REPO_ROOT, {
+  let env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
     repoPackages: getPackages(REPO_ROOT),
   });
 
-  const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+  let rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
   rawConfigService.loadConfig();
 
+  // Hack to load the extra serverless config files if `serverless: {projectType}` is found in it.
+  const rawConfig = await firstValueFrom(rawConfigService.getConfig$());
+  const serverlessProjectType = rawConfig?.serverless;
+  if (
+    typeof serverlessProjectType === 'string' &&
+    VALID_SERVERLESS_PROJECT_TYPES.includes(serverlessProjectType)
+  ) {
+    const extendedConfigs = [
+      ...['serverless.yml', `serverless.${serverlessProjectType}.yml`]
+        .map((name) => resolve(getConfigDirectory(), name))
+        .filter(configFileExists),
+      ...configs,
+    ];
+
+    env = Env.createDefault(REPO_ROOT, {
+      configs: extendedConfigs,
+      cliArgs: { ...cliArgs, serverless: true },
+      repoPackages: getPackages(REPO_ROOT),
+    });
+
+    rawConfigService.stop();
+    rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
+    rawConfigService.loadConfig();
+  }
+
   const root = new Root(rawConfigService, env, onRootShutdown);
 
   process.on('SIGHUP', () => reloadConfiguration());
@@ -128,3 +158,15 @@ function onRootShutdown(reason?: any) {
 
   process.exit(0);
 }
+
+function configFileExists(path: string) {
+  try {
+    return statSync(path).isFile();
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return false;
+    }
+
+    throw err;
+  }
+}

packages/core/root/core-root-server-internal/src/register_service_config.ts

@@ -28,6 +28,7 @@ import { uiSettingsConfig } from '@kbn/core-ui-settings-server-internal';
 
 import { config as pluginsConfig } from '@kbn/core-plugins-server-internal';
 import { elasticApmConfig } from './root/elastic_config';
+import { serverlessConfig } from './root/serverless_config';
 
 const rootConfigPath = '';
 
@@ -49,6 +50,7 @@ export function registerServiceConfig(configService: ConfigService) {
     pluginsConfig,
     savedObjectsConfig,
     savedObjectsMigrationConfig,
+    serverlessConfig,
     statusConfig,
     uiSettingsConfig,
   ];

packages/core/root/core-root-server-internal/src/root/serverless_config.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf, Type } from '@kbn/config-schema';
+import { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
+
+// Config validation for how to run Kibana in Serverless mode.
+// Clients need to specify the project type to run in.
+// Going for a simple `serverless` string because it serves as
+// a direct replacement to the legacy --serverless CLI flag.
+// If we even decide to extend this further, and converting it into an object,
+// BWC can be ensured by adding the object definition as another alternative to `schema.oneOf`.
+
+export const VALID_SERVERLESS_PROJECT_TYPES = ['es', 'oblt', 'security'];
+
+const serverlessConfigSchema = schema.maybe(
+  schema.oneOf(
+    VALID_SERVERLESS_PROJECT_TYPES.map((projectName) => schema.literal(projectName)) as [
+      Type<typeof VALID_SERVERLESS_PROJECT_TYPES[number]> // This cast is needed because it's different to Type<T>[] :sight:
+    ]
+  )
+);
+
+export type ServerlessConfigType = TypeOf<typeof serverlessConfigSchema>;
+
+export const serverlessConfig: ServiceConfigDescriptor<ServerlessConfigType> = {
+  path: 'serverless',
+  schema: serverlessConfigSchema,
+};

src/cli/serve/integration_tests/serverless_config_flag.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { spawn, spawnSync } from 'child_process';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { filter, firstValueFrom, from, take, concatMap } from 'rxjs';
+
+import { REPO_ROOT } from '@kbn/repo-info';
+import { getConfigDirectory } from '@kbn/utils';
+
+describe('cli serverless project type', () => {
+  it(
+    'exits with statusCode 1 and logs an error when serverless project type is invalid',
+    () => {
+      const { error, status, stdout } = spawnSync(
+        process.execPath,
+        ['scripts/kibana', '--serverless=non-existing-project-type'],
+        {
+          cwd: REPO_ROOT,
+        }
+      );
+      expect(error).toBe(undefined);
+
+      expect(stdout.toString('utf8')).toContain(
+        'FATAL CLI ERROR Error: invalid --serverless value, must be one of es, oblt, security'
+      );
+
+      expect(status).toBe(1);
+    },
+    20 * 1000
+  );
+
+  // Skipping this one because on CI it fails to read the config file
+  it.skip.each(['es', 'oblt', 'security'])(
+    'writes the serverless project type %s in config/serverless.recent.yml',
+    async (mode) => {
+      // Making sure `--serverless` translates into the `serverless` config entry, and validates against the accepted values
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait for 5 lines in the logs
+      await firstValueFrom(from(child.stdout).pipe(take(5)));
+
+      expect(
+        readFileSync(resolve(getConfigDirectory(), 'serverless.recent.yml'), 'utf-8')
+      ).toContain(`serverless: ${mode}\n`);
+
+      child.kill('SIGKILL');
+    }
+  );
+
+  it.each(['es', 'oblt', 'security'])(
+    'Kibana does not crash when running project type %s',
+    async (mode) => {
+      const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], {
+        cwd: REPO_ROOT,
+      });
+
+      // Wait until Kibana starts listening to the port
+      let leftover = '';
+      const found = await firstValueFrom(
+        from(child.stdout).pipe(
+          concatMap((chunk: Buffer) => {
+            const data = leftover + chunk.toString('utf-8');
+            const msgs = data.split('\n');
+            leftover = msgs.pop() ?? '';
+            return msgs;
+          }),
+          filter(
+            (msg) =>
+              msg.includes('http server running at http://localhost:5601') || msg.includes('FATAL')
+          )
+        )
+      );
+
+      child.kill('SIGKILL');
+
+      expect(found).not.toContain('FATAL');
+    }
+  );
+});

src/cli/serve/serve.js

@@ -8,7 +8,7 @@
 
 import { set as lodashSet } from '@kbn/safer-lodash-set';
 import _ from 'lodash';
-import { statSync } from 'fs';
+import { statSync, existsSync, readFileSync, writeFileSync } from 'fs';
 import { resolve } from 'path';
 import url from 'url';
 
@@ -22,14 +22,14 @@ const VALID_SERVERLESS_PROJECT_MODE = ['es', 'oblt', 'security'];
 
 /**
  * @param {Record<string, unknown>} opts
- * @returns {ServerlessProjectMode | null}
+ * @returns {ServerlessProjectMode | true | null}
  */
 function getServerlessProjectMode(opts) {
   if (!opts.serverless) {
     return null;
   }
 
-  if (VALID_SERVERLESS_PROJECT_MODE.includes(opts.serverless)) {
+  if (VALID_SERVERLESS_PROJECT_MODE.includes(opts.serverless) || opts.serverless === true) {
     return opts.serverless;
   }
 
@@ -94,16 +94,6 @@ function configFileExists(name) {
   }
 }
 
-/**
- * @returns {boolean} Whether the distribution can run in Serverless mode
- */
-function isServerlessCapableDistribution() {
-  // For now, checking if the `serverless.yml` config file exists should be enough
-  // We could also check the following as well, but I don't think it's necessary:
-  // VALID_SERVERLESS_PROJECT_MODE.some((projectType) => configFileExists(`serverless.${projectType}.yml`))
-  return configFileExists('serverless.yml');
-}
-
 /**
  * @param {string} name
  * @param {string[]} configs
@@ -115,6 +105,48 @@ function maybeAddConfig(name, configs, method) {
   }
 }
 
+/**
+ * @param {string} file
+ * @param {'es' | 'security' | 'oblt' | true} projectType
+ * @param {boolean} isDevMode
+ * @param {string[]} configs
+ * @param {'push' | 'unshift'} method
+ */
+function maybeSetRecentConfig(file, projectType, isDevMode, configs, method) {
+  const path = resolve(getConfigDirectory(), file);
+
+  function writeMode(selectedProjectType) {
+    writeFileSync(
+      path,
+      `${
+        isDevMode ? 'xpack.serverless.plugin.developer.projectSwitcher.enabled: true\n' : ''
+      }serverless: ${selectedProjectType}\n`
+    );
+  }
+
+  try {
+    if (projectType === true) {
+      if (!existsSync(path)) {
+        writeMode('es');
+      }
+    } else {
+      const data = readFileSync(path, 'utf-8');
+      const match = data.match(/serverless: (\w+)\n/);
+      if (!match || match[1] !== projectType) {
+        writeMode(projectType);
+      }
+    }
+
+    configs[method](path);
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return;
+    }
+
+    throw err;
+  }
+}
+
 /**
  * @returns {string[]}
  */
@@ -251,13 +283,14 @@ export default function (program) {
       .option(
         '--run-examples',
         'Adds plugin paths for all the Kibana example plugins and runs with no base path'
+      )
+      .option(
+        '--serverless [oblt|security|es]',
+        'Start Kibana in a specific serverless project mode. ' +
+          'If no mode is provided, it starts Kibana in the most recent serverless project mode (default is es)'
       );
   }
 
-  if (isServerlessCapableDistribution()) {
-    command.option('--serverless <oblt|security|es>', 'Start Kibana in a serverless project mode');
-  }
-
   if (DEV_MODE_SUPPORTED) {
     command
       .option('--dev', 'Run the server with development mode defaults')
@@ -282,10 +315,8 @@ export default function (program) {
     const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])];
     const serverlessMode = getServerlessProjectMode(opts);
 
-    // we "unshift" .serverless. config so that it only overrides defaults
     if (serverlessMode) {
-      maybeAddConfig(`serverless.yml`, configs, 'push');
-      maybeAddConfig(`serverless.${serverlessMode}.yml`, configs, 'unshift');
+      maybeSetRecentConfig('serverless.recent.yml', serverlessMode, opts.dev, configs, 'push');
     }
 
     // .dev. configs are "pushed" so that they override all other config files
@@ -293,7 +324,7 @@ export default function (program) {
       maybeAddConfig('kibana.dev.yml', configs, 'push');
       if (serverlessMode) {
         maybeAddConfig(`serverless.dev.yml`, configs, 'push');
-        maybeAddConfig(`serverless.${serverlessMode}.dev.yml`, configs, 'push');
+        maybeAddConfig('serverless.recent.dev.yml', configs, 'push');
       }
     }
 
@@ -315,7 +346,6 @@ export default function (program) {
       oss: !!opts.oss,
       cache: !!opts.cache,
       dist: !!opts.dist,
-      serverless: !!opts.serverless,
     };
 
     // In development mode, the main process uses the @kbn/dev-cli-mode