[Security Solution][Endpoint] Add Serverless support to data loading utilities (#166402)

## Summary

PR enables the existing data loading utilities/services, used in e2e
testing and CLI tools, to support being run against a serverless Env..
Changes include:

- `createRuntimeServices()` and the associated methods that create the
ES and KBN clients, will now by default add a CA cert to the ES and KBN
clients if the URL protocol is `https`
- an option was also added to the mothods that allows a developer to
turn this behaviour off if necessary (`noCertForSsl`)
- `createRuntimeServices()` option `asSuperuser` will NOT attempt to
create a new user in ES if it detects its running against serverless. It
will instead set the `username` to `system_indices_superuser`
- `resolver_generator.js` script was updated so that it can be run
against a serverless env. (note: tested only in local dev, not agains
cloud environments)
- new utility to determine if Kibana is running in serverless mode
(`isServerlessKibanaFlavor()`)
- Cypress tests that don't require specific user/role were updated to
use `system_indices_superuser` as the default username (instead of
`elastic`)

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_rule_alerts.ts

@@ -5488,10 +5488,6 @@ const getAlertsIndexMappings = (): IndexMappings => {
         index: {
           auto_expand_replicas: '0-1',
           hidden: 'true',
-          lifecycle: {
-            name: '.alerts-ilm-policy',
-            rollover_alias: '.alerts-security.alerts-default',
-          },
           mapping: {
             total_fields: {
               limit: 1900,

x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts

@@ -21,7 +21,12 @@ import {
 } from '@kbn/fleet-plugin/common';
 import { ToolingLog } from '@kbn/tooling-log';
 import { UsageTracker } from './usage_tracker';
-import { EndpointDataLoadingError, retryOnError, wrapErrorAndRejectPromise } from './utils';
+import {
+  EndpointDataLoadingError,
+  RETRYABLE_TRANSIENT_ERRORS,
+  retryOnError,
+  wrapErrorAndRejectPromise,
+} from './utils';
 
 const usageTracker = new UsageTracker({ dumpOnProcessExit: true });
 
@@ -165,13 +170,7 @@ export const installOrUpgradeEndpointFleetPackage = async (
     return bulkResp[0] as BulkInstallPackageInfo;
   };
 
-  return retryOnError(
-    updatePackages,
-    ['no_shard_available_action_exception', 'illegal_index_shard_state_exception'],
-    logger,
-    5,
-    10000
-  )
+  return retryOnError(updatePackages, RETRYABLE_TRANSIENT_ERRORS, logger, 5, 10000)
     .then((result) => {
       usageRecord.set('success');
 

x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts

@@ -8,6 +8,11 @@
 import { mergeWith } from 'lodash';
 import { ToolingLog } from '@kbn/tooling-log';
 
+export const RETRYABLE_TRANSIENT_ERRORS: Readonly<Array<string | RegExp>> = [
+  'no_shard_available_action_exception',
+  'illegal_index_shard_state_exception',
+];
+
 export class EndpointDataLoadingError extends Error {
   constructor(message: string, public meta?: unknown) {
     super(message);
@@ -43,7 +48,7 @@ export const mergeAndAppendArrays = <T, S>(destinationObj: T, srcObj: S): T => {
  */
 export const retryOnError = async <T>(
   callback: () => Promise<T>,
-  errors: Array<string | RegExp>,
+  errors: Array<string | RegExp> | Readonly<Array<string | RegExp>>,
   logger?: ToolingLog,
   tryCount: number = 5,
   interval: number = 10000
@@ -60,6 +65,8 @@ export const retryOnError = async <T>(
     });
   };
 
+  log.indent(4);
+
   let attempt = 1;
   let responsePromise: Promise<T>;
 
@@ -71,13 +78,20 @@ export const retryOnError = async <T>(
 
     try {
       responsePromise = callback(); // store promise so that if it fails and no more attempts, we return the last failure
-      return await responsePromise;
+      const result = await responsePromise;
+
+      log.info(msg(`attempt ${thisAttempt} was successful. Exiting retry`));
+      log.indent(-4);
+
+      return result;
     } catch (err) {
       log.info(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err);
 
       // If not an error that is retryable, then end loop here and return that error;
       if (!isRetryableError(err)) {
         log.error(err);
+        log.error(msg('non-retryable error encountered'));
+        log.indent(-4);
         return Promise.reject(err);
       }
     }
@@ -85,6 +99,10 @@ export const retryOnError = async <T>(
     await new Promise((resolve) => setTimeout(resolve, interval));
   }
 
+  log.error(msg(`max retry attempts reached. returning last failure`));
+  log.indent(-4);
+
+  // Last resort: return the last rejected Promise.
   // @ts-expect-error TS2454: Variable 'responsePromise' is used before being assigned.
   return responsePromise;
 };

x-pack/plugins/security_solution/public/management/cypress.config.ts

@@ -37,7 +37,7 @@ export default defineCypressConfig({
     ELASTICSEARCH_URL: 'http://localhost:9200',
     FLEET_SERVER_URL: 'https://localhost:8220',
     // Username/password used for both elastic and kibana
-    KIBANA_USERNAME: 'elastic',
+    KIBANA_USERNAME: 'system_indices_superuser',
     KIBANA_PASSWORD: 'changeme',
     ELASTICSEARCH_USERNAME: 'system_indices_superuser',
     ELASTICSEARCH_PASSWORD: 'changeme',

x-pack/plugins/security_solution/public/management/cypress_endpoint.config.ts

@@ -39,6 +39,10 @@ export default defineCypressConfig({
     'cypress-react-selector': {
       root: '#security-solution-app',
     },
+    KIBANA_USERNAME: 'system_indices_superuser',
+    KIBANA_PASSWORD: 'changeme',
+    ELASTICSEARCH_USERNAME: 'system_indices_superuser',
+    ELASTICSEARCH_PASSWORD: 'changeme',
   },
 
   e2e: {

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -37,6 +37,10 @@ import type {
 import nodeFetch from 'node-fetch';
 import semver from 'semver';
 import axios from 'axios';
+import {
+  RETRYABLE_TRANSIENT_ERRORS,
+  retryOnError,
+} from '../../../common/endpoint/data_loaders/utils';
 import { fetchKibanaStatus } from './stack_services';
 import { catchAxiosErrorFormatAndThrow } from './format_axios_error';
 import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator';
@@ -137,11 +141,15 @@ export const waitForHostToEnroll = async (
   let found: Agent | undefined;
 
   while (!found && !hasTimedOut()) {
-    found = await fetchFleetAgents(kbnClient, {
-      perPage: 1,
-      kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`,
-      showInactive: false,
-    }).then((response) => response.items[0]);
+    found = await retryOnError(
+      async () =>
+        fetchFleetAgents(kbnClient, {
+          perPage: 1,
+          kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`,
+          showInactive: false,
+        }).then((response) => response.items[0]),
+      RETRYABLE_TRANSIENT_ERRORS
+    );
 
     if (!found) {
       // sleep and check again

x-pack/plugins/security_solution/scripts/endpoint/common/format_axios_error.ts

@@ -22,7 +22,11 @@ export class FormattedAxiosError extends Error {
   };
 
   constructor(axiosError: AxiosError) {
-    super(axiosError.message);
+    super(
+      `${axiosError.message}${
+        axiosError?.response?.data ? `: ${JSON.stringify(axiosError?.response?.data)}` : ''
+      }`
+    );
 
     this.request = {
       method: axiosError.config?.method ?? '?',

x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts

@@ -15,11 +15,15 @@ import nodeFetch from 'node-fetch';
 import type { ReqOptions } from '@kbn/test/src/kbn_client/kbn_client_requester';
 import { type AxiosResponse } from 'axios';
 import type { ClientOptions } from '@elastic/elasticsearch/lib/client';
+import fs from 'fs';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { catchAxiosErrorFormatAndThrow } from './format_axios_error';
 import { isLocalhost } from './is_localhost';
 import { getLocalhostRealIp } from './localhost_services';
 import { createSecuritySuperuser } from './security_user_services';
 
+const CA_CERTIFICATE: Buffer = fs.readFileSync(CA_CERT_PATH);
+
 export interface RuntimeServices {
   kbnClient: KbnClient;
   esClient: Client;
@@ -64,6 +68,8 @@ interface CreateRuntimeServicesOptions {
   esPassword?: string;
   log?: ToolingLog;
   asSuperuser?: boolean;
+  /** If true, then a certificate will not be used when creating the Kbn/Es clients when url is `https` */
+  noCertForSsl?: boolean;
 }
 
 class KbnClientExtended extends KbnClient {
@@ -105,26 +111,39 @@ export const createRuntimeServices = async ({
   esPassword,
   log = new ToolingLog({ level: 'info', writeTo: process.stdout }),
   asSuperuser = false,
+  noCertForSsl,
 }: CreateRuntimeServicesOptions): Promise<RuntimeServices> => {
   let username = _username;
   let password = _password;
 
   if (asSuperuser) {
     await waitForKibana(kibanaUrl);
+    const tmpEsClient = createEsClient({
+      url: elasticsearchUrl,
+      username,
+      password,
+      log,
+      noCertForSsl,
+    });
 
-    const superuserResponse = await createSecuritySuperuser(
-      createEsClient({
-        url: elasticsearchUrl,
-        username,
-        password,
-        log,
-      })
-    );
+    const isServerlessEs = (await tmpEsClient.info()).version.build_flavor === 'serverless';
 
-    ({ username, password } = superuserResponse);
+    if (isServerlessEs) {
+      log?.warning(
+        'Creating Security Superuser is not supported in current environment. ES is running in serverless mode. ' +
+          'Will use username [system_indices_superuser] instead.'
+      );
 
-    if (superuserResponse.created) {
-      log.info(`Kibana user [${username}] was crated with password [${password}]`);
+      username = 'system_indices_superuser';
+      password = 'changeme';
+    } else {
+      const superuserResponse = await createSecuritySuperuser(tmpEsClient);
+
+      ({ username, password } = superuserResponse);
+
+      if (superuserResponse.created) {
+        log.info(`Kibana user [${username}] was crated with password [${password}]`);
+      }
     }
   }
 
@@ -133,16 +152,17 @@ export const createRuntimeServices = async ({
   const fleetURL = new URL(fleetServerUrl);
 
   return {
-    kbnClient: createKbnClient({ log, url: kibanaUrl, username, password, apiKey }),
+    kbnClient: createKbnClient({ log, url: kibanaUrl, username, password, apiKey, noCertForSsl }),
     esClient: createEsClient({
       log,
       url: elasticsearchUrl,
       username: esUsername ?? username,
       password: esPassword ?? password,
       apiKey,
+      noCertForSsl,
     }),
     log,
-    localhostRealIp: await getLocalhostRealIp(),
+    localhostRealIp: getLocalhostRealIp(),
     apiKey: apiKey ?? '',
     user: {
       username,
@@ -188,6 +208,7 @@ export const createEsClient = ({
   password,
   apiKey,
   log,
+  noCertForSsl,
 }: {
   url: string;
   username: string;
@@ -195,11 +216,19 @@ export const createEsClient = ({
   /** If defined, both `username` and `password` will be ignored */
   apiKey?: string;
   log?: ToolingLog;
+  noCertForSsl?: boolean;
 }): Client => {
+  const isHttps = new URL(url).protocol.startsWith('https');
   const clientOptions: ClientOptions = {
     node: buildUrlWithCredentials(url, apiKey ? '' : username, apiKey ? '' : password),
   };
 
+  if (isHttps && !noCertForSsl) {
+    clientOptions.tls = {
+      ca: [CA_CERTIFICATE],
+    };
+  }
+
   if (apiKey) {
     clientOptions.auth = { apiKey };
   }
@@ -217,6 +246,7 @@ export const createKbnClient = ({
   password,
   apiKey,
   log = new ToolingLog(),
+  noCertForSsl,
 }: {
   url: string;
   username: string;
@@ -224,16 +254,28 @@ export const createKbnClient = ({
   /** If defined, both `username` and `password` will be ignored */
   apiKey?: string;
   log?: ToolingLog;
+  noCertForSsl?: boolean;
 }): KbnClient => {
-  const kbnUrl = buildUrlWithCredentials(url, username, password);
+  const isHttps = new URL(url).protocol.startsWith('https');
+  const clientOptions: ConstructorParameters<typeof KbnClientExtended>[0] = {
+    log,
+    apiKey,
+    url: buildUrlWithCredentials(url, username, password),
+  };
+
+  if (isHttps && !noCertForSsl) {
+    clientOptions.certificateAuthorities = [CA_CERTIFICATE];
+  }
 
   if (log) {
     log.verbose(
-      `Creating Kibana client with URL: ${kbnUrl} ${apiKey ? ` + ApiKey: ${apiKey}` : ''}`
+      `Creating Kibana client with URL: ${clientOptions.url} ${
+        apiKey ? ` + ApiKey: ${apiKey}` : ''
+      }`
     );
   }
 
-  return new KbnClientExtended({ log, url: kbnUrl, apiKey });
+  return new KbnClientExtended(clientOptions);
 };
 
 /**
@@ -287,3 +329,18 @@ export const waitForKibana = async (kbnUrl: string): Promise<void> => {
     { maxTimeout: 10000 }
   );
 };
+
+export const isServerlessKibanaFlavor = async (kbnClient: KbnClient): Promise<boolean> => {
+  const kbnStatus = await fetchKibanaStatus(kbnClient);
+
+  // If we don't have status for plugins, then error
+  // the Status API will always return something (its an open API), but if auth was successful,
+  // it will also return more data.
+  if (!kbnStatus.status.plugins) {
+    throw new Error(
+      `Unable to retrieve Kibana plugins status (likely an auth issue with the username being used for kibana)`
+    );
+  }
+
+  return kbnStatus.status.plugins?.serverless?.level === 'available';
+};

x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts

@@ -19,7 +19,7 @@ import { METADATA_DATASTREAM } from '../../common/endpoint/constants';
 import { EndpointMetadataGenerator } from '../../common/endpoint/data_generators/endpoint_metadata_generator';
 import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
 import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';
-import { fetchStackVersion } from './common/stack_services';
+import { fetchStackVersion, isServerlessKibanaFlavor } from './common/stack_services';
 import { ENDPOINT_ALERTS_INDEX, ENDPOINT_EVENTS_INDEX } from './common/constants';
 import { getWithResponseActionsRole } from './common/roles_users/with_response_actions_role';
 import { getNoResponseActionsRole } from './common/roles_users/without_response_actions_role';
@@ -161,6 +161,8 @@ function updateURL({
 }
 
 async function main() {
+  const startTime = new Date().getTime();
+
   const argv = yargs.help().options({
     seed: {
       alias: 's',
@@ -318,17 +320,16 @@ async function main() {
       default: false,
     },
   }).argv;
-  let ca: Buffer;
 
+  let ca: Buffer;
   let clientOptions: ClientOptions;
   let url: string;
   let node: string;
-  const toolingLogOptions = {
-    log: new ToolingLog({
-      level: 'info',
-      writeTo: process.stdout,
-    }),
-  };
+  const logger = new ToolingLog({
+    level: 'info',
+    writeTo: process.stdout,
+  });
+  const toolingLogOptions = { log: logger };
 
   let kbnClientOptions: KbnClientOptions = {
     ...toolingLogOptions,
@@ -350,38 +351,62 @@ async function main() {
     clientOptions = { node: argv.node };
   }
   let client = new Client(clientOptions);
+  let kbnClient = new KbnClient({ ...kbnClientOptions });
   let user: UserInfo | undefined;
+  const isServerless = await isServerlessKibanaFlavor(kbnClient);
+
+  logger.info(`Build flavor: ${isServerless ? 'serverless' : 'non-serverless'}`);
+
+  if (argv.fleet && !argv.withNewUser && !isServerless) {
+    // warn and exit when using fleet flag
+    logger.error(
+      'Please use the --withNewUser=username:password flag to add a custom user with required roles when --fleet is enabled!'
+    );
+    // eslint-disable-next-line no-process-exit
+    process.exit(0);
+  }
+
   // if fleet flag is used
   if (argv.fleet) {
-    // add endpoint user if --withNewUser flag has values as username:password
-    const newUserCreds =
-      argv.withNewUser.indexOf(':') !== -1 ? argv.withNewUser.split(':') : undefined;
-    user = await addUser(
-      client,
-      newUserCreds
-        ? {
-            username: newUserCreds[0],
-            password: newUserCreds[1],
-          }
-        : undefined
-    );
+    if (!isServerless) {
+      // add endpoint user if --withNewUser flag has values as username:password
+      const newUserCreds =
+        argv.withNewUser.indexOf(':') !== -1 ? argv.withNewUser.split(':') : undefined;
+      user = await addUser(
+        client,
+        newUserCreds
+          ? {
+              username: newUserCreds[0],
+              password: newUserCreds[1],
+            }
+          : undefined
+      );
 
-    // update client and kibana options before instantiating
-    if (user) {
-      // use endpoint user for Es and Kibana URLs
+      // update client and kibana options before instantiating
+      if (user) {
+        // use endpoint user for Es and Kibana URLs
 
-      url = updateURL({ url: argv.kibana, user });
-      node = updateURL({ url: argv.node, user });
+        url = updateURL({ url: argv.kibana, user });
+        node = updateURL({ url: argv.node, user });
 
-      kbnClientOptions = {
-        ...kbnClientOptions,
-        url,
-      };
-      client = new Client({ ...clientOptions, node });
+        kbnClientOptions = {
+          ...kbnClientOptions,
+          url,
+        };
+
+        client = new Client({ ...clientOptions, node });
+        kbnClient = new KbnClient({ ...kbnClientOptions });
+
+        logger.verbose(`ES/KBN clients updated to login using: ${JSON.stringify(user)}`);
+      }
+    } else {
+      logger.warning(
+        'Option `--withNewUser` not supported in serverless.\n' +
+          'Ensure that `--kibana` and `--node` options are defined with username/password of ' +
+          '`system_indices_superuser:changeme`'
+      );
     }
   }
-  // instantiate kibana client
-  const kbnClient = new KbnClient({ ...kbnClientOptions });
 
   if (argv.delete) {
     await deleteIndices(
@@ -391,6 +416,12 @@ async function main() {
   }
 
   if (argv.rbacUser) {
+    if (isServerless) {
+      // FIXME:PT create users in serverless when that capability is available
+
+      throw new Error(`Can not use '--rbacUser' option against serverless deployment`);
+    }
+
     // Add roles and users with response actions kibana privileges
     for (const role of Object.keys(rolesMapping)) {
       const addedRole = await addRole(kbnClient, {
@@ -398,32 +429,15 @@ async function main() {
         ...rolesMapping[role],
       });
       if (addedRole) {
-        console.log(`Successfully added ${role} role`);
+        logger.info(`Successfully added ${role} role`);
         await addUser(client, { username: role, password: 'changeme', roles: [role] });
       } else {
-        console.log(`Failed to add role, ${role}`);
+        logger.warning(`Failed to add role, ${role}`);
       }
     }
   }
 
-  let seed = argv.seed;
-
-  if (!seed) {
-    seed = Math.random().toString();
-    console.log(`No seed supplied, using random seed: ${seed}`);
-  }
-
-  const startTime = new Date().getTime();
-
-  if (argv.fleet && !argv.withNewUser) {
-    // warn and exit when using fleet flag
-    console.log(
-      'Please use the --withNewUser=username:password flag to add a custom user with required roles when --fleet is enabled!'
-    );
-    // eslint-disable-next-line no-process-exit
-    process.exit(0);
-  }
-
+  const seed = argv.seed || Math.random().toString();
   let DocGenerator: typeof EndpointDocGenerator = EndpointDocGenerator;
 
   // If `--randomVersions` is NOT set, then use custom generator that ensures all data generated
@@ -446,6 +460,7 @@ async function main() {
     };
   }
 
+  logger.info('Indexing host and alerts...');
   await indexHostsAndAlerts(
     client,
     kbnClient,
@@ -475,11 +490,12 @@ async function main() {
   );
 
   // delete endpoint_user after
-  if (user) {
+  if (user && !isServerless) {
     const deleted = await deleteUser(client, user.username);
     if (deleted.found) {
-      console.log(`User ${user.username} deleted successfully!`);
+      logger.info(`User ${user.username} deleted successfully!`);
     }
   }
-  console.log(`Creating and indexing documents took: ${new Date().getTime() - startTime}ms`);
+
+  logger.info(`Creating and indexing documents took: ${new Date().getTime() - startTime}ms`);
 }

x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts

@@ -20,6 +20,12 @@ export default defineCypressConfig({
   viewportHeight: 946,
   viewportWidth: 1680,
   numTestsKeptInMemory: 10,
+  env: {
+    KIBANA_USERNAME: 'system_indices_superuser',
+    KIBANA_PASSWORD: 'changeme',
+    ELASTICSEARCH_USERNAME: 'system_indices_superuser',
+    ELASTICSEARCH_PASSWORD: 'changeme',
+  },
   e2e: {
     experimentalRunAllSpecs: true,
     experimentalMemoryManagement: true,