[kbn/es serverless] invert teardown logic to always kill cluster (#166546)

## Summary

Closes #166543 

- It is counterintuitive to `CTRL + C` from the `yarn es serverless`
process and the cluster is still running. This has caused issues when
switching between stateful and serverless work flows for some
developers. This PR inverts the logic to always teardown the cluster
unless a flag is passed.
- Small docs update for changing ES memory allocation on all operation
systems.
- Fixes bug were cluster status would continue looping after `SIGINT`.
- Bind to `SIGINT` earlier so nodes are always killed

packages/core/test-helpers/core-test-helpers-kbn-server/src/create_serverless_root.ts

@@ -81,7 +81,6 @@ function createServerlessES() {
       await es.runServerless({
         basePath: ES_BASE_PATH_DIR,
         port: esPort,
-        teardown: true,
         background: true,
         clean: true,
         kill: true,

packages/kbn-es/README.mdx

@@ -21,7 +21,7 @@ If running elasticsearch serverless or a docker container, there is some require
 
 1. Install Docker. Instructions can be found [here](https://www.docker.com/).
 1. Authentication with Elastic's Docker registry [here](https://docker-auth.elastic.co/github_auth).
-1. Increase OS virtual memory limits. More info in the [ES docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html).
+1. Increase OS virtual memory limits. More info in the [ES docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites).
 
 
 ### Examples

packages/kbn-es/src/cli_commands/serverless.ts

@@ -37,7 +37,7 @@ export const serverless: Command = {
       --kill              Kill running ESS nodes if detected on startup
       --port              The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}]
       --ssl               Enable HTTP SSL on Elasticsearch
-      --teardown          If this process exits, teardown the ES cluster as well
+      --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
       
       -E                  Additional key=value settings to pass to Elasticsearch
@@ -66,11 +66,21 @@ export const serverless: Command = {
       },
 
       string: ['tag', 'image', 'basePath'],
-      boolean: ['clean', 'ssl', 'kill', 'background', 'teardown', 'waitForReady'],
+      boolean: ['clean', 'ssl', 'kill', 'background', 'skipTeardown', 'waitForReady'],
 
       default: defaults,
     }) as unknown as ServerlessOptions;
 
+    /*
+     * The nodes will be killed immediately if background = true and skipTeardown = false
+     * because the CLI process exits after starting the nodes. We handle this here instead of
+     * in runServerless because in FTR we run the nodes in the background but the parent
+     * process continues for testing and we want to be able to SIGINT for teardown.
+     */
+    if (options.background && !options.skipTeardown) {
+      options.skipTeardown = true;
+    }
+
     const cluster = new Cluster();
     await cluster.runServerless({
       reportTime,

packages/kbn-es/src/cluster.ts

@@ -538,9 +538,7 @@ export class Cluster {
       throw new Error('ES serverless docker cluster has already been started');
     }
 
-    this.serverlessNodes = await runServerlessCluster(this.log, options);
-
-    if (options.teardown) {
+    if (!options.skipTeardown) {
       /**
        * Ideally would be async and an event like beforeExit or SIGINT,
        * but those events are not being triggered in FTR child process.
@@ -548,6 +546,8 @@ export class Cluster {
       process.on('exit', () => teardownServerlessClusterSync(this.log, options));
     }
 
+    this.serverlessNodes = await runServerlessCluster(this.log, options);
+
     return this.serverlessNodes;
   }
 

packages/kbn-es/src/utils/docker.ts

@@ -57,8 +57,8 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
   clean?: boolean;
   /** Path to the directory where the ES cluster will store data */
   basePath: string;
-  /** If this process exits, teardown the ES cluster as well */
-  teardown?: boolean;
+  /** If this process exits, leave the ES cluster running in the background */
+  skipTeardown?: boolean;
   /** Start the ES cluster in the background instead of remaining attached: useful for running tests */
   background?: boolean;
   /** Wait for the ES cluster to be ready to serve requests */
@@ -596,6 +596,11 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
     `);
   }
 
+  if (!options.skipTeardown) {
+    // SIGINT will not trigger in FTR (see cluster.runServerless for FTR signal)
+    process.on('SIGINT', () => teardownServerlessClusterSync(log, options));
+  }
+
   if (options.waitForReady) {
     log.info('Waiting until ES is ready to serve requests...');
 
@@ -625,24 +630,16 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
     await waitUntilClusterReady({ client, expectedStatus: 'green', log });
   }
 
-  if (options.teardown) {
-    // SIGINT will not trigger in FTR (see cluster.runServerless for FTR signal)
-    process.on('SIGINT', () => teardownServerlessClusterSync(log, options));
-  }
-
   if (!options.background) {
     // The ESS cluster has to be started detached, so we attach a logger afterwards for output
     await execa('docker', ['logs', '-f', SERVERLESS_NODES[0].name], {
       // inherit is required to show Docker output and Java console output for pw, enrollment token, etc
       stdio: ['ignore', 'inherit', 'inherit'],
-    }).catch((error) => {
+    }).catch(() => {
       /**
-       * 255 is a generic exit code which is triggered from docker logs command
-       * if we teardown the cluster since the entrypoint doesn't exit normally
+       * docker logs will throw errors when the nodes are killed through SIGINT
+       * and the entrypoint doesn't exit normally, so we silence the errors.
        */
-      if (error.exitCode !== 255) {
-        log.error(error.message);
-      }
     });
   }
 

packages/kbn-es/src/utils/wait_until_cluster_ready.ts

@@ -37,6 +37,9 @@ export async function waitUntilClusterReady({
   let attempt = 0;
   const start = Date.now();
 
+  // The loop will continue until timeout even if SIGINT is signaled, so force exit
+  process.on('SIGINT', () => process.exit());
+
   log.info(`waiting for ES cluster to report a ${expectedStatus} status`);
 
   const isReady = checkStatus(expectedStatus);

packages/kbn-test/src/es/test_es_cluster.ts

@@ -241,7 +241,6 @@ export function createTestEsCluster<
           esArgs: customEsArgs,
           port,
           clean: true,
-          teardown: true,
           background: true,
           files,
           ssl,