[kbn-es] Allow overriding of Java heap (#99517)

* [kbn-es] Allow overriding of Java heap

Fixes #99494

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

packages/kbn-es/README.md

@@ -7,6 +7,8 @@ If running elasticsearch from source, elasticsearch needs to be cloned to a sibl
 
 To run, go to the Kibana root and run `node scripts/es --help` to get the latest command line options.
 
+The script attempts to preserve the existing interfaces used by Elasticsearch CLI. This includes passing through options with the `-E` argument and the `ES_JAVA_OPTS` environment variable for Java options.
+
 ### Examples
 
 Run a snapshot install with a trial license

packages/kbn-es/src/cluster.js

@@ -236,6 +236,7 @@ exports.Cluster = class Cluster {
    * @param {String} installPath
    * @param {Object} options
    * @property {string|Array} options.esArgs
+   * @property {string} options.esJavaOpts
    * @return {undefined}
    */
   _exec(installPath, options = {}) {
@@ -268,14 +269,17 @@ exports.Cluster = class Cluster {
 
     this._log.debug('%s %s', ES_BIN, args.join(' '));
 
-    options.esEnvVars = options.esEnvVars || {};
+    let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`;
 
     // ES now automatically sets heap size to 50% of the machine's available memory
     // so we need to set it to a smaller size for local dev and CI
     // especially because we currently run many instances of ES on the same machine during CI
-    options.esEnvVars.ES_JAVA_OPTS =
-      (options.esEnvVars.ES_JAVA_OPTS ? `${options.esEnvVars.ES_JAVA_OPTS} ` : '') +
-      '-Xms1g -Xmx1g';
+    // inital and max must be the same, so we only need to check the max
+    if (!esJavaOpts.includes('Xmx')) {
+      esJavaOpts += ' -Xms1g -Xmx1g';
+    }
+
+    this._log.debug('ES_JAVA_OPTS: %s', esJavaOpts.trim());
 
     this._process = execa(ES_BIN, args, {
       cwd: installPath,
@@ -283,7 +287,7 @@ exports.Cluster = class Cluster {
         ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}),
         ...process.env,
         JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
-        ...(options.esEnvVars || {}),
+        ES_JAVA_OPTS: esJavaOpts.trim(),
       },
       stdio: ['ignore', 'pipe', 'pipe'],
     });

packages/kbn-es/src/integration_tests/cluster.test.js

@@ -71,11 +71,17 @@ function mockEsBin({ exitCode, start }) {
   );
 }
 
+const initialEnv = { ...process.env };
+
 beforeEach(() => {
   jest.resetAllMocks();
   extractConfigFiles.mockImplementation((config) => config);
 });
 
+afterEach(() => {
+  process.env = { ...initialEnv };
+});
+
 describe('#installSource()', () => {
   it('awaits installSource() promise and returns { installPath }', async () => {
     let resolveInstallSource;
@@ -355,6 +361,25 @@ describe('#run()', () => {
       ]
     `);
   });
+
+  it('sets default Java heap', async () => {
+    mockEsBin({ start: true });
+
+    const cluster = new Cluster({ log });
+    await cluster.run();
+
+    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms1g -Xmx1g');
+  });
+
+  it('allows Java heap to be overwritten', async () => {
+    mockEsBin({ start: true });
+    process.env.ES_JAVA_OPTS = '-Xms5g -Xmx5g';
+
+    const cluster = new Cluster({ log });
+    await cluster.run();
+
+    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms5g -Xmx5g');
+  });
 });
 
 describe('#stop()', () => {

packages/kbn-test/src/es/test_es_cluster.ts

@@ -36,7 +36,7 @@ interface TestClusterFactoryOptions {
    * */
   dataArchive?: string;
   esArgs?: string[];
-  esEnvVars?: Record<string, any>;
+  esJavaOpts?: string;
   clusterName?: string;
   log: ToolingLog;
   ssl?: boolean;
@@ -52,7 +52,7 @@ export function createTestEsCluster(options: TestClusterFactoryOptions) {
     esFrom = esTestConfig.getBuildFrom(),
     dataArchive,
     esArgs: customEsArgs = [],
-    esEnvVars,
+    esJavaOpts,
     clusterName: customClusterName = 'es-test-cluster',
     ssl,
   } = options;
@@ -107,7 +107,7 @@ export function createTestEsCluster(options: TestClusterFactoryOptions) {
       await cluster.start(installPath, {
         password: config.password,
         esArgs,
-        esEnvVars,
+        esJavaOpts,
       });
     }
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -172,7 +172,7 @@ export const schema = Joi.object()
         license: Joi.string().default('basic'),
         from: Joi.string().default('snapshot'),
         serverArgs: Joi.array(),
-        serverEnvVars: Joi.object(),
+        esJavaOpts: Joi.string(),
         dataArchive: Joi.string(),
         ssl: Joi.boolean().default(false),
       })

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts

@@ -29,7 +29,7 @@ export async function runElasticsearch({
   const ssl = config.get('esTestCluster.ssl');
   const license = config.get('esTestCluster.license');
   const esArgs = config.get('esTestCluster.serverArgs');
-  const esEnvVars = config.get('esTestCluster.serverEnvVars');
+  const esJavaOpts = config.get('esTestCluster.esJavaOpts');
   const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true');
 
   const cluster = createTestEsCluster({
@@ -43,7 +43,7 @@ export async function runElasticsearch({
     esFrom: esFrom || config.get('esTestCluster.from'),
     dataArchive: config.get('esTestCluster.dataArchive'),
     esArgs,
-    esEnvVars,
+    esJavaOpts,
     ssl,
   });
 

x-pack/test/security_api_integration/kerberos.config.ts

@@ -32,11 +32,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         'xpack.security.authc.realms.kerberos.kerb1.order=0',
         `xpack.security.authc.realms.kerberos.kerb1.keytab.path=${kerberosKeytabPath}`,
       ],
-      serverEnvVars: {
-        // We're going to use the same TGT multiple times and during a short period of time, so we
-        // have to disable replay cache so that ES doesn't complain about that.
-        ES_JAVA_OPTS: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`,
-      },
+
+      // We're going to use the same TGT multiple times and during a short period of time, so we
+      // have to disable replay cache so that ES doesn't complain about that.
+      esJavaOpts: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`,
     },
 
     kbnTestServer: {

x-pack/test/security_api_integration/login_selector.config.ts

@@ -96,11 +96,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         `xpack.security.authc.realms.saml.saml2.sp.acs=http://localhost:${kibanaPort}/api/security/saml/callback`,
         'xpack.security.authc.realms.saml.saml2.attributes.principal=urn:oid:0.0.7',
       ],
-      serverEnvVars: {
-        // We're going to use the same TGT multiple times and during a short period of time, so we
-        // have to disable replay cache so that ES doesn't complain about that.
-        ES_JAVA_OPTS: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`,
-      },
+
+      // We're going to use the same TGT multiple times and during a short period of time, so we
+      // have to disable replay cache so that ES doesn't complain about that.
+      esJavaOpts: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`,
     },
 
     kbnTestServer: {