Extend yarn es: plugin support + adding secure files to es keystore (#126938)

Co-authored-by: Spencer <email@spalger.com>

packages/kbn-es/src/cli_commands/snapshot.js

@@ -33,6 +33,8 @@ exports.help = (defaults = {}) => {
       --use-cached      Skips cache verification and use cached ES snapshot.
       --skip-ready-check  Disable the ready check,
       --ready-timeout   Customize the ready check timeout, in seconds or "Xm" format, defaults to 1m
+      --plugins         Comma seperated list of Elasticsearch plugins to install
+      --secure-files     Comma seperated list of secure_setting_name=/path pairs
 
     Example:
 
@@ -58,6 +60,7 @@ exports.run = async (defaults = {}) => {
       useCached: 'use-cached',
       skipReadyCheck: 'skip-ready-check',
       readyTimeout: 'ready-timeout',
+      secureFiles: 'secure-files',
     },
 
     string: ['version', 'ready-timeout'],
@@ -76,6 +79,13 @@ exports.run = async (defaults = {}) => {
     if (options.dataArchive) {
       await cluster.extractDataDirectory(installPath, options.dataArchive);
     }
+    if (options.plugins) {
+      await cluster.installPlugins(installPath, options.plugins, options);
+    }
+    if (options.secureFiles) {
+      const pairs = options.secureFiles.split(',').map((kv) => kv.split('=').map((v) => v.trim()));
+      await cluster.configureKeystoreWithSecureSettingsFiles(installPath, pairs);
+    }
 
     reportTime(installStartTime, 'installed', {
       success: true,

packages/kbn-es/src/cli_commands/source.js

@@ -27,6 +27,8 @@ exports.help = (defaults = {}) => {
       --password        Sets password for elastic user [default: ${password}]
       --password.[user] Sets password for native realm user [default: ${password}]
       --ssl             Sets up SSL on Elasticsearch
+      --plugins         Comma seperated list of Elasticsearch plugins to install
+      --secure-files     Comma seperated list of secure_setting_name=/path pairs
       -E                Additional key=value settings to pass to Elasticsearch
       --skip-ready-check  Disable the ready check,
       --ready-timeout   Customize the ready check timeout, in seconds or "Xm" format, defaults to 1m
@@ -47,6 +49,7 @@ exports.run = async (defaults = {}) => {
       dataArchive: 'data-archive',
       skipReadyCheck: 'skip-ready-check',
       readyTimeout: 'ready-timeout',
+      secureFiles: 'secure-files',
       esArgs: 'E',
     },
 
@@ -62,6 +65,13 @@ exports.run = async (defaults = {}) => {
   if (options.dataArchive) {
     await cluster.extractDataDirectory(installPath, options.dataArchive);
   }
+  if (options.plugins) {
+    await cluster.installPlugins(installPath, options.plugins, options);
+  }
+  if (options.secureFiles) {
+    const pairs = options.secureFiles.split(',').map((kv) => kv.split('=').map((v) => v.trim()));
+    await cluster.configureKeystoreWithSecureSettingsFiles(installPath, pairs);
+  }
 
   await cluster.run(installPath, {
     ...options,

packages/kbn-es/src/cluster.js

@@ -12,7 +12,7 @@ const chalk = require('chalk');
 const path = require('path');
 const { Client } = require('@elastic/elasticsearch');
 const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install');
-const { ES_BIN } = require('./paths');
+const { ES_BIN, ES_PLUGIN_BIN, ES_KEYSTORE_BIN } = require('./paths');
 const {
   log: defaultLog,
   parseEsLog,
@@ -150,6 +150,42 @@ exports.Cluster = class Cluster {
     });
   }
 
+  /**
+   * Starts ES and returns resolved promise once started
+   *
+   * @param {String} installPath
+   * @param {String} plugins - comma separated list of plugins to install
+   * @param {Object} options
+   * @returns {Promise}
+   */
+  async installPlugins(installPath, plugins, options) {
+    const esJavaOpts = this.javaOptions(options);
+    for (const plugin of plugins.split(',')) {
+      await execa(ES_PLUGIN_BIN, ['install', plugin.trim()], {
+        cwd: installPath,
+        env: {
+          JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
+          ES_JAVA_OPTS: esJavaOpts.trim(),
+        },
+      });
+    }
+  }
+
+  async configureKeystoreWithSecureSettingsFiles(installPath, secureSettingsFiles) {
+    const env = { JAVA_HOME: '' };
+    for (const [secureSettingName, secureSettingFile] of secureSettingsFiles) {
+      this._log.info(
+        `setting secure setting %s to %s`,
+        chalk.bold(secureSettingName),
+        chalk.bold(secureSettingFile)
+      );
+      await execa(ES_KEYSTORE_BIN, ['add-file', secureSettingName, secureSettingFile], {
+        cwd: installPath,
+        env,
+      });
+    }
+  }
+
   /**
    * Starts ES and returns resolved promise once started
    *
@@ -280,19 +316,9 @@ exports.Cluster = class Cluster {
     );
 
     this._log.info('%s %s', ES_BIN, args.join(' '));
+    const esJavaOpts = this.javaOptions(options);
 
-    let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`;
-
-    // ES now automatically sets heap size to 50% of the machine's available memory
-    // so we need to set it to a smaller size for local dev and CI
-    // especially because we currently run many instances of ES on the same machine during CI
-    // inital and max must be the same, so we only need to check the max
-    if (!esJavaOpts.includes('Xmx')) {
-      // 1536m === 1.5g
-      esJavaOpts += ' -Xms1536m -Xmx1536m';
-    }
-
-    this._log.info('ES_JAVA_OPTS: %s', esJavaOpts.trim());
+    this._log.info('ES_JAVA_OPTS: %s', esJavaOpts);
 
     this._process = execa(ES_BIN, args, {
       cwd: installPath,
@@ -300,7 +326,7 @@ exports.Cluster = class Cluster {
         ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}),
         ...process.env,
         JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
-        ES_JAVA_OPTS: esJavaOpts.trim(),
+        ES_JAVA_OPTS: esJavaOpts,
       },
       stdio: ['ignore', 'pipe', 'pipe'],
     });
@@ -429,4 +455,18 @@ exports.Cluster = class Cluster {
       }
     }
   }
+
+  javaOptions(options) {
+    let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`;
+
+    // ES now automatically sets heap size to 50% of the machine's available memory
+    // so we need to set it to a smaller size for local dev and CI
+    // especially because we currently run many instances of ES on the same machine during CI
+    // inital and max must be the same, so we only need to check the max
+    if (!esJavaOpts.includes('Xmx')) {
+      // 1536m === 1.5g
+      esJavaOpts += ' -Xms1536m -Xmx1536m';
+    }
+    return esJavaOpts.trim();
+  }
 };

packages/kbn-es/src/paths.ts

@@ -19,6 +19,7 @@ export const BASE_PATH = Path.resolve(tempDir, 'kbn-es');
 
 export const GRADLE_BIN = maybeUseBat('./gradlew');
 export const ES_BIN = maybeUseBat('bin/elasticsearch');
+export const ES_PLUGIN_BIN = maybeUseBat('bin/elasticsearch-plugin');
 export const ES_CONFIG = 'config/elasticsearch.yml';
 
 export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore');