[@kbn/es] Add a predefined list of Elasticsearch secure settings to be added into keystore. Re-enable OIDC tests. (#42239)

packages/kbn-es/src/cluster.js

@@ -32,6 +32,7 @@ const {
 const { createCliError } = require('./errors');
 const { promisify } = require('util');
 const treeKillAsync = promisify(require('tree-kill'));
+const { parseSettings, SettingsFilter } = require('./settings');
 
 // listen to data on stream until map returns anything but undefined
 const first = (stream, map) =>
@@ -250,9 +251,13 @@ exports.Cluster = class Cluster {
     this._log.info(chalk.bold('Starting'));
     this._log.indent(4);
 
-    const args = extractConfigFiles(options.esArgs || [], installPath, {
-      log: this._log,
-    }).reduce((acc, cur) => acc.concat(['-E', cur]), []);
+    const args = parseSettings(
+      extractConfigFiles(options.esArgs || [], installPath, { log: this._log }),
+      { filter: SettingsFilter.NonSecureOnly }
+    ).reduce(
+      (acc, [settingName, settingValue]) => acc.concat(['-E', `${settingName}=${settingValue}`]),
+      []
+    );
 
     this._log.debug('%s %s', ES_BIN, args.join(' '));
 

packages/kbn-es/src/install/archive.js

@@ -26,6 +26,7 @@ const url = require('url');
 const { log: defaultLog, decompress } = require('../utils');
 const { BASE_PATH, ES_CONFIG, ES_KEYSTORE_BIN } = require('../paths');
 const { Artifact } = require('../artifact');
+const { parseSettings, SettingsFilter } = require('../settings');
 
 /**
  * Extracts an ES archive and optionally installs plugins
@@ -45,6 +46,7 @@ exports.installArchive = async function installArchive(archive, options = {}) {
     installPath = path.resolve(basePath, path.basename(archive, '.tar.gz')),
     log = defaultLog,
     bundledJDK = false,
+    esArgs = [],
   } = options;
 
   let dest = archive;
@@ -69,7 +71,10 @@ exports.installArchive = async function installArchive(archive, options = {}) {
     await appendToConfig(installPath, 'xpack.security.enabled', 'true');
 
     await appendToConfig(installPath, 'xpack.license.self_generated.type', license);
-    await configureKeystore(installPath, password, log, bundledJDK);
+    await configureKeystore(installPath, log, bundledJDK, [
+      ['bootstrap.password', password],
+      ...parseSettings(esArgs, { filter: SettingsFilter.SecureOnly }),
+    ]);
   }
 
   return { installPath };
@@ -90,21 +95,33 @@ async function appendToConfig(installPath, key, value) {
  * Creates and configures Keystore
  *
  * @param {String} installPath
- * @param {String} password
  * @param {ToolingLog} log
+ * @param {boolean} bundledJDK
+ * @param {Array<[string, string]>} secureSettings List of custom Elasticsearch secure settings to
+ * add into the keystore.
  */
-async function configureKeystore(installPath, password, log = defaultLog, bundledJDK = false) {
-  log.info('setting bootstrap password to %s', chalk.bold(password));
-
+async function configureKeystore(
+  installPath,
+  log = defaultLog,
+  bundledJDK = false,
+  secureSettings
+) {
   const env = {};
   if (bundledJDK) {
     env.JAVA_HOME = '';
   }
   await execa(ES_KEYSTORE_BIN, ['create'], { cwd: installPath, env });
 
-  await execa(ES_KEYSTORE_BIN, ['add', 'bootstrap.password', '-x'], {
-    input: password,
-    cwd: installPath,
-    env,
-  });
+  for (const [secureSettingName, secureSettingValue] of secureSettings) {
+    log.info(
+      `setting secure setting %s to %s`,
+      chalk.bold(secureSettingName),
+      chalk.bold(secureSettingValue)
+    );
+    await execa(ES_KEYSTORE_BIN, ['add', secureSettingName, '-x'], {
+      input: secureSettingValue,
+      cwd: installPath,
+      env,
+    });
+  }
 }

packages/kbn-es/src/install/snapshot.js

@@ -73,6 +73,7 @@ exports.installSnapshot = async function installSnapshot({
   installPath = path.resolve(basePath, version),
   log = defaultLog,
   bundledJDK = true,
+  esArgs,
 }) {
   const { downloadPath } = await exports.downloadSnapshot({
     license,
@@ -89,5 +90,6 @@ exports.installSnapshot = async function installSnapshot({
     installPath,
     log,
     bundledJDK,
+    esArgs,
   });
 };

packages/kbn-es/src/install/source.js

@@ -45,6 +45,7 @@ exports.installSource = async function installSource({
   basePath = BASE_PATH,
   installPath = path.resolve(basePath, 'source'),
   log = defaultLog,
+  esArgs,
 }) {
   log.info('source path: %s', chalk.bold(sourcePath));
   log.info('install path: %s', chalk.bold(installPath));
@@ -70,6 +71,7 @@ exports.installSource = async function installSource({
     basePath,
     installPath,
     log,
+    esArgs,
   });
 };
 

packages/kbn-es/src/settings.test.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { parseSettings, SettingsFilter } from './settings';
+
+const mockSettings = [
+  'abc.def=1',
+  'xpack.security.authc.realms.oidc.oidc1.rp.client_secret=secret',
+  'xpack.security.authc.realms.oidc.oidc1.rp.client_id=client id',
+  'discovery.type=single-node',
+];
+
+test('`parseSettings` parses and returns all settings by default', () => {
+  expect(parseSettings(mockSettings)).toEqual([
+    ['abc.def', '1'],
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'secret'],
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'client id'],
+    ['discovery.type', 'single-node'],
+  ]);
+});
+
+test('`parseSettings` parses and returns all settings with `SettingsFilter.All` filter', () => {
+  expect(parseSettings(mockSettings, { filter: SettingsFilter.All })).toEqual([
+    ['abc.def', '1'],
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'secret'],
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'client id'],
+    ['discovery.type', 'single-node'],
+  ]);
+});
+
+test('`parseSettings` parses and returns only secure settings with `SettingsFilter.SecureOnly` filter', () => {
+  expect(parseSettings(mockSettings, { filter: SettingsFilter.SecureOnly })).toEqual([
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'secret'],
+  ]);
+});
+
+test('`parseSettings` parses and returns only non-secure settings with `SettingsFilter.NonSecureOnly` filter', () => {
+  expect(parseSettings(mockSettings, { filter: SettingsFilter.NonSecureOnly })).toEqual([
+    ['abc.def', '1'],
+    ['xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'client id'],
+    ['discovery.type', 'single-node'],
+  ]);
+});

packages/kbn-es/src/settings.ts

@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * List of the patterns for the settings names that are supposed to be secure and stored in the keystore.
+ */
+const SECURE_SETTINGS_LIST = [
+  /^xpack\.security\.authc\.realms\.oidc\.[a-zA-Z0-9_]+\.rp\.client_secret$/,
+];
+
+function isSecureSetting(settingName: string) {
+  return SECURE_SETTINGS_LIST.some(secureSettingNameRegex =>
+    secureSettingNameRegex.test(settingName)
+  );
+}
+
+export enum SettingsFilter {
+  All = 'all',
+  SecureOnly = 'secure-only',
+  NonSecureOnly = 'non-secure-only',
+}
+
+/**
+ * Accepts an array of `esSettingName=esSettingValue` strings and parses them into an array of
+ * [esSettingName, esSettingValue] tuples optionally filter out secure or non-secure settings.
+ * @param rawSettingNameValuePairs Array of strings to parse
+ * @param [filter] Optional settings filter.
+ */
+export function parseSettings(
+  rawSettingNameValuePairs: string[],
+  { filter }: { filter: SettingsFilter } = { filter: SettingsFilter.All }
+) {
+  const settings: Array<[string, string]> = [];
+  for (const rawSettingNameValuePair of rawSettingNameValuePairs) {
+    const [settingName, settingValue] = rawSettingNameValuePair.split('=');
+
+    const includeSetting =
+      filter === SettingsFilter.All ||
+      (filter === SettingsFilter.SecureOnly && isSecureSetting(settingName)) ||
+      (filter === SettingsFilter.NonSecureOnly && !isSecureSetting(settingName));
+    if (includeSetting) {
+      settings.push([settingName, settingValue]);
+    }
+  }
+
+  return settings;
+}

packages/kbn-es/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": [
+    "src/**/*.ts"
+  ]
+}

packages/kbn-test/src/es/es_test_cluster.js

@@ -37,6 +37,7 @@ export function createEsTestCluster(options = {}) {
     basePath = resolve(KIBANA_ROOT, '.es'),
     esFrom = esTestConfig.getBuildFrom(),
     dataArchive,
+    esArgs,
   } = options;
 
   const randomHash = Math.random()
@@ -50,6 +51,7 @@ export function createEsTestCluster(options = {}) {
     password,
     license,
     basePath,
+    esArgs,
   };
 
   const cluster = new Cluster(log);

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js

@@ -40,6 +40,7 @@ export async function runElasticsearch({ config, options }) {
     basePath: resolve(KIBANA_ROOT, '.es'),
     esFrom: esFrom || config.get('esTestCluster.from'),
     dataArchive: config.get('esTestCluster.dataArchive'),
+    esArgs,
   });
 
   await cluster.start(esArgs, esEnvVars);

scripts/es.js

@@ -17,12 +17,12 @@
  * under the License.
  */
 
+require('../src/setup_node_env');
+
 var resolve = require('path').resolve;
 var pkg = require('../package.json');
 var kbnEs = require('@kbn/es');
 
-require('../src/setup_node_env');
-
 kbnEs
   .run({
     license: 'basic',

x-pack/scripts/functional_tests.js

@@ -17,7 +17,7 @@ require('@kbn/test').runTestsCli([
   require.resolve('../test/kerberos_api_integration/anonymous_access.config'),
   require.resolve('../test/saml_api_integration/config.js'),
   require.resolve('../test/token_api_integration/config.js'),
-  // require.resolve('../test/oidc_api_integration/config.js'),
+  require.resolve('../test/oidc_api_integration/config.js'),
   require.resolve('../test/spaces_api_integration/spaces_only/config'),
   require.resolve('../test/spaces_api_integration/security_and_spaces/config_trial'),
   require.resolve('../test/spaces_api_integration/security_and_spaces/config_basic'),

x-pack/test/oidc_api_integration/config.js

@@ -32,6 +32,7 @@ export default async function ({ readConfigFile }) {
         'xpack.security.authc.token.timeout=15s',
         'xpack.security.authc.realms.oidc.oidc1.order=0',
         `xpack.security.authc.realms.oidc.oidc1.rp.client_id=0oa8sqpov3TxMWJOt356`,
+        `xpack.security.authc.realms.oidc.oidc1.rp.client_secret=0oa8sqpov3TxMWJOt356`,
         `xpack.security.authc.realms.oidc.oidc1.rp.response_type=code`,
         `xpack.security.authc.realms.oidc.oidc1.rp.redirect_uri=http://localhost:${kibanaPort}/api/security/v1/oidc`,
         `xpack.security.authc.realms.oidc.oidc1.op.authorization_endpoint=https://test-op.elastic.co/oauth2/v1/authorize`,