[FIPS / CI] Fix ES ML startup issues, UUID permissions, FTR tests status, OpenSSL target. Switch to Ubuntu (#182295)

Closes elastic/kibana-operations#96

[new pipeline run](https://buildkite.com/elastic/kibana-fips/builds/28)
[old pipeline run](https://buildkite.com/elastic/kibana-fips/builds/24)

- Fixes OpenSSL compilation so that it does not overwrite the OS version
and break OS functionality.
- Fixes issue where ES would not start due to ML pipe being unable to
write to Vagrant synced folder.
  - Reenabled ML in FIPS pipeline
- Fixes permission where Kibana could not write UUID file.
- Fixes smoke test exit code not reporting correctly.
- Fixes Buildkite annotation for failed tests
[example](https://buildkite.com/elastic/kibana-fips/builds/23).
- Switches the base VM image from RHEL9 to Ubuntu due to RHEL9
subscription requirements to install repo packages.
- This blocked installing Chrome, causing tests using Webdriver to fail.

---------

Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -36,9 +36,9 @@ for config in "${configs[@]}"; do
     echo "^^^ +++"
 
     if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"$config"
+      failedConfigs="${failedConfigs}"$'\n'"- ${config}"
     else
-      failedConfigs="$config"
+      failedConfigs="### Failed FTR Configs"$'\n'"- ${config}"
     fi
   fi
 done

.buildkite/scripts/steps/package_testing/test.sh

@@ -58,12 +58,16 @@ trap "echoKibanaLogs" EXIT
 if [[ "$TEST_PACKAGE" == "fips" ]]; then
   set +e
   vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh"
+  exitCode=$?
+
   vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs
   set -e
 
   if [ -s ftr_failed_configs ]; then
-    buildkite-agent meta-data set "ftr-failed-configs" <./ftr_failed_configs
+    cat ftr_failed_configs | buildkite-agent annotate --style "error"
   fi
+
+  exit $exitCode
 else
   vagrant provision "$TEST_PACKAGE"
 

packages/kbn-es/src/cluster.ts

@@ -89,8 +89,9 @@ export class Cluster {
   async installSource(options: InstallSourceOptions) {
     this.log.info(chalk.bold('Installing from source'));
     return await this.log.indent(4, async () => {
-      const { installPath } = await installSource({ log: this.log, ...options });
-      return { installPath };
+      const { installPath, disableEsTmpDir } = await installSource({ log: this.log, ...options });
+
+      return { installPath, disableEsTmpDir };
     });
   }
 
@@ -115,12 +116,12 @@ export class Cluster {
   async installSnapshot(options: InstallSnapshotOptions) {
     this.log.info(chalk.bold('Installing from snapshot'));
     return await this.log.indent(4, async () => {
-      const { installPath } = await installSnapshot({
+      const { installPath, disableEsTmpDir } = await installSnapshot({
         log: this.log,
         ...options,
       });
 
-      return { installPath };
+      return { installPath, disableEsTmpDir };
     });
   }
 
@@ -130,12 +131,12 @@ export class Cluster {
   async installArchive(archivePath: string, options?: InstallArchiveOptions) {
     this.log.info(chalk.bold('Installing from an archive'));
     return await this.log.indent(4, async () => {
-      const { installPath } = await installArchive(archivePath, {
+      const { installPath, disableEsTmpDir } = await installArchive(archivePath, {
         log: this.log,
         ...(options || {}),
       });
 
-      return { installPath };
+      return { installPath, disableEsTmpDir };
     });
   }
 
@@ -317,6 +318,7 @@ export class Cluster {
       skipReadyCheck,
       readyTimeout,
       writeLogsToPath,
+      disableEsTmpDir,
       ...options
     } = opts;
 
@@ -389,7 +391,9 @@ export class Cluster {
     this.process = execa(ES_BIN, args, {
       cwd: installPath,
       env: {
-        ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}),
+        ...(installPath && !disableEsTmpDir
+          ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') }
+          : {}),
         ...process.env,
         JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used
         ES_JAVA_OPTS: esJavaOpts,

packages/kbn-es/src/cluster_exec_options.ts

@@ -17,4 +17,6 @@ export interface EsClusterExecOptions {
   readyTimeout?: number;
   onEarlyExit?: (msg: string) => void;
   writeLogsToPath?: string;
+  /** Disable creating a temp directory, allowing ES to write to OS's /tmp directory */
+  disableEsTmpDir?: boolean;
 }

packages/kbn-es/src/install/install_archive.ts

@@ -40,6 +40,7 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
     installPath = path.resolve(basePath, path.basename(archive, '.tar.gz')),
     log = defaultLog,
     esArgs = [],
+    disableEsTmpDir = process.env.FTR_DISABLE_ES_TMPDIR?.toLowerCase() === 'true',
   } = options || {};
 
   let dest = archive;
@@ -62,9 +63,16 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
   });
   log.info('extracted to %s', chalk.bold(installPath));
 
-  const tmpdir = path.resolve(installPath, 'ES_TMPDIR');
-  fs.mkdirSync(tmpdir, { recursive: true });
-  log.info('created %s', chalk.bold(tmpdir));
+  /**
+   * If we're running inside a Vagrant VM, and this is running in a synced folder,
+   * ES will fail to start due to ML being unable to write a pipe in the synced folder.
+   * Disabling allows ES to write to the OS's /tmp directory.
+   */
+  if (!disableEsTmpDir) {
+    const tmpdir = path.resolve(installPath, 'ES_TMPDIR');
+    fs.mkdirSync(tmpdir, { recursive: true });
+    log.info('created %s', chalk.bold(tmpdir));
+  }
 
   // starting in 6.3, security is disabled by default. Since we bootstrap
   // the keystore, we can enable security ourselves.
@@ -76,7 +84,7 @@ export async function installArchive(archive: string, options?: InstallArchiveOp
     ...parseSettings(esArgs, { filter: SettingsFilter.SecureOnly }),
   ]);
 
-  return { installPath };
+  return { installPath, disableEsTmpDir };
 }
 
 /**

packages/kbn-es/src/install/types.ts

@@ -40,4 +40,6 @@ export interface InstallArchiveOptions {
   installPath?: string;
   log?: ToolingLog;
   esArgs?: string[];
+  /** Disable creating a temp directory, allowing ES to write to OS's /tmp directory */
+  disableEsTmpDir?: boolean;
 }

packages/kbn-es/src/integration_tests/cluster.test.ts

@@ -179,13 +179,13 @@ describe('#downloadSnapshot()', () => {
 });
 
 describe('#installSource()', () => {
-  test('awaits installSource() promise and returns { installPath }', async () => {
+  test('awaits installSource() promise and returns { installPath, disableEsTmpDir }', async () => {
     let resolveInstallSource: Function;
     installSourceMock.mockImplementationOnce(
       () =>
         new Promise((resolve) => {
           resolveInstallSource = () => {
-            resolve({ installPath: 'foo' });
+            resolve({ installPath: 'foo', disableEsTmpDir: false });
           };
         })
     );
@@ -196,11 +196,12 @@ describe('#installSource()', () => {
     resolveInstallSource!();
     await expect(ensureResolve(promise, 'installSource()')).resolves.toEqual({
       installPath: 'foo',
+      disableEsTmpDir: false,
     });
   });
 
   test('passes through all options+log to installSource()', async () => {
-    installSourceMock.mockResolvedValue({ installPath: 'foo' });
+    installSourceMock.mockResolvedValue({ installPath: 'foo', disableEsTmpDir: false });
     const options: InstallSourceOptions = {
       sourcePath: 'bar',
       license: 'trial',
@@ -228,13 +229,13 @@ describe('#installSource()', () => {
 });
 
 describe('#installSnapshot()', () => {
-  test('awaits installSnapshot() promise and returns { installPath }', async () => {
+  test('awaits installSnapshot() promise and returns { installPath, disableEsTmpDir }', async () => {
     let resolveInstallSnapshot: Function;
     installSnapshotMock.mockImplementationOnce(
       () =>
         new Promise((resolve) => {
           resolveInstallSnapshot = () => {
-            resolve({ installPath: 'foo' });
+            resolve({ installPath: 'foo', disableEsTmpDir: false });
           };
         })
     );
@@ -245,11 +246,12 @@ describe('#installSnapshot()', () => {
     resolveInstallSnapshot!();
     await expect(ensureResolve(promise, 'installSnapshot()')).resolves.toEqual({
       installPath: 'foo',
+      disableEsTmpDir: false,
     });
   });
 
   test('passes through all options+log to installSnapshot()', async () => {
-    installSnapshotMock.mockResolvedValue({ installPath: 'foo' });
+    installSnapshotMock.mockResolvedValue({ installPath: 'foo', disableEsTmpDir: false });
     const options: InstallSnapshotOptions = {
       version: '8.10.0',
       license: 'trial',
@@ -278,13 +280,13 @@ describe('#installSnapshot()', () => {
 });
 
 describe('#installArchive()', () => {
-  test('awaits installArchive() promise and returns { installPath }', async () => {
+  test('awaits installArchive() promise and returns { installPath, disableEsTmpDir }', async () => {
     let resolveInstallArchive: Function;
     installArchiveMock.mockImplementationOnce(
       () =>
         new Promise((resolve) => {
           resolveInstallArchive = () => {
-            resolve({ installPath: 'foo' });
+            resolve({ installPath: 'foo', disableEsTmpDir: false });
           };
         })
     );
@@ -295,11 +297,12 @@ describe('#installArchive()', () => {
     resolveInstallArchive!();
     await expect(ensureResolve(promise, 'installArchive()')).resolves.toEqual({
       installPath: 'foo',
+      disableEsTmpDir: false,
     });
   });
 
   test('passes through all options+log to installArchive()', async () => {
-    installArchiveMock.mockResolvedValue({ installPath: 'foo' });
+    installArchiveMock.mockResolvedValue({ installPath: 'foo', disableEsTmpDir: true });
     const options: InstallArchiveOptions = {
       license: 'trial',
       password: 'changeme',
@@ -307,6 +310,7 @@ describe('#installArchive()', () => {
       installPath: 'someInstallPath',
       esArgs: ['foo=true'],
       log,
+      disableEsTmpDir: true,
     };
     const cluster = new Cluster({ log });
     await cluster.installArchive('bar', options);

packages/kbn-test/src/es/test_es_cluster.ts

@@ -182,7 +182,6 @@ export function createTestEsCluster<
   } = options;
 
   const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`;
-  const isFIPSMode = process.env.FTR_FIPS_MODE === '1';
 
   const defaultEsArgs = [
     `cluster.name=${clusterName}`,
@@ -193,12 +192,7 @@ export function createTestEsCluster<
       : ['discovery.type=single-node']),
   ];
 
-  const esArgs = assignArgs(
-    defaultEsArgs,
-    // ML has issues running in FIPS mode due to custom OpenSSL
-    // Remove after https://github.com/elastic/kibana-operations/issues/96
-    isFIPSMode ? [...customEsArgs, 'xpack.ml.enabled=false'] : customEsArgs
-  );
+  const esArgs = assignArgs(defaultEsArgs, customEsArgs);
 
   const config = {
     version: esTestConfig.getVersion(),
@@ -231,22 +225,21 @@ export function createTestEsCluster<
 
     async start() {
       let installPath: string;
+      let disableEsTmpDir: boolean;
 
       // We only install once using the first node. If the cluster has
       // multiple nodes, they'll all share the same ESinstallation.
       const firstNode = this.nodes[0];
       if (esFrom === 'source') {
-        installPath = (
-          await firstNode.installSource({
-            sourcePath: config.sourcePath,
-            license: config.license,
-            password: config.password,
-            basePath: config.basePath,
-            esArgs: config.esArgs,
-          })
-        ).installPath;
+        ({ installPath, disableEsTmpDir } = await firstNode.installSource({
+          sourcePath: config.sourcePath,
+          license: config.license,
+          password: config.password,
+          basePath: config.basePath,
+          esArgs: config.esArgs,
+        }));
       } else if (esFrom === 'snapshot') {
-        installPath = (await firstNode.installSnapshot(config)).installPath;
+        ({ installPath, disableEsTmpDir } = await firstNode.installSnapshot(config));
       } else if (esFrom === 'serverless') {
         if (!esServerlessOptions) {
           throw new Error(
@@ -308,6 +301,7 @@ export function createTestEsCluster<
             skipReadyCheck: this.nodes.length > 1 && i < this.nodes.length - 1,
             onEarlyExit,
             writeLogsToPath,
+            disableEsTmpDir,
           });
         });
       }

test/package/Vagrantfile

@@ -46,14 +46,7 @@ Vagrant.configure("2") do |config|
       vb.memory = 4096
       vb.cpus = 2
     end
-    fips.vm.box = 'generic/rhel9'
-    fips.vm.provision "shell", inline: <<-SHELL
-      echo "export OPENSSL_MODULES=/usr/local/lib64/ossl-modules" >> /etc/profile.d/kibana-fips-env.sh
-      echo "export TEST_BROWSER_HEADLESS=1" >> /etc/profile.d/kibana-fips-env.sh
-      echo "export ES_TMPDIR=/home/vagrant/kibana/.es/tmp" >> /etc/profile.d/kibana-fips-env.sh
-      # Remove after https://github.com/elastic/kibana-operations/issues/96
-      echo "export FTR_FIPS_MODE=1" >> /etc/profile.d/kibana-fips-env.sh
-    SHELL
+    fips.vm.box = 'ubuntu/jammy64'
     fips.vm.provision "ansible" do |ansible|
       ansible.playbook = "fips.yml"
     end

test/package/fips.yml

@@ -6,7 +6,9 @@
     nvm_ver: "0.39.7"
     openssl_sha: "sha256:6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e"
     openssl_ver: "3.0.8"
+    openssl_src_path: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    openssl_path: "{{ kibana_dist_path }}/openssl"
   roles:
-    - upgrade_yum_packages
+    - upgrade_apt_packages
     - install_kibana_fips
     - assert_fips_enabled

test/package/roles/assert_fips_enabled/tasks/main.yml

@@ -1,5 +1,7 @@
 - name: register kibana node getFips
-  command: "{{ kibana_dist_path }}/node/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
+  shell: 
+    cmd: "source /home/vagrant/.profile && {{ kibana_dist_path }}/node/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
+    executable: /bin/bash
   register: kibana_node_fips
 
 - debug:

test/package/roles/install_kibana_fips/tasks/main.yml

@@ -6,18 +6,37 @@
       - "processor_cores"
   when: ansible_processor_vcpus is not defined
 
-- name: fix /var/log permissions for kibana
-  become: yes
-  file:
-    path: /var/log
-    state: directory
-    recurse: true
-    mode: "0777"
+- name: setup env variables
+  blockinfile:
+    path: "/home/vagrant/.profile"
+    block: |
+      export OPENSSL_MODULES=/usr/share/kibana/openssl/lib/ossl-modules
+      export TEST_BROWSER_HEADLESS=1
+      export FTR_DISABLE_ES_TMPDIR=true
+    owner: vagrant
+    group: vagrant
+    mode: '0644'
 
-- name: create tmp dir for ES
-  file:
-    path: "{{ kibana_src_path }}/.es/tmp"
-    state: directory
+- name: add chrome apt signing key
+  become: yes
+  apt_key:
+    url: https://dl.google.com/linux/linux_signing_key.pub
+    state: present
+
+- name: add chrome apt repository
+  become: yes
+  apt_repository:
+    repo: deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main
+    state: present
+
+- name: install apt packages
+  become: yes
+  apt:
+    pkg:
+      - build-essential
+      - google-chrome-stable
+      - unzip
+    state: latest
 
 - name: slurp kibana node version
   slurp:
@@ -31,7 +50,7 @@
 - name: install nvm
   shell:
     chdir: "$HOME"
-    cmd: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v{{ nvm_ver }}/install.sh | bash
+    cmd: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v{{ nvm_ver }}/install.sh | PROFILE=/home/vagrant/.profile bash
 
 - name: install kibana node version
   shell:
@@ -40,12 +59,11 @@
   args:
     executable: /bin/bash
 
-- name: "ensure {{ kibana_dist_path }} dir exists"
+- name: "ensure {{ openssl_path }} dir exists"
   become: yes
   file:
-    path: "{{ kibana_dist_path }}"
+    path: "{{ openssl_path }}"
     state: directory
-    mode: "0777"
 
 - name: find kibana distribution
   find:
@@ -99,35 +117,54 @@
   delay: 10
   get_url:
     url: "https://www.openssl.org/source/openssl-{{ openssl_ver }}.tar.gz"
-    dest: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}.tar.gz"
+    dest: "{{ openssl_src_path }}.tar.gz"
     checksum: "{{ openssl_sha }}"
 
 - name: extract OpenSSL
   become: yes
   unarchive:
-    src: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}.tar.gz"
+    src: "{{ openssl_src_path }}.tar.gz"
     dest: "{{ kibana_dist_path }}"
     remote_src: yes
 
 - name: configure OpenSSL for FIPS
   become: yes
   shell:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
-    cmd: ./Configure enable-fips
+    chdir: "{{ openssl_src_path }}"
+    cmd: "./Configure --prefix={{ openssl_path }} --openssldir={{ openssl_path }}/ssl --libdir={{ openssl_path }}/lib enable-fips"
 
 - name: compile OpenSSL with FIPS
   become: yes
   make:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    chdir: "{{ openssl_src_path }}"
     jobs: "{{ ansible_facts['processor_vcpus'] }}"
 
 - name: install OpenSSL with FIPS
   become: yes
   make:
-    chdir: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
+    chdir: "{{ openssl_src_path }}"
     target: install
 
-- name: link OpenSSL package
+- name: "change owner of {{ kibana_dist_path }} to vagrant"
   become: yes
-  shell:
-    cmd: ldconfig /usr/local/lib64/
+  file:
+    path: "{{ kibana_dist_path }}"
+    owner: vagrant
+    group: vagrant
+    recurse: yes
+
+- name: fix /var/log permissions for kibana
+  become: yes
+  file:
+    path: /var/log
+    state: directory
+    recurse: true
+    mode: "0777"
+
+- name: increase vm.max_map_count for ES
+  become: yes
+  sysctl:
+    name: vm.max_map_count
+    value: '262144'
+    state: present
+    reload: yes

test/package/templates/fips/nodejs.cnf

@@ -9,7 +9,7 @@
 ##########################################################################
 
 nodejs_conf = nodejs_init
-.include /usr/local/ssl/fipsmodule.cnf
+.include /usr/share/kibana/openssl/ssl/fipsmodule.cnf
 
 [nodejs_init]
 providers = provider_sect