[CI / FIPS] Convert smoke tests to use FIPS agent (#186858)

## Summary

Closes elastic/kibana-operations#137

- Convert FIPS daily smoke test pipeline to use FIPS agents
  - This allows the smoke tests to run in parallel now
- Removes FIPS Vagrant box from package testing
- Adds verify FIPS enabled check to pipeline 

Pipeline run: https://buildkite.com/elastic/kibana-fips/builds/90

.buildkite/pipelines/fips.yml

@@ -1,32 +1,63 @@
 env:
-  DISABLE_CI_STATS_SHIPPING: "true"
+  DISABLE_CI_STATS_SHIPPING: 'true'
+  KBN_ENABLE_FIPS: 'true'
+  TEST_BROWSER_HEADLESS: 1
+agents:
+  provider: 'gcp'
+  image: 'family/kibana-fips-ubuntu-2004'
+  imageProject: 'elastic-images-prod'
 steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    key: pre-build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+
+  - wait
+
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    depends_on: pre-build
     timeout_in_minutes: 60
     retry:
       automatic:
-        - exit_status: "-1"
+        - exit_status: '-1'
           limit: 3
 
   - wait
 
-  - command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh
-    label: "Smoke testing for FIPS"
+  - command: .buildkite/scripts/steps/checks/verify_fips_enabled.sh
+    label: 'Verify FIPS Enabled'
+    depends_on: build
+    timeout_in_minutes: 10
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      enableNestedVirtualization: true
-      localSsds: 1
-      localSsdInterface: nvme
-      machineType: n2-standard-4
-    timeout_in_minutes: 600
+      machineType: n2-standard-2
+      preemptible: true
+
+  - command: .buildkite/scripts/steps/fips/smoke_test.sh
+    label: 'Pick Smoke Test Group Run Order'
+    depends_on: build
+    timeout_in_minutes: 10
+    env:
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
+      LIMIT_CONFIG_TYPE: 'functional'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2

.buildkite/scripts/steps/checks/verify_fips_enabled.sh

@@ -2,7 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
+# This script is part of checks.sh in the PR pipeline but is called directly in the FIPS pipeline, so we need to bootstrap
+if [[ -z "${BASH_SOURCE[1]+x}" || "${BASH_SOURCE[1]}" != *"checks.sh"* ]]; then
+  export DISABLE_BOOTSTRAP_VALIDATION=false
+  .buildkite/scripts/bootstrap.sh
+fi
 
 .buildkite/scripts/download_build_artifacts.sh
 

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -1,12 +1,10 @@
 #!/usr/bin/env bash
 
-if [ -z "$KIBANA_BUILD_LOCATION" ]; then
-  export KIBANA_BUILD_LOCATION="/usr/share/kibana"
-fi
-
-# a FTR failure will result in the script returning an exit code of 10
-exitCode=0
+set -euo pipefail
 
+# Limit the FTR configs for now to avoid running all the tests. Once we're
+# ready to utilize the full FTR suite in FIPS mode, we can remove this file and
+# call pick_test_group_run_order.sh directly in .buildkite/pipelines/fips.yml.
 configs=(
   "x-pack/test/reporting_functional/reporting_and_security.config.ts"
   "x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts"
@@ -19,34 +17,8 @@ configs=(
   "x-pack/test/functional/apps/security/config.ts"
 )
 
-cd /home/vagrant/kibana
+printf -v FTR_CONFIG_PATTERNS '%s,' "${configs[@]}"
+FTR_CONFIG_PATTERNS="${FTR_CONFIG_PATTERNS%,}"
+export FTR_CONFIG_PATTERNS
 
-for config in "${configs[@]}"; do
-  set +e
-  node /home/vagrant/kibana/scripts/functional_tests \
-    --bail \
-    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-    --config="$config"
-  lastCode=$?
-  set -e
-
-  if [ $lastCode -ne 0 ]; then
-    exitCode=10
-    echo "FTR exited with code $lastCode"
-    echo "^^^ +++"
-
-    if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"- ${config}"
-    else
-      failedConfigs="### Failed FTR Configs"$'\n'"- ${config}"
-    fi
-  fi
-done
-
-if [[ "$failedConfigs" ]]; then
-  echo "$failedConfigs" >/home/vagrant/ftr_failed_configs
-fi
-
-echo "--- FIPS smoke test complete"
-
-exit $exitCode
+.buildkite/scripts/steps/test/pick_test_group_run_order.sh

.buildkite/scripts/steps/package_testing/test.sh

@@ -21,25 +21,17 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
   download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
-elif [[ "$TEST_PACKAGE" == "fips" ]]; then
-  download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 fi
 cd ..
 
 export VAGRANT_CWD=$PWD/test/package
+vagrant up "$TEST_PACKAGE" --no-provision
 
-if [[ "$TEST_PACKAGE" == "fips" ]]; then
-  vagrant up "$TEST_PACKAGE"
-else
-  vagrant up "$TEST_PACKAGE" --no-provision
-
-  node scripts/es snapshot \
+node scripts/es snapshot \
   -E network.bind_host=127.0.0.1,192.168.56.1 \
   -E discovery.type=single-node \
   --license=trial &
-  while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
-fi
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
 
 function echoKibanaLogs {
   if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
@@ -55,29 +47,13 @@ function echoKibanaLogs {
 }
 trap "echoKibanaLogs" EXIT
 
-if [[ "$TEST_PACKAGE" == "fips" ]]; then
-  set +e
-  vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh"
-  exitCode=$?
+vagrant provision "$TEST_PACKAGE"
 
-  vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs
-  set -e
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
+export TEST_ES_URL="http://elastic:changeme@192.168.56.1:9200"
 
-  if [ -s ftr_failed_configs ]; then
-    cat ftr_failed_configs | buildkite-agent annotate --style "error"
-  fi
+cd x-pack
 
-  exit $exitCode
-else
-  vagrant provision "$TEST_PACKAGE"
-
-  export TEST_BROWSER_HEADLESS=1
-  export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
-  export TEST_ES_URL="http://elastic:changeme@192.168.56.1:9200"
-
-  echo "--- FTR - Reporting"
-
-  cd x-pack
-
-  node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet
-fi
+echo "--- FTR - Reporting"
+node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet

test/package/Vagrantfile

@@ -39,16 +39,4 @@ Vagrant.configure("2") do |config|
     end
     docker.vm.network "private_network", ip: "192.168.56.7"
   end
-
-  config.vm.define "fips" do |fips|
-    fips.vm.synced_folder '../../', '/home/vagrant/kibana', SharedFoldersEnableSymlinksCreate: false
-    fips.vm.provider :virtualbox do |vb|
-      vb.memory = 4096
-      vb.cpus = 2
-    end
-    fips.vm.box = 'ubuntu/jammy64'
-    fips.vm.provision "ansible" do |ansible|
-      ansible.playbook = "fips.yml"
-    end
-  end
 end

test/package/fips.yml

@@ -1,14 +0,0 @@
-- name: test kibana fips docker package
-  hosts: fips
-  vars:
-    kibana_dist_path: "/usr/share/kibana"
-    kibana_src_path: "/home/vagrant/kibana"
-    nvm_ver: "0.39.7"
-    openssl_sha: "sha256:6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e"
-    openssl_ver: "3.0.8"
-    openssl_src_path: "{{ kibana_dist_path }}/openssl-{{ openssl_ver }}"
-    openssl_path: "{{ kibana_dist_path }}/openssl"
-  roles:
-    - upgrade_apt_packages
-    - install_kibana_fips
-    - assert_fips_enabled

test/package/roles/assert_fips_enabled/tasks/main.yml

@@ -1,13 +0,0 @@
-- name: register kibana node getFips
-  shell: 
-    cmd: "source /home/vagrant/.profile && {{ kibana_dist_path }}/node/glibc-217/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
-    executable: /bin/bash
-  register: kibana_node_fips
-
-- debug:
-    msg: "{{ kibana_node_fips }}"
-
-- name: assert FIPS enabled
-  assert:
-    that:
-      - kibana_node_fips.stdout == "1"

test/package/roles/install_kibana_fips/tasks/main.yml

@@ -1,170 +0,0 @@
-- name: gather ansible processor facts
-  setup:
-    gather_subset:
-      - "!all"
-      - "!min"
-      - "processor_cores"
-  when: ansible_processor_vcpus is not defined
-
-- name: setup env variables
-  blockinfile:
-    path: "/home/vagrant/.profile"
-    block: |
-      export OPENSSL_MODULES=/usr/share/kibana/openssl/lib/ossl-modules
-      export TEST_BROWSER_HEADLESS=1
-      export FTR_DISABLE_ES_TMPDIR=true
-    owner: vagrant
-    group: vagrant
-    mode: '0644'
-
-- name: add chrome apt signing key
-  become: yes
-  apt_key:
-    url: https://dl.google.com/linux/linux_signing_key.pub
-    state: present
-
-- name: add chrome apt repository
-  become: yes
-  apt_repository:
-    repo: deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main
-    state: present
-
-- name: install apt packages
-  become: yes
-  apt:
-    pkg:
-      - build-essential
-      - google-chrome-stable
-      - unzip
-    state: latest
-
-- name: slurp kibana node version
-  slurp:
-    src: "{{ kibana_src_path }}/.node-version"
-  register: node_ver_file
-
-- name: set kibana node version
-  set_fact:
-    node_version: "{{ node_ver_file['content'] | b64decode | trim }}"
-
-- name: install nvm
-  shell:
-    chdir: "$HOME"
-    cmd: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v{{ nvm_ver }}/install.sh | PROFILE=/home/vagrant/.profile bash
-
-- name: install kibana node version
-  shell:
-    chdir: "$HOME/.nvm"
-    cmd: "source nvm.sh && nvm install {{ node_version }}"
-  args:
-    executable: /bin/bash
-
-- name: "ensure {{ openssl_path }} dir exists"
-  become: yes
-  file:
-    path: "{{ openssl_path }}"
-    state: directory
-
-- name: find kibana distribution
-  find:
-    paths: /packages/
-    patterns: kibana-default.tar.gz
-  register: kibana_tar
-
-- name: extract kibana distribution
-  become: yes
-  unarchive:
-    src: "{{ kibana_tar.files[0].path }}"
-    dest: "{{ kibana_dist_path }}"
-    remote_src: yes
-    extra_opts: ["--strip-components=1"]
-
-- name: find kibana plugins distribution
-  find:
-    paths: /packages/
-    patterns: kibana-default-plugins.tar.gz
-  register: kibana_plugins_tar
-
-- name: extract kibana plugins distribution
-  become: yes
-  unarchive:
-    src: "{{ kibana_plugins_tar.files[0].path }}"
-    dest: "{{ kibana_dist_path }}"
-    remote_src: yes
-
-- name: copy kibana yml configuration
-  become: yes
-  template:
-    src: templates/fips/kibana.yml
-    dest: "{{ kibana_dist_path }}/config/kibana.yml"
-  register: config
-
-- name: copy FIPS node.options
-  become: yes
-  template:
-    src: templates/fips/node.options
-    dest: "{{ kibana_dist_path }}/config/node.options"
-
-- name: copy FIPS openssl config
-  become: yes
-  template:
-    src: templates/fips/nodejs.cnf
-    dest: "{{ kibana_dist_path }}/config/nodejs.cnf"
-
-- name: download FIPS certified OpenSSL
-  become: yes
-  retries: 5
-  delay: 10
-  get_url:
-    url: "https://www.openssl.org/source/openssl-{{ openssl_ver }}.tar.gz"
-    dest: "{{ openssl_src_path }}.tar.gz"
-    checksum: "{{ openssl_sha }}"
-
-- name: extract OpenSSL
-  become: yes
-  unarchive:
-    src: "{{ openssl_src_path }}.tar.gz"
-    dest: "{{ kibana_dist_path }}"
-    remote_src: yes
-
-- name: configure OpenSSL for FIPS
-  become: yes
-  shell:
-    chdir: "{{ openssl_src_path }}"
-    cmd: "./Configure --prefix={{ openssl_path }} --openssldir={{ openssl_path }}/ssl --libdir={{ openssl_path }}/lib enable-fips"
-
-- name: compile OpenSSL with FIPS
-  become: yes
-  make:
-    chdir: "{{ openssl_src_path }}"
-    jobs: "{{ ansible_facts['processor_vcpus'] }}"
-
-- name: install OpenSSL with FIPS
-  become: yes
-  make:
-    chdir: "{{ openssl_src_path }}"
-    target: install
-
-- name: "change owner of {{ kibana_dist_path }} to vagrant"
-  become: yes
-  file:
-    path: "{{ kibana_dist_path }}"
-    owner: vagrant
-    group: vagrant
-    recurse: yes
-
-- name: fix /var/log permissions for kibana
-  become: yes
-  file:
-    path: /var/log
-    state: directory
-    recurse: true
-    mode: "0777"
-
-- name: increase vm.max_map_count for ES
-  become: yes
-  sysctl:
-    name: vm.max_map_count
-    value: '262144'
-    state: present
-    reload: yes

test/package/templates/fips/kibana.yml

@@ -1,16 +0,0 @@
-server.host: 0.0.0.0
-
-elasticsearch.username: "{{ elasticsearch_username }}"
-elasticsearch.password: "{{ elasticsearch_password }}"
-
-logging:
-  appenders:
-    file:
-      type: file
-      fileName: /var/log/kibana/kibana.log
-      layout:
-        type: json
-  root:
-    appenders:
-      - default
-      - file

test/package/templates/fips/node.options

@@ -1,4 +0,0 @@
---max-old-space-size=812
---unhandled-rejections=warn
---enable-fips
---openssl-config=/usr/share/kibana/config/nodejs.cnf

test/package/templates/fips/nodejs.cnf

@@ -1,28 +0,0 @@
-##########################################################################
-##                                                                      ##
-## This OpenSSL config is only loaded when running Kibana in FIPS mode. ##
-##                                                                      ##
-## See:                                                                 ##
-## https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md   ##
-## https://www.openssl.org/docs/man3.0/man7/fips_module.html            ##
-##                                                                      ##
-##########################################################################
-
-nodejs_conf = nodejs_init
-.include /usr/share/kibana/openssl/ssl/fipsmodule.cnf
-
-[nodejs_init]
-providers = provider_sect
-alg_section = algorithm_sect
-
-[provider_sect]
-default = default_sect
-# The fips section name should match the section name inside the
-# included fipsmodule.cnf.
-fips = fips_sect
-
-[default_sect]
-activate = 1
-
-[algorithm_sect]
-default_properties = fips=yes