[build] Add Docker images with FIPS (#175946)

## Summary
Closes elastic/kibana-operations/issues/24

This adds a second flavor of UBI image (`kibana-ubi-fips`) which has a
FIPS compliant version of OpenSSL compiled and linked to Node. Using the
label `ci:build-docker-fips` will create the image in CI and push to the
registry.

The FIPS image start the Kibana NodeJS process using the FIPS compliant
OpenSSL version. Kibana will start in this state but crash during
runtime because there are many code changes required for it to be FIPS
compliant, including `node_module` usage. I attempted numerous ways to
load other OpenSSL providers alongside the FIPS provider, but it always
led to Kibana crashing on invalid algorithm usage.

---------

Co-authored-by: Tiago Costa <tiago.costa@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/pull_request/fips.yml

@@ -0,0 +1,14 @@
+steps:
+  - command: .buildkite/scripts/steps/fips/build.sh
+    label: 'Build FIPS Image'
+    agents:
+      queue: n2-2-spot
+    depends_on:
+      - build
+      - quick_checks
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/scripts/build_kibana.sh

@@ -31,6 +31,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --docker-tag-qualifier="$GIT_COMMIT" \
   --docker-push \
   --skip-docker-ubi \
+  --skip-docker-fips \
   --skip-docker-ubuntu \
   --skip-docker-serverless \
   --skip-docker-contexts

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -139,6 +139,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-docker-fips')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
+    }
+
     if (
       GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -36,6 +36,7 @@ node scripts/build \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
   --skip-docker-ubi \
+  --skip-docker-fips \
   --skip-docker-cloud \
   --skip-docker-contexts
 

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -42,6 +42,7 @@ else
     --docker-tag-qualifier="$GIT_COMMIT" \
     --docker-push \
     --skip-docker-ubi \
+    --skip-docker-fips \
     --skip-docker-ubuntu \
     --skip-docker-serverless \
     --skip-docker-contexts

.buildkite/scripts/steps/demo_env/kibana.sh

@@ -9,7 +9,7 @@ source "$(dirname "${0}")/config.sh"
 export KIBANA_IMAGE="gcr.io/elastic-kibana-184716/demo/kibana:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
 
 echo '--- Build Kibana'
-node scripts/build --debug --docker-images --example-plugins --skip-docker-ubi --skip-docker-cloud --skip-docker-serverless --skip-docker-contexts
+node scripts/build --debug --docker-images --example-plugins --skip-docker-ubi --skip-docker-fips --skip-docker-cloud --skip-docker-serverless --skip-docker-contexts
 
 echo '--- Build Docker image with example plugins'
 cd target/example_plugins

.buildkite/scripts/steps/fips/build.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+mkdir -p target
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+
+echo "--- Build FIPS image"
+node scripts/build \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-cdn-assets \
+    --skip-archives \
+    --docker-images \
+    --docker-namespace="kibana-ci" \
+    --docker-tag-qualifier="$BUILDKITE_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-cloud \
+    --skip-docker-serverless \
+    --skip-docker-contexts
+
+docker logout docker.elastic.co
+
+# Moving to `target/` first will keep `buildkite-agent` from including directories in the artifact name
+cd "$KIBANA_DIR/target"
+buildkite-agent artifact upload "./*docker-image*.tar.gz"

.buildkite/scripts/steps/package_testing/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts --skip-cdn-assets
+node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-fips --skip-docker-contexts --skip-cdn-assets
 
 DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"
 

dev_docs/tutorials/ci.mdx

@@ -51,6 +51,10 @@ Build an archive that can be used to serve Kibana's static assets.
 
 Build cloud Docker images that can be used for testing deployments on Elastic Cloud.
 
+#### `ci:build-docker-fips`
+
+Build Docker UBI x64 image with FIPS enabled.
+
 #### `ci:build-os-packages`
 
 Build Docker images, and Debian and RPM packages.

src/dev/build/args.test.ts

@@ -33,6 +33,7 @@ it('build default and oss dist for current platform, without packages, by defaul
         "createDebPackage": false,
         "createDockerCloud": false,
         "createDockerContexts": true,
+        "createDockerFIPS": false,
         "createDockerServerless": false,
         "createDockerUBI": false,
         "createDockerUbuntu": false,
@@ -72,6 +73,7 @@ it('builds packages if --all-platforms is passed', () => {
         "createDebPackage": true,
         "createDockerCloud": true,
         "createDockerContexts": true,
+        "createDockerFIPS": true,
         "createDockerServerless": true,
         "createDockerUBI": true,
         "createDockerUbuntu": true,
@@ -111,6 +113,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "createDebPackage": false,
         "createDockerCloud": false,
         "createDockerContexts": true,
+        "createDockerFIPS": false,
         "createDockerServerless": false,
         "createDockerUBI": false,
         "createDockerUbuntu": false,
@@ -150,6 +153,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "createDebPackage": true,
         "createDockerCloud": false,
         "createDockerContexts": true,
+        "createDockerFIPS": false,
         "createDockerServerless": false,
         "createDockerUBI": false,
         "createDockerUbuntu": false,
@@ -190,6 +194,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "createDebPackage": false,
         "createDockerCloud": true,
         "createDockerContexts": true,
+        "createDockerFIPS": true,
         "createDockerServerless": true,
         "createDockerUBI": true,
         "createDockerUbuntu": true,
@@ -237,6 +242,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "createDebPackage": false,
         "createDockerCloud": true,
         "createDockerContexts": true,
+        "createDockerFIPS": true,
         "createDockerServerless": true,
         "createDockerUBI": false,
         "createDockerUbuntu": true,
@@ -277,6 +283,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
         "createDebPackage": true,
         "createDockerCloud": true,
         "createDockerContexts": true,
+        "createDockerFIPS": true,
         "createDockerServerless": true,
         "createDockerUBI": true,
         "createDockerUbuntu": false,
@@ -305,3 +312,44 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
     }
   `);
 });
+
+it('limits packages if --all-platforms passed with --skip-docker-fips', () => {
+  expect(readCliArgs(['node', 'scripts/build', '--all-platforms', '--skip-docker-fips']))
+    .toMatchInlineSnapshot(`
+    Object {
+      "buildOptions": Object {
+        "buildCanvasShareableRuntime": true,
+        "createArchives": true,
+        "createCdnAssets": true,
+        "createDebPackage": true,
+        "createDockerCloud": true,
+        "createDockerContexts": true,
+        "createDockerFIPS": false,
+        "createDockerServerless": true,
+        "createDockerUBI": true,
+        "createDockerUbuntu": true,
+        "createGenericFolders": true,
+        "createPlatformFolders": true,
+        "createRpmPackage": true,
+        "dockerContextUseLocalArtifact": null,
+        "dockerCrossCompile": false,
+        "dockerNamespace": null,
+        "dockerPush": false,
+        "dockerTag": null,
+        "dockerTagQualifier": null,
+        "downloadCloudDependencies": true,
+        "downloadFreshNode": true,
+        "eprRegistry": "snapshot",
+        "initialize": true,
+        "isRelease": false,
+        "targetAllPlatforms": true,
+        "versionQualifier": "",
+        "withExamplePlugins": false,
+        "withTestPlugins": false,
+      },
+      "log": <ToolingLog>,
+      "showHelp": false,
+      "unknownFlags": Array [],
+    }
+  `);
+});

src/dev/build/args.ts

@@ -33,6 +33,7 @@ export function readCliArgs(argv: string[]) {
       'skip-docker-ubuntu',
       'skip-docker-cloud',
       'skip-docker-serverless',
+      'skip-docker-fips',
       'release',
       'skip-node-download',
       'skip-cloud-dependencies-download',
@@ -143,6 +144,7 @@ export function readCliArgs(argv: string[]) {
       isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-serverless']),
     createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']),
     createDockerContexts: !Boolean(flags['skip-docker-contexts']),
+    createDockerFIPS: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-fips']),
     targetAllPlatforms: Boolean(flags['all-platforms']),
     eprRegistry: flags['epr-registry'],
     buildCanvasShareableRuntime: !Boolean(flags['skip-canvas-shareable-runtime']),

src/dev/build/build_distributables.ts

@@ -34,6 +34,7 @@ export interface BuildOptions {
   createDockerCloud: boolean;
   createDockerServerless: boolean;
   createDockerContexts: boolean;
+  createDockerFIPS: boolean;
   versionQualifier: string | undefined;
   targetAllPlatforms: boolean;
   withExamplePlugins: boolean;
@@ -163,6 +164,11 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await run(Tasks.CreateDockerServerless);
   }
 
+  if (options.createDockerFIPS) {
+    // control w/ --docker-images or --skip-docker-fips or --skip-os-packages
+    await run(Tasks.CreateDockerFIPS);
+  }
+
   if (options.createDockerContexts) {
     // control w/ --skip-docker-contexts
     await run(Tasks.CreateDockerContexts);

src/dev/build/cli.ts

@@ -45,6 +45,7 @@ if (showHelp) {
         --skip-canvas-shareable-runtime      {dim Don't build the Canvas shareable runtime}
         --skip-docker-ubi                    {dim Don't build the docker ubi image}
         --skip-docker-ubuntu                 {dim Don't build the docker ubuntu image}
+        --skip-docker-fips                   {dim Don't build the docker fips image}
         --release                            {dim Produce a release-ready distributable}
         --version-qualifier                  {dim Suffix version with a qualifier}
         --skip-node-download                 {dim Reuse existing downloads of node.js}

src/dev/build/tasks/os_packages/create_os_package_tasks.ts

@@ -137,6 +137,20 @@ export const CreateDockerCloud: Task = {
   },
 };
 
+export const CreateDockerFIPS: Task = {
+  description: 'Creating Docker FIPS image',
+
+  async run(config, log, build) {
+    await runDockerGenerator(config, log, build, {
+      architecture: 'x64',
+      baseImage: 'ubi',
+      context: false,
+      image: true,
+      fips: true,
+    });
+  },
+};
+
 export const CreateDockerContexts: Task = {
   description: 'Creating Docker build contexts',
 
@@ -170,5 +184,11 @@ export const CreateDockerContexts: Task = {
       context: true,
       image: false,
     });
+    await runDockerGenerator(config, log, build, {
+      baseImage: 'ubi',
+      context: true,
+      image: false,
+      fips: true,
+    });
   },
 };

src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts

@@ -54,6 +54,12 @@ export async function bundleDockerFiles(config: Config, log: ToolingLog, scope:
       await write(resolve(dockerFilesBuildDir, template), output);
     }
   }
+  if (scope.fips) {
+    await copyAll(
+      resolve(scope.dockerBuildDir, 'openssl'),
+      resolve(dockerFilesBuildDir, 'openssl')
+    );
+  }
 
   // Compress dockerfiles dir created inside
   // docker build dir as output it as a target

src/dev/build/tasks/os_packages/docker_generator/resources/fips/openssl/nodejs.cnf

@@ -0,0 +1,28 @@
+##########################################################################
+##                                                                      ##
+## This OpenSSL config is only loaded when running Kibana in FIPS mode. ##
+##                                                                      ##
+## See:                                                                 ##
+## https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md   ##
+## https://www.openssl.org/docs/man3.0/man7/fips_module.html            ##
+##                                                                      ##
+##########################################################################
+
+nodejs_conf = nodejs_init
+.include /usr/local/ssl/fipsmodule.cnf
+
+[nodejs_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+default = default_sect
+# The fips section name should match the section name inside the
+# included fipsmodule.cnf.
+fips = fips_sect
+
+[default_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=yes 

src/dev/build/tasks/os_packages/docker_generator/run.ts

@@ -36,6 +36,7 @@ export async function runDockerGenerator(
     cloud?: boolean;
     serverless?: boolean;
     dockerBuildDate?: string;
+    fips?: boolean;
   }
 ) {
   let baseImageName = '';
@@ -47,6 +48,7 @@ export async function runDockerGenerator(
   if (flags.ironbank) imageFlavor += '-ironbank';
   if (flags.cloud) imageFlavor += '-cloud';
   if (flags.serverless) imageFlavor += '-serverless';
+  if (flags.fips) imageFlavor += '-fips';
 
   // General docker var config
   const license = 'Elastic License';
@@ -111,6 +113,7 @@ export async function runDockerGenerator(
     architecture: flags.architecture,
     revision: config.getBuildSha(),
     publicArtifactSubdomain,
+    fips: flags.fips,
   };
 
   type HostArchitectureToDocker = Record<string, string>;
@@ -148,6 +151,14 @@ export async function runDockerGenerator(
     dockerBuildDir
   );
 
+  // Copy fips related resources
+  if (flags.fips) {
+    await copyAll(
+      config.resolveFromRepo('src/dev/build/tasks/os_packages/docker_generator/resources/fips'),
+      dockerBuildDir
+    );
+  }
+
   // Build docker image into the target folder
   // In order to do this we just call the file we
   // created from the templates/build_docker_sh.template.js

src/dev/build/tasks/os_packages/docker_generator/template_context.ts

@@ -33,4 +33,5 @@ export interface TemplateContext {
   ironbank?: boolean;
   revision: string;
   architecture?: string;
+  fips?: boolean;
 }

src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile

@@ -61,7 +61,7 @@ EXPOSE 5601
 RUN for iter in {1..10}; do \
       {{packageManager}} update --setopt=tsflags=nodocs -y && \
       {{packageManager}} install --setopt=tsflags=nodocs -y \
-        fontconfig freetype shadow-utils nss findutils && \
+        fontconfig freetype shadow-utils nss findutils {{#fips}}perl make gcc tar {{/fips}}&& \
       {{packageManager}} clean all && exit_code=0 && break || exit_code=$? && echo "{{packageManager}} error: retry $iter in 10s" && \
       sleep 10; \
     done; \
@@ -112,6 +112,37 @@ COPY --from=builder --chown=1000:0 /usr/share/kibana /usr/share/kibana
 COPY --from=builder --chown=0:0 /opt /opt
 {{/cloud}}
 WORKDIR /usr/share/kibana
+{{#fips}}
+
+# OpenSSL requires specific versions that are FIPS certified. Further, the FIPS modules
+# need to be compiled on the machine to pass its own self validation on startup.
+#
+# See:
+# https://github.com/openssl/openssl/blob/openssl-3.0/README-FIPS.md
+# https://www.openssl.org/docs/man3.0/man7/fips_module.html
+
+# Ideally we would handle this in the builder step, but make is installing over the OS version
+# of OpenSSL and requires linking of many submodules.
+RUN set -e ; \
+ curl --retry 8 -S -L -O https://www.openssl.org/source/openssl-3.0.8.tar.gz ; \
+ curl --retry 8 -S -L -O https://www.openssl.org/source/openssl-3.0.8.tar.gz.sha256  ; \
+ echo "$(cat openssl-3.0.8.tar.gz.sha256) openssl-3.0.8.tar.gz" | sha256sum -c ; \
+ tar -zxf openssl-3.0.8.tar.gz  ; \
+ rm -rf openssl-3.0.8.tar*  ; \
+ cd /usr/share/kibana/openssl-3.0.8   ; \
+ ./Configure enable-fips  ; \
+ make -j $(nproc)  ; \
+ make install  ; \
+ ldconfig /usr/local/lib64/  ; \
+ chown -R 1000:0 /usr/share/kibana/openssl-3.0.8
+
+# Enable FIPS for Kibana only. In the future we can override OS wide with ENV OPENSSL_CONF
+RUN echo -e '\n--enable-fips' >> config/node.options
+RUN echo '--openssl-config=/usr/share/kibana/openssl-3.0.8/nodejs.cnf' >> config/node.options
+COPY --chown=1000:0 openssl/nodejs.cnf /usr/share/kibana/openssl-3.0.8/nodejs.cnf
+ENV OPENSSL_MODULES=/usr/local/lib64/ossl-modules
+
+{{/fips}}
 RUN ln -s /usr/share/kibana /opt/kibana
 
 {{! Please notify @elastic/kibana-security if you want to remove or change this environment variable. }}
@@ -127,7 +158,7 @@ COPY --chown=1000:0 config/serverless.es.yml /usr/share/kibana/config/serverless
 COPY --chown=1000:0 config/serverless.oblt.yml /usr/share/kibana/config/serverless.oblt.yml
 COPY --chown=1000:0 config/serverless.security.yml /usr/share/kibana/config/serverless.security.yml
 # Supportability enhancement: enable capturing heap snapshots. See https://nodejs.org/api/cli.html#--heapsnapshot-signalsignal
-RUN echo '\n--heapsnapshot-signal=SIGUSR2' >> config/node.options
+RUN echo -e '\n--heapsnapshot-signal=SIGUSR2' >> config/node.options
 RUN echo '--diagnostic-dir=./data' >> config/node.options
 {{/serverless}}
 {{^opensslLegacyProvider}}

src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts

@@ -19,7 +19,7 @@ function generator(options: TemplateContext) {
     packageManager: options.baseImage === 'ubi' ? 'microdnf' : 'apt-get',
     ubi: options.baseImage === 'ubi',
     ubuntu: options.baseImage === 'ubuntu',
-    opensslLegacyProvider: !(options.cloud || options.serverless),
+    opensslLegacyProvider: !(options.cloud || options.serverless || options.fips),
     ...options,
   });
 }

x-pack/plugins/profiling/server/lib/setup/README.md

@@ -26,7 +26,7 @@ Build and push a Kibana image with the latest changes.
 Choose a unique identifier for the build, then:
 
 ```
-node scripts/build --docker-images --skip-docker-ubi --skip-docker-ubuntu
+node scripts/build --docker-images --skip-docker-ubi --skip-docker-ubuntu --skip-docker-fips
 docker tag docker.elastic.co/kibana-ci/kibana-cloud:8.7.0-SNAPSHOT docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
 docker push docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
 ```