From 38c90ca8d47001002a4c6c9bf34858088068d8c6 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 19 May 2025 10:47:34 -0700 Subject: [PATCH] Restructure docker files for docker distributions (#127960) Restructures docker files for docker distributions - Put Dockerfiles in specific distro specific folders keeping "Dockerfile" naming convention - Allows better ide support - Allows easier renovate integration - Explicitly set base image in dockerfile - simplify renovate configuration - Cleanup DockerBase file to not contain ess fips base image information This lives now in the Dockerfile content directly * Workaround docker test issue * Fix labels for fips image --- .../gradle/internal/DockerBase.java | 13 +- distribution/docker/build.gradle | 4 +- .../cloud_ess_fips/Dockerfile} | 4 +- .../default/Dockerfile} | 0 .../src/docker/dockerfiles/wolfi/Dockerfile | 176 ++++++++++++++++++ .../packaging/util/docker/Docker.java | 5 +- renovate.json | 21 +-- 7 files changed, 193 insertions(+), 30 deletions(-) rename distribution/docker/src/docker/{Dockerfile.ess-fips => dockerfiles/cloud_ess_fips/Dockerfile} (97%) rename distribution/docker/src/docker/{Dockerfile.default => dockerfiles/default/Dockerfile} (100%) create mode 100644 distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 3ae5a4253133..abfefa8fe07f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -14,30 +14,27 @@ package org.elasticsearch.gradle.internal; */ public enum DockerBase { // "latest" here is intentional, since the image name specifies "9" - DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"), + DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "dockerfiles/default/Dockerfile"), // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"), // Chainguard based wolfi image with latest jdk - // This is usually updated via renovatebot - // spotless:off WOLFI( - "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a", + null, "-wolfi", "apk", - "Dockerfile" + "dockerfiles/wolfi/Dockerfile" ), - // spotless:on // Based on WOLFI above, with more extras. We don't set a base image because // we programmatically extend from the wolfi image. CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"), CLOUD_ESS_FIPS( - "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", + null, "-cloud-ess-fips", "apk", - "Dockerfile.ess-fips" + "dockerfiles/cloud_ess_fips/Dockerfile" ); private final String image; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index c1262a31c8d8..e993c4610145 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -482,8 +482,10 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { baseImages = [baseImage] buildArgs = buildArgsMap - } else { + } else if(base.image != null) { baseImages = [base.image] + } else { + baseImages = [] } Provider serviceProvider = GradleUtils.getBuildService( diff --git a/distribution/docker/src/docker/Dockerfile.ess-fips b/distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile similarity index 97% rename from distribution/docker/src/docker/Dockerfile.ess-fips rename to distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile index 27f03a40a056..cd190776da0d 100644 --- a/distribution/docker/src/docker/Dockerfile.ess-fips +++ b/distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile @@ -24,7 +24,7 @@ # Extract Elasticsearch artifact ################################################################################ -FROM ${base_image} AS builder +FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7 AS builder # Install required packages to extract the Elasticsearch distribution RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> @@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config # Add entrypoint ################################################################################ -FROM ${base_image} +FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && \n" + diff --git a/distribution/docker/src/docker/Dockerfile.default b/distribution/docker/src/docker/dockerfiles/default/Dockerfile similarity index 100% rename from distribution/docker/src/docker/Dockerfile.default rename to distribution/docker/src/docker/dockerfiles/default/Dockerfile diff --git a/distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile b/distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile new file mode 100644 index 000000000000..8d6e090dbdaa --- /dev/null +++ b/distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile @@ -0,0 +1,176 @@ +################################################################################ +# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile +# +# Beginning of multi stage Dockerfile +################################################################################ + +<% /* + This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes + have to be escaped in order for them to appear in the final Dockerfile. You + can also comment out blocks, like this one. See: + + https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html + + We use control-flow tags in this file to conditionally render the content. The + layout/presentation here has been adjusted so that it looks reasonable when rendered, + at the slight expense of how it looks here. + + Note that this file is also filtered to squash together newlines, so we can + add as many newlines here as necessary to improve legibility. +*/ %> + +################################################################################ +# Build stage 1 `builder`: +# Extract Elasticsearch artifact +################################################################################ + +FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a AS builder + +# Install required packages to extract the Elasticsearch distribution +RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> + +# `tini` is a tiny but valid init for containers. This is used to cleanly +# control how ES and any child processes are shut down. +# For wolfi we pick it from the blessed wolfi package registry. +# +# The tini GitHub page gives instructions for verifying the binary using +# gpg, but the keyservers are slow to return the key and this can fail the +# build. Instead, we check the binary against the published checksum. + +RUN mkdir /usr/share/elasticsearch +WORKDIR /usr/share/elasticsearch + + +# Fetch the appropriate Elasticsearch distribution for this architecture. +# Keep this command on one line - it is replaced with a `COPY` during local builds. +# It uses the `arch` shell command to fetch the correct distro for the build machine, +RUN curl --retry 10 -S -L --output /tmp/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\${arch}.tar.gz + +RUN tar -zxf /tmp/elasticsearch.tar.gz --strip-components=1 + +# The distribution includes a `config` directory, no need to create it +COPY ${config_dir}/elasticsearch.yml config/ +COPY ${config_dir}/log4j2.properties config/log4j2.docker.properties + +# 1. Configure the distribution for Docker +# 2. Create required directory +# 3. Move the distribution's default logging config aside +# 4. Move the generated docker logging config so that it is the default +# 5. Reset permissions on all directories +# 6. Reset permissions on all files +# 7. Make CLI tools executable +# 8. Make some directories writable. `bin` must be writable because +# plugins can install their own CLI utilities. +# 9. Make some files writable +RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\ + mkdir data && \\ + mv config/log4j2.properties config/log4j2.file.properties && \\ + mv config/log4j2.docker.properties config/log4j2.properties && \\ + find . -type d -exec chmod 0555 {} + && \\ + find . -type f -exec chmod 0444 {} + && \\ + chmod 0555 bin/* jdk/bin/* jdk/lib/jspawnhelper modules/x-pack-ml/platform/linux-*/bin/* && \\ + chmod 0775 bin config config/jvm.options.d data logs plugins && \\ + find config -type f -exec chmod 0664 {} + + +################################################################################ +# Build stage 2 (the actual Elasticsearch image): +# +# Copy elasticsearch from stage 1 +# Add entrypoint +################################################################################ + +FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a + +RUN <%= retry.loop(package_manager, + "export DEBIAN_FRONTEND=noninteractive && \n" + + " ${package_manager} update && \n" + + " ${package_manager} upgrade && \n" + + " ${package_manager} add --no-cache \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + + " rm -rf /var/cache/apk/* " + ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +# Optionally set Bash as the default shell in the container at runtime +CMD ["/bin/bash"] + +RUN groupadd -g 1000 elasticsearch && \ + adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \ + adduser elasticsearch root && \ + chown -R 0:0 /usr/share/elasticsearch + +ENV ELASTIC_CONTAINER=true + +WORKDIR /usr/share/elasticsearch + +COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch + +ENV PATH=/usr/share/elasticsearch/bin:\$PATH +ENV SHELL=/bin/bash +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +# 1. Sync the user and group permissions of /etc/passwd +# 2. Set correct permissions of the entrypoint +# 3. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks. +# We've already run this in previous layers so it ought to be a no-op. +# 4. Replace OpenJDK's built-in CA certificate keystore with the one from the OS +# vendor. The latter is superior in several ways. +# REF: https://github.com/elastic/elasticsearch-docker/issues/171 +# 5. Tighten up permissions on the ES home dir (the permissions of the contents are handled earlier) +# 6. You can't install plugins that include configuration when running as `elasticsearch` and the `config` +# dir is owned by `root`, because the installed tries to manipulate the permissions on the plugin's +# config directory. +RUN chmod g=u /etc/passwd && \\ + chmod 0555 /usr/local/bin/docker-entrypoint.sh && \\ + find / -xdev -perm -4000 -exec chmod ug-s {} + && \\ + chmod 0775 /usr/share/elasticsearch && \\ + chown elasticsearch bin config config/jvm.options.d data logs plugins + +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts + +EXPOSE 9200 9300 + + +LABEL org.label-schema.build-date="${build_date}" \\ + org.label-schema.license="${license}" \\ + org.label-schema.name="Elasticsearch" \\ + org.label-schema.schema-version="1.0" \\ + org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\ + org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.label-schema.vcs-ref="${git_revision}" \\ + org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\ + org.label-schema.vendor="Elastic" \\ + org.label-schema.version="${version}" \\ + org.opencontainers.image.created="${build_date}" \\ + org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.opencontainers.image.licenses="${license}" \\ + org.opencontainers.image.revision="${git_revision}" \\ + org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\ + org.opencontainers.image.title="Elasticsearch" \\ + org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ + org.opencontainers.image.vendor="Elastic" \\ + org.opencontainers.image.version="${version}" + +LABEL name="Elasticsearch" \\ + maintainer="infra@elastic.co" \\ + vendor="Elastic" \\ + version="${version}" \\ + release="1" \\ + summary="Elasticsearch" \\ + description="You know, for search." + +RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE + +# Our actual entrypoint is `tini`, a minimal but functional init program. It +# calls the entrypoint we provide, while correctly forwarding signals. +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] +# Dummy overridable parameter parsed by entrypoint +CMD ["eswrapper"] + +USER 1000:0 + +################################################################################ +# End of multi-stage Dockerfile +################################################################################ diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index ff1f3a931459..ab167d7663be 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -75,12 +75,15 @@ public class Docker { public static final int STARTUP_SLEEP_INTERVAL_MILLISECONDS = 1000; public static final int STARTUP_ATTEMPTS_MAX = 30; + /** + * The length of the command exceeds what we can use for COLUMNS so we use + * a workaround to find the process we're looking for + */ private static final String ELASTICSEARCH_FULL_CLASSNAME = "org.elasticsearch.bootstrap.Elasticsearch"; private static final String FIND_ELASTICSEARCH_PROCESS = "for pid in $(ps -eo pid,comm | grep java | awk '\\''{print $1}'\\''); " + "do cmdline=$(tr \"\\0\" \" \" < /proc/$pid/cmdline 2>/dev/null); [[ $cmdline == *" + ELASTICSEARCH_FULL_CLASSNAME + "* ]] && echo \"$pid: $cmdline\"; done"; - // The length of the command exceeds what we can use for COLUMNS so we use a pipe to detect the process we're looking for /** * Tracks the currently running Docker image. An earlier implementation used a fixed container name, diff --git a/renovate.json b/renovate.json index 0cd2d4b6b0e0..4ccbd8e59942 100644 --- a/renovate.json +++ b/renovate.json @@ -18,8 +18,7 @@ "9.0", "8.19", "8.18", - "8.17", - "7.17" + "8.17" ], "packageRules": [ { @@ -30,23 +29,9 @@ "docker" ], "matchPackageNames": [ - "/^docker.elastic.co/wolfi/chainguard-base$/" + "/^docker.elastic.co/wolfi/chainguard-base$/", + "/^docker.elastic.co/wolfi/chainguard-base-fips$/" ] } - ], - "customManagers": [ - { - "description": "Extract Wolfi images from elasticsearch DockerBase configuration", - "customType": "regex", - "fileMatch": [ - "build\\-tools\\-internal\\/src\\/main\\/java\\/org\\/elasticsearch\\/gradle\\/internal\\/DockerBase\\.java$" - ], - "matchStrings": [ - "\\s*\"?(?[^\\s:@\"]+)(?::(?[-a-zA-Z0-9.]+))?(?:@(?sha256:[a-zA-Z0-9]+))?\"?" - ], - "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}", - "autoReplaceStringTemplate": "{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", - "datasourceTemplate": "docker" - } ] }