diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index 8bec706bb758..ffc1350aceab 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -3,65 +3,9 @@ config: steps: - group: packaging-tests-unix steps: - - label: "{{matrix.image}} / docker / packaging-tests-unix" - key: "packaging-tests-unix-docker" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess - timeout_in_minutes: 300 - matrix: - setup: - image: - - debian-11 - - debian-12 - - opensuse-leap-15 - - oraclelinux-7 - - oraclelinux-8 - - sles-12 - - sles-15 - - ubuntu-1804 - - ubuntu-2004 - - ubuntu-2204 - - rocky-8 - - rocky-9 - - rhel-7 - - rhel-8 - - rhel-9 - - almalinux-8 - agents: - provider: gcp - image: family/elasticsearch-{{matrix.image}} - diskSizeGb: 350 - machineType: custom-16-32768 - - label: "{{matrix.image}} / packages / packaging-tests-unix" - key: "packaging-tests-unix-packages" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.packages - timeout_in_minutes: 300 - matrix: - setup: - image: - - debian-11 - - debian-12 - - opensuse-leap-15 - - oraclelinux-7 - - oraclelinux-8 - - sles-12 - - sles-15 - - ubuntu-1804 - - ubuntu-2004 - - ubuntu-2204 - - rocky-8 - - rocky-9 - - rhel-7 - - rhel-8 - - rhel-9 - - almalinux-8 - agents: - provider: gcp - image: family/elasticsearch-{{matrix.image}} - diskSizeGb: 350 - machineType: custom-16-32768 - - label: "{{matrix.image}} / archives / packaging-tests-unix" - key: "packaging-tests-unix-archives" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.archives + - label: "{{matrix.image}} / {{matrix.PACKAGING_TASK}} / packaging-tests-unix" + key: "packaging-tests-unix" + command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.{{matrix.PACKAGING_TASK}} timeout_in_minutes: 300 matrix: setup: @@ -82,6 +26,11 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + PACKAGING_TASK: + - docker + - docker-cloud-ess + - packages + - archives agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 3e0a47a8f453..bf901fef9045 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -13,10 +13,8 @@ package org.elasticsearch.gradle.internal; * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch. */ public enum DockerBase { - DEFAULT("ubuntu:20.04", "", "apt-get"), - // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi", "microdnf"), + DEFAULT("docker.elastic.co/ubi8/ubi-minimal:latest", "", "microdnf"), // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"), diff --git a/distribution/docker/README.md b/distribution/docker/README.md index 49facab461ed..9438b4f1e82c 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -3,8 +3,7 @@ The ES build can generate several types of Docker image. These are enumerated in the [DockerBase] enum. - * Default - this is what most people use, and is based on Ubuntu - * UBI - the same as the default image, but based upon [RedHat's UBI + * Default - this is what most people use, and is based on [RedHat's UBI images][ubi], specifically their minimal flavour. * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev) * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins @@ -23,14 +22,7 @@ the [DockerBase] enum. software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is another UBI build, this time on the regular UBI image, with extra hardening. See below for more details. - * Cloud - this is mostly the same as the default image, with some notable differences: - * `filebeat` and `metricbeat` are included - * `wget` is included - * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is - `/app/elasticsearch.sh`. In normal use this file would be bind-mounted - in, but the image ships a stub version of this file so that the image - can still be tested. -The long-term goal is for both Cloud images to be retired in favour of the +The long-term goal is for Cloud ESS image to be retired in favour of the default image. diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index d73f9c395f15..f5b94fb9dfd9 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -527,9 +527,7 @@ subprojects { Project subProject -> final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 DockerBase base = DockerBase.DEFAULT - if (subProject.name.contains('ubi-')) { - base = DockerBase.UBI - } else if (subProject.name.contains('ironbank-')) { + if (subProject.name.contains('ironbank-')) { base = DockerBase.IRON_BANK } else if (subProject.name.contains('cloud-ess-')) { base = DockerBase.CLOUD_ESS @@ -538,11 +536,11 @@ subprojects { Project subProject -> } final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' - final String extension = base == DockerBase.UBI ? 'ubi.tar' : + final String extension = (base == DockerBase.IRON_BANK ? 'ironbank.tar' : - (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : - (base == DockerBase.WOLFI ? 'wolfi.tar' : - 'docker.tar'))) + (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : + (base == DockerBase.WOLFI ? 'wolfi.tar' : + 'docker.tar'))) final String artifactName = "elasticsearch${arch}${base.suffix}_test" final String exportTaskName = taskName("export", architecture, base, 'DockerImage') diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index fd2516f2fdc9..6cb030565d9d 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -41,9 +41,7 @@ RUN chmod 0555 /bin/tini <% } else { %> # Install required packages to extract the Elasticsearch distribution -<% if (docker_base == 'default' || docker_base == 'cloud') { %> -RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl ") %> -<% } else if (docker_base == "wolfi") { %> +<% if (docker_base == "wolfi") { %> RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> <% } else { %> RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %> @@ -117,27 +115,6 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas chmod 0775 bin config config/jvm.options.d data logs plugins && \\ find config -type f -exec chmod 0664 {} + -<% if (docker_base == "cloud") { %> -COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ -RUN set -eux ; \\ - for beat in filebeat metricbeat ; do \\ - if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ - echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ - echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - mkdir -p /opt/\$beat ; \\ - tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ - done - -# Add plugins infrastructure -RUN mkdir -p /opt/plugins/archive -RUN chmod -R 0555 /opt/plugins -<% } %> - ################################################################################ # Build stage 2 (the actual Elasticsearch image): # @@ -173,21 +150,6 @@ SHELL ["/bin/bash", "-c"] # Optionally set Bash as the default shell in the container at runtime CMD ["/bin/bash"] -<% } else if (docker_base == "default" || docker_base == "cloud") { %> - -# Change default shell to bash, then install required packages with retries. -RUN yes no | dpkg-reconfigure dash && \\ - <%= retry.loop( - package_manager, - "export DEBIAN_FRONTEND=noninteractive && \n" + - " ${package_manager} update && \n" + - " ${package_manager} upgrade -y && \n" + - " ${package_manager} install -y --no-install-recommends \n" + - " ca-certificates curl netcat p11-kit unzip zip ${docker_base == 'cloud' ? 'wget' : '' } && \n" + - " ${package_manager} clean && \n" + - " rm -rf /var/lib/apt/lists/*" - ) %> - <% } else { %> RUN <%= retry.loop( @@ -201,12 +163,7 @@ RUN <%= retry.loop( <% } %> -<% if (docker_base == "default" || docker_base == "cloud") { %> -RUN groupadd -g 1000 elasticsearch && \\ - adduser --uid 1000 --gid 1000 --home /usr/share/elasticsearch elasticsearch && \\ - adduser elasticsearch root && \\ - chown -R 0:0 /usr/share/elasticsearch -<% } else if (docker_base == "wolfi") { %> +<% if (docker_base == "wolfi") { %> RUN groupadd -g 1000 elasticsearch && \ adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \ adduser elasticsearch root && \ @@ -226,10 +183,6 @@ COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearc COPY --from=builder --chown=0:0 /bin/tini /bin/tini <% } %> -<% if (docker_base == 'cloud') { %> -COPY --from=builder --chown=0:0 /opt /opt -<% } %> - ENV PATH /usr/share/elasticsearch/bin:\$PATH ENV SHELL /bin/bash COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh @@ -251,12 +204,7 @@ RUN chmod g=u /etc/passwd && \\ chmod 0775 /usr/share/elasticsearch && \\ chown elasticsearch bin config config/jvm.options.d data logs plugins -<% if (docker_base == 'default' || docker_base == 'cloud') { %> -# Update "cacerts" bundle to use Ubuntu's CA certificates (and make sure it -# stays up-to-date with changes to Ubuntu's store) -COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk -RUN /etc/ca-certificates/update.d/docker-openjdk -<% } else if (docker_base == 'wolfi') { %> +<% if (docker_base == 'wolfi') { %> RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts @@ -284,9 +232,7 @@ LABEL org.label-schema.build-date="${build_date}" \\ org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" -<% } %> -<% if (docker_base == 'ubi') { %> LABEL name="Elasticsearch" \\ maintainer="infra@elastic.co" \\ vendor="Elastic" \\ @@ -296,21 +242,12 @@ LABEL name="Elasticsearch" \\ description="You know, for search." <% } %> -<% if (docker_base == 'ubi') { %> -RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE -<% } else if (docker_base == 'iron_bank') { %> RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE +<% if (docker_base == 'iron_bank') { %> COPY LICENSE /licenses/LICENSE.addendum <% } %> -<% if (docker_base == "cloud") { %> -ENTRYPOINT ["/bin/tini", "--"] -CMD ["/app/elasticsearch.sh"] -# Generate a stub command that will be overwritten at runtime -RUN mkdir /app && \\ - echo -e '#!/bin/bash\\nexec /usr/local/bin/docker-entrypoint.sh eswrapper' > /app/elasticsearch.sh && \\ - chmod 0555 /app/elasticsearch.sh -<% } else if (docker_base == "wolfi") { %> +<% if (docker_base == "wolfi") { %> # Our actual entrypoint is `tini`, a minimal but functional init program. It # calls the entrypoint we provide, while correctly forwarding signals. ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] diff --git a/distribution/docker/ubi-docker-aarch64-export/build.gradle b/distribution/docker/ubi-docker-aarch64-export/build.gradle deleted file mode 100644 index 537b5a093683..000000000000 --- a/distribution/docker/ubi-docker-aarch64-export/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// export is done in the parent project. diff --git a/distribution/docker/ubi-docker-export/build.gradle b/distribution/docker/ubi-docker-export/build.gradle deleted file mode 100644 index 537b5a093683..000000000000 --- a/distribution/docker/ubi-docker-export/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// export is done in the parent project. diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 8cb8354eb5d7..3ad4c247a8b9 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -96,11 +96,10 @@ import static org.junit.Assume.assumeTrue; /** * This class tests the Elasticsearch Docker images. We have several: * */ @ThreadLeakFilters(defaultFilters = true, filters = { HttpClientThreadsFilter.class }) @@ -383,15 +382,14 @@ public class DockerTests extends PackagingTestCase { public void test040JavaUsesTheOsProvidedKeystore() { final String path = sh.run("realpath jdk/lib/security/cacerts").stdout(); - if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { + if (distribution.packaging == Packaging.DOCKER || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { - // Whereas on other images, it's a real file so the real path is the same - assertThat(path, equalTo("/usr/share/elasticsearch/jdk/lib/security/cacerts")); + fail("Unknown distribution: " + distribution.packaging); } } @@ -1126,25 +1124,25 @@ public class DockerTests extends PackagingTestCase { } /** - * Check that the UBI images has the correct license information in the correct place. + * Check that the Docker images have the correct license information in the correct place. */ - public void test200UbiImagesHaveLicenseDirectory() { - assumeTrue(distribution.packaging == Packaging.DOCKER_UBI); + public void test200ImagesHaveLicenseDirectory() { + assumeTrue(distribution.packaging != Packaging.DOCKER_IRON_BANK); final String[] files = sh.run("find /licenses -type f").stdout().split("\n"); assertThat(files, arrayContaining("/licenses/LICENSE")); // UBI image doesn't contain `diff` - final String ubiLicense = sh.run("cat /licenses/LICENSE").stdout(); + final String imageLicense = sh.run("cat /licenses/LICENSE").stdout(); final String distroLicense = sh.run("cat /usr/share/elasticsearch/LICENSE.txt").stdout(); - assertThat(ubiLicense, equalTo(distroLicense)); + assertThat(imageLicense, equalTo(distroLicense)); } /** - * Check that the UBI image has the expected labels + * Check that the images has the expected labels */ - public void test210UbiLabels() throws Exception { - assumeTrue(distribution.packaging == Packaging.DOCKER_UBI); + public void test210Labels() throws Exception { + assumeTrue(distribution.packaging != Packaging.DOCKER_IRON_BANK); final Map labels = getImageLabels(distribution); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 02e1ce35764c..a47dd0e57642 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,7 @@ public class KeystoreManagementTests extends PackagingTestCase { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660)); + case DOCKER, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660)); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index b4a00ca56924..a157cc84e624 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ public abstract class PackagingTestCase extends Assert { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { + case DOCKER, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -333,7 +333,6 @@ public abstract class PackagingTestCase extends Assert { case RPM: return Packages.runElasticsearchStartCommand(sh); case DOCKER: - case DOCKER_UBI: case DOCKER_IRON_BANK: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: @@ -355,7 +354,6 @@ public abstract class PackagingTestCase extends Assert { Packages.stopElasticsearch(sh); break; case DOCKER: - case DOCKER_UBI: case DOCKER_IRON_BANK: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: @@ -371,7 +369,7 @@ public abstract class PackagingTestCase extends Assert { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 11b832438463..55c59db6219d 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -29,8 +29,6 @@ public class Distribution { this.packaging = Packaging.TAR; } else if (filename.endsWith(".docker.tar")) { this.packaging = Packaging.DOCKER; - } else if (filename.endsWith(".ubi.tar")) { - this.packaging = Packaging.DOCKER_UBI; } else if (filename.endsWith(".ironbank.tar")) { this.packaging = Packaging.DOCKER_IRON_BANK; } else if (filename.endsWith(".cloud-ess.tar")) { @@ -61,7 +59,7 @@ public class Distribution { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; + case DOCKER, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -73,7 +71,6 @@ public class Distribution { DEB(".deb", Platforms.isDPKG()), RPM(".rpm", Platforms.isRPM()), DOCKER(".docker.tar", Platforms.isDocker()), - DOCKER_UBI(".ubi.tar", Platforms.isDocker()), DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index e3eac23d3ecc..5dc47993072a 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -163,7 +163,6 @@ public class DockerRun { public static String getImageName(Distribution distribution) { String suffix = switch (distribution.packaging) { case DOCKER -> ""; - case DOCKER_UBI -> "-ubi"; case DOCKER_IRON_BANK -> "-ironbank"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; case DOCKER_WOLFI -> "-wolfi"; diff --git a/settings.gradle b/settings.gradle index d04d45bffc3a..333f8272447c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -66,8 +66,6 @@ List projects = [ 'distribution:docker:docker-export', 'distribution:docker:ironbank-docker-aarch64-export', 'distribution:docker:ironbank-docker-export', - 'distribution:docker:ubi-docker-aarch64-export', - 'distribution:docker:ubi-docker-export', 'distribution:docker:wolfi-docker-aarch64-export', 'distribution:docker:wolfi-docker-export', 'distribution:packages:aarch64-deb',