Support building Iron Bank Docker context (#64336)

This PR adds support for building a Docker context for Iron Bank. It doesn't actually build the image - we could add that at a later stage, but this is an attempt to automate at least some of the process. Iron Bank is a lot like our UBI build, except it uses a hardened version of the full UBI image, not the minimal UBI image. They have particular requirements around how the Docker context should be arranged. The Docker build cannot fetch its own artefacts, but instead the context provides a descriptor that locates what is needed for the build. I also added a filter so that after performing expansions on the `Dockerfile`, we squash long runs on newlines together. This makes the output cleaner, while allowing us to break up the unprocessed `Dockerfile` for clarity.
2025-04-24 15:17:30 -04:00 · 2020-11-16 12:21:35 +00:00 · 2020-11-16 12:21:35 +00:00 · e07adb75c9
commit e07adb75c9
parent 988583efb8
9 changed files with 191 additions and 26 deletions
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@ -6,6 +6,9 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.docker.DockerBuildTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
+
+import java.nio.file.Path
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
@ -46,6 +49,15 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool

  final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz"

+  String buildArgs = '#'
+  if (base == DockerBase.IRON_BANK) {
+    buildArgs = """
+ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082
+ARG BASE_IMAGE=redhat/ubi/ubi8
+ARG BASE_TAG=8.2
+"""
+  }
+
  /* Both the following Dockerfile commands put the resulting artifact at
   * the same location, regardless of classifier, so that the commands that
   * follow in the Dockerfile don't have to know about the runtime
@ -61,23 +73,40 @@ RUN curl --retry 8 -S -L \\
 """.trim()
  }

+  def (major,minor) = VersionProperties.elasticsearch.split("\\.")
+
  return [
    'base_image'          : base.getImage(),
+    'bin_dir'             : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
+    'build_args'          : buildArgs,
    'build_date'          : BuildParams.buildDate,
+    'config_dir'          : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
    'git_revision'        : BuildParams.gitRevision,
    'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
    'package_manager'     : base == DockerBase.UBI ? 'microdnf' : 'yum',
    'source_elasticsearch': sourceElasticsearch,
    'docker_base'         : base.name().toLowerCase(),
-    'version'             : VersionProperties.elasticsearch
+    'version'             : VersionProperties.elasticsearch,
+    'major_minor_version' : "${major}.${minor}"
  ]
 }

+/**
+ * This filter squashes long runs of newlines so that the output
+ * is a little more aesthetically pleasing.
+ */
+class SquashNewlinesFilter extends FilterReader {
+  SquashNewlinesFilter(Reader input) {
+    super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n")))
+  }
+}
+
 private static String buildPath(Architecture architecture, boolean oss, DockerBase base) {
  return 'build/' +
    (architecture == Architecture.AARCH64 ? 'aarch64-' : '') +
    (oss ? 'oss-' : '') +
    (base == DockerBase.UBI ? 'ubi-' : '') +
+    (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) +
    'docker'
 }

@ -85,34 +114,46 @@ private static String taskName(String prefix, Architecture architecture, boolean
  return prefix +
    (architecture == Architecture.AARCH64 ? 'Aarch64' : '') +
    (oss ? 'Oss' : '') +
-    (base == DockerBase.UBI ? 'Ubi' : '') +
+    (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) +
    suffix
 }

 project.ext {
  dockerBuildContext = { Architecture architecture, boolean oss, DockerBase base, boolean local ->
    copySpec {
-      into('bin') {
-        from project.projectDir.toPath().resolve("src/docker/bin")
-      }
+      final Map<String,String> varExpansions = expansions(architecture, oss, base, local)
+      final Path projectDir = project.projectDir.toPath()

-      into('config') {
-        /*
-         * The OSS and default distributions have different configurations, therefore we want to allow overriding the default configuration
-         * from files in the 'oss' sub-directory. We don't want the 'oss' sub-directory to appear in the final build context, however.
-         */
-        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-        from(project.projectDir.toPath().resolve("src/docker/config")) {
-          exclude 'oss'
+      if (base == DockerBase.IRON_BANK) {
+        into('scripts') {
+          from projectDir.resolve("src/docker/bin")
+          from(projectDir.resolve("src/docker/config")) {
+            exclude '**/oss'
+          }
        }
-        if (oss) {
-          // Overlay the config file
-          from project.projectDir.toPath().resolve("src/docker/config/oss")
+        from(projectDir.resolve("src/docker/iron_bank")) {
+          expand(varExpansions)
+        }
+      } else {
+        into('bin') {
+          from projectDir.resolve("src/docker/bin")
+        }
+
+        into('config') {
+          // The OSS and default distribution can have different configuration, therefore we want to
+          // allow overriding the default configuration by creating config files in oss or default
+          // build-context sub-modules.
+          duplicatesStrategy = DuplicatesStrategy.INCLUDE
+          from projectDir.resolve("src/docker/config")
+          if (oss) {
+            from projectDir.resolve("src/docker/config/oss")
+          }
        }
      }

      from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
-        expand(expansions(architecture, oss, base, local))
+        expand(varExpansions)
+        filter SquashNewlinesFilter
      }
    }
  }
@ -324,6 +365,8 @@ subprojects { Project subProject ->

    final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64
    final boolean oss = subProject.name.contains('oss-')
+    // We can ignore Iron Bank at the moment as we don't
+    // build those images ourselves.
    final DockerBase base = subProject.name.contains('ubi-') ? DockerBase.UBI : DockerBase.CENTOS

    final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
--- a/distribution/docker/ironbank-docker-build-context/build.gradle
+++ b/distribution/docker/ironbank-docker-build-context/build.gradle
@ -0,0 +1,14 @@
+import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.DockerBase
+
+apply plugin: 'base'
+
+tasks.register("buildIronBankDockerBuildContext", Tar) {
+  archiveExtension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-ironbank"
+  // We always treat Iron Bank builds as local, because that is how they
+  // are built
+  with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true)
+}
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@ -3,6 +3,7 @@
 #
 # Beginning of multi stage Dockerfile
 ################################################################################
+
 <% /*
  This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
  have to be escaped in order for them to appear in the final Dockerfile. You
@ -13,13 +14,16 @@
  We use control-flow tags in this file to conditionally render the content. The
  layout/presentation here has been adjusted so that it looks reasonable when rendered,
  at the slight expense of how it looks here.
+
+  Note that this file is also filtered to squash together newlines, so we can
+  add as many newlines here as necessary to improve legibility.
 */ %>
+
 <% if (docker_base == "ubi") { %>
 ################################################################################
 # Build stage 0 `builder`:
 # Extract Elasticsearch artifact
 ################################################################################
-
 FROM ${base_image} AS builder

 # Install required packages to extract the Elasticsearch distribution
@ -44,7 +48,21 @@ RUN set -eux ; \\
    rm \${tini_bin}.sha256sum ; \\
    mv \${tini_bin} /bin/tini ; \\
    chmod +x /bin/tini
+
+<% } else if (docker_base == 'iron_bank') { %>
+${build_args}
+
+FROM ${base_image} AS builder
+
+# `tini` is a tiny but valid init for containers. This is used to cleanly
+# control how ES and any child processes are shut down.
+COPY tini /bin/tini
+RUN chmod 0755 /bin/tini
+
 <% } else { %>
+
+<% /* CentOS builds are actaully a custom base image with a minimal set of dependencies */ %>
+
 ################################################################################
 # Stage 1. Build curl statically. Installing it from RPM on CentOS pulls in too
 # many dependencies.
@ -194,6 +212,7 @@ COPY --from=curl /work/curl /rootfs/usr/bin/curl
 # Step 3. Fetch the Elasticsearch distribution and configure it for Docker
 ################################################################################
 FROM ${base_image} AS builder
+
 <% } %>

 RUN mkdir /usr/share/elasticsearch
@ -202,16 +221,17 @@ WORKDIR /usr/share/elasticsearch
 # Fetch the appropriate Elasticsearch distribution for this architecture
 ${source_elasticsearch}

-RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1
+RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1

 # Configure the distribution for Docker
 RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config config/jvm.options.d data logs
+RUN mkdir -p config config/jvm.options.d data logs plugins
 RUN chmod 0775 config config/jvm.options.d data logs plugins
-COPY config/elasticsearch.yml config/log4j2.properties config/
+COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties

-<% if (docker_base == "ubi") { %>
+<% if (docker_base == "ubi" || docker_base == "iron_bank") { %>
+
 ################################################################################
 # Build stage 1 (the actual Elasticsearch image):
 #
@ -221,6 +241,8 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties

 FROM ${base_image}

+<% if (docker_base == "ubi") { %>
+
 RUN for iter in {1..10}; do \\
      ${package_manager} update --setopt=tsflags=nodocs -y && \\
      ${package_manager} install --setopt=tsflags=nodocs -y \\
@ -231,11 +253,26 @@ RUN for iter in {1..10}; do \\
    done; \\
    (exit \$exit_code)

+%> } else { %>
+
+<%
+/* Reviews of the Iron Bank Dockerfile said that they preferred simpler */
+/* scripting so this version doesn't have the retry loop featured above. */
+%>
+RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\
+    ${package_manager} install --setopt=tsflags=nodocs -y \\
+      nc shadow-utils zip unzip && \\
+    ${package_manager} clean all
+
+<% } %>
+
 RUN groupadd -g 1000 elasticsearch && \\
    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
    chmod 0775 /usr/share/elasticsearch && \\
    chown -R 1000:0 /usr/share/elasticsearch
+
 <% } else { %>
+
 ################################################################################
 # Stage 4. Build the final image, using the rootfs above as the basis, and
 # copying in the Elasticsearch distribution
@ -250,13 +287,15 @@ RUN addgroup -g 1000 elasticsearch && \\
    addgroup elasticsearch root && \\
    chmod 0775 /usr/share/elasticsearch && \\
    chgrp 0 /usr/share/elasticsearch
+
 <% } %>

 ENV ELASTIC_CONTAINER true

 WORKDIR /usr/share/elasticsearch
 COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch
-<% if (docker_base == "ubi") { %>
+
+<% if (docker_base == "ubi" || docker_base == "iron_bank") { %>
 COPY --from=builder --chown=0:0 /bin/tini /bin/tini
 <% } %>

@ -267,7 +306,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk

 ENV PATH /usr/share/elasticsearch/bin:\$PATH

-COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh

 # 1. The JDK's directories' permissions don't allow `java` to be executed under a different
 #    group to the default. Fix this.
@ -303,7 +342,8 @@ LABEL org.label-schema.build-date="${build_date}" \\
  org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
  org.opencontainers.image.vendor="Elastic" \\
  org.opencontainers.image.version="${version}"
-<% if (docker_base == 'ubi') { %>
+
+<% if (docker_base == 'ubi' || docker_base == 'iron_bank') { %>
 LABEL name="Elasticsearch" \\
  maintainer="infra@elastic.co" \\
  vendor="Elastic" \\
@ -324,6 +364,10 @@ ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
 # Dummy overridable parameter parsed by entrypoint
 CMD ["eswrapper"]

+<% if (docker_base == 'iron_bank') { %>
+HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1
+<% } %>
+
 ################################################################################
 # End of multi-stage Dockerfile
 ################################################################################
--- a/distribution/docker/src/docker/iron_bank/.gitignore
+++ b/distribution/docker/src/docker/iron_bank/.gitignore
@ -0,0 +1,2 @@
+# Ignore any locally downloaded or dropped releases
+*.tar.gz
--- a/distribution/docker/src/docker/iron_bank/Jenkinsfile
+++ b/distribution/docker/src/docker/iron_bank/Jenkinsfile
@ -0,0 +1,2 @@
+@Library('DCCSCR@master') _
+dccscrPipeline(version: '${version}')
--- a/distribution/docker/src/docker/iron_bank/README.md
+++ b/distribution/docker/src/docker/iron_bank/README.md
@ -0,0 +1,37 @@
+# Elasticsearch
+
+**Elasticsearch** is a distributed, RESTful search and analytics engine capable of
+solving a growing number of use cases. As the heart of the Elastic Stack, it
+centrally stores your data so you can discover the expected and uncover the
+unexpected.
+
+For more information about Elasticsearch, please visit
+https://www.elastic.co/products/elasticsearch.
+
+### Installation instructions
+
+Please follow the documentation on [how to install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html).
+
+### Where to file issues and PRs
+
+- [Issues](https://github.com/elastic/elasticsearch/issues)
+- [PRs](https://github.com/elastic/elasticsearch/pulls)
+
+### Where to get help
+
+- [Elasticsearch Discuss Forums](https://discuss.elastic.co/c/elasticsearch)
+- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/index.html)
+
+### Still need help?
+
+You can learn more about the Elastic Community and also understand how to get more help
+visiting [Elastic Community](https://www.elastic.co/community).
+
+
+This software is governed by the [Elastic
+License](https://github.com/elastic/elasticsearch/blob/${major_minor_version}/licenses/ELASTIC-LICENSE.txt),
+and includes the full set of [free
+features](https://www.elastic.co/subscriptions).
+
+View the detailed release notes
+[here](https://www.elastic.co/guide/en/elasticsearch/reference/${major_minor_version}/es-release-notes.html).
--- a/distribution/docker/src/docker/iron_bank/download.json
+++ b/distribution/docker/src/docker/iron_bank/download.json
@ -0,0 +1,20 @@
+{
+  "resources": [
+    {
+      "url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-x86_64.tar.gz",
+      "filename": "elasticsearch-${version}-linux-x86_64.tar.gz",
+      "validation": {
+        "type": "sha512",
+        "value": "<insert hash here>"
+      }
+    },
+    {
+      "url": "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64",
+      "filename": "tini",
+      "validation": {
+        "type": "sha256",
+        "value": "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"
+      }
+    }
+  ]
+}