[build/docker] Add support for centos ARM builds (#84831)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tyler Smalley <tyler.smalley@elastic.co>

src/dev/build/args.test.ts

@@ -30,8 +30,9 @@ it('build default and oss dist for current platform, without packages, by defaul
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": false,
-        "createDockerPackage": false,
-        "createDockerUbiPackage": false,
+        "createDockerCentOS": false,
+        "createDockerContexts": false,
+        "createDockerUBI": false,
         "createRpmPackage": false,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -53,8 +54,9 @@ it('builds packages if --all-platforms is passed', () => {
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": true,
-        "createDockerPackage": true,
-        "createDockerUbiPackage": true,
+        "createDockerCentOS": true,
+        "createDockerContexts": true,
+        "createDockerUBI": true,
         "createRpmPackage": true,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -76,8 +78,9 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": false,
-        "createDockerPackage": false,
-        "createDockerUbiPackage": false,
+        "createDockerCentOS": false,
+        "createDockerContexts": false,
+        "createDockerUBI": false,
         "createRpmPackage": true,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -99,8 +102,9 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": true,
-        "createDockerPackage": false,
-        "createDockerUbiPackage": false,
+        "createDockerCentOS": false,
+        "createDockerContexts": false,
+        "createDockerUBI": false,
         "createRpmPackage": false,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -115,7 +119,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
 });
 
 it('limits packages if --docker passed with --all-platforms', () => {
-  expect(readCliArgs(['node', 'scripts/build', '--all-platforms', '--docker']))
+  expect(readCliArgs(['node', 'scripts/build', '--all-platforms', '--docker-images']))
     .toMatchInlineSnapshot(`
     Object {
       "buildOptions": Object {
@@ -123,8 +127,9 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": false,
-        "createDockerPackage": true,
-        "createDockerUbiPackage": true,
+        "createDockerCentOS": true,
+        "createDockerContexts": false,
+        "createDockerUBI": true,
         "createRpmPackage": false,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -139,16 +144,24 @@ it('limits packages if --docker passed with --all-platforms', () => {
 });
 
 it('limits packages if --docker passed with --skip-docker-ubi and --all-platforms', () => {
-  expect(readCliArgs(['node', 'scripts/build', '--all-platforms', '--docker', '--skip-docker-ubi']))
-    .toMatchInlineSnapshot(`
+  expect(
+    readCliArgs([
+      'node',
+      'scripts/build',
+      '--all-platforms',
+      '--docker-images',
+      '--skip-docker-ubi',
+    ])
+  ).toMatchInlineSnapshot(`
     Object {
       "buildOptions": Object {
         "buildDefaultDist": true,
         "buildOssDist": true,
         "createArchives": true,
         "createDebPackage": false,
-        "createDockerPackage": true,
-        "createDockerUbiPackage": false,
+        "createDockerCentOS": true,
+        "createDockerContexts": false,
+        "createDockerUBI": false,
         "createRpmPackage": false,
         "downloadFreshNode": true,
         "isRelease": false,
@@ -161,3 +174,28 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
     }
   `);
 });
+
+it('limits packages if --all-platforms passed with --skip-docker-centos', () => {
+  expect(readCliArgs(['node', 'scripts/build', '--all-platforms', '--skip-docker-centos']))
+    .toMatchInlineSnapshot(`
+    Object {
+      "buildOptions": Object {
+        "buildDefaultDist": true,
+        "buildOssDist": true,
+        "createArchives": true,
+        "createDebPackage": true,
+        "createDockerCentOS": false,
+        "createDockerContexts": true,
+        "createDockerUBI": true,
+        "createRpmPackage": true,
+        "downloadFreshNode": true,
+        "isRelease": false,
+        "targetAllPlatforms": true,
+        "versionQualifier": "",
+      },
+      "log": <ToolingLog>,
+      "showHelp": false,
+      "unknownFlags": Array [],
+    }
+  `);
+});

src/dev/build/args.ts

@@ -21,8 +21,10 @@ export function readCliArgs(argv: string[]) {
       'skip-os-packages',
       'rpm',
       'deb',
-      'docker',
+      'docker-images',
+      'docker-contexts',
       'skip-docker-ubi',
+      'skip-docker-centos',
       'release',
       'skip-node-download',
       'verbose',
@@ -42,7 +44,8 @@ export function readCliArgs(argv: string[]) {
       debug: true,
       rpm: null,
       deb: null,
-      docker: null,
+      'docker-images': null,
+      'docker-contexts': null,
       oss: null,
       'version-qualifier': '',
     },
@@ -69,7 +72,7 @@ export function readCliArgs(argv: string[]) {
 
   // In order to build a docker image we always need
   // to generate all the platforms
-  if (flags.docker) {
+  if (flags['docker-images'] || flags['docker-contexts']) {
     flags['all-platforms'] = true;
   }
 
@@ -79,7 +82,12 @@ export function readCliArgs(argv: string[]) {
     }
 
     // build all if no flags specified
-    if (flags.rpm === null && flags.deb === null && flags.docker === null) {
+    if (
+      flags.rpm === null &&
+      flags.deb === null &&
+      flags['docker-images'] === null &&
+      flags['docker-contexts'] === null
+    ) {
       return true;
     }
 
@@ -95,8 +103,10 @@ export function readCliArgs(argv: string[]) {
     createArchives: !Boolean(flags['skip-archives']),
     createRpmPackage: isOsPackageDesired('rpm'),
     createDebPackage: isOsPackageDesired('deb'),
-    createDockerPackage: isOsPackageDesired('docker'),
-    createDockerUbiPackage: isOsPackageDesired('docker') && !Boolean(flags['skip-docker-ubi']),
+    createDockerCentOS:
+      isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-centos']),
+    createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']),
+    createDockerContexts: isOsPackageDesired('docker-contexts'),
     targetAllPlatforms: Boolean(flags['all-platforms']),
   };
 

src/dev/build/build_distributables.ts

@@ -19,8 +19,9 @@ export interface BuildOptions {
   createArchives: boolean;
   createRpmPackage: boolean;
   createDebPackage: boolean;
-  createDockerPackage: boolean;
-  createDockerUbiPackage: boolean;
+  createDockerUBI: boolean;
+  createDockerCentOS: boolean;
+  createDockerContexts: boolean;
   versionQualifier: string | undefined;
   targetAllPlatforms: boolean;
 }
@@ -95,12 +96,19 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     // control w/ --rpm or --skip-os-packages
     await run(Tasks.CreateRpmPackage);
   }
-  if (options.createDockerPackage) {
-    // control w/ --docker or --skip-docker-ubi or --skip-os-packages
-    await run(Tasks.CreateDockerPackage);
-    if (options.createDockerUbiPackage) {
-      await run(Tasks.CreateDockerUbiPackage);
-    }
+  if (options.createDockerUBI) {
+    // control w/ --docker-images or --skip-docker-ubi or --skip-os-packages
+    await run(Tasks.CreateDockerUBI);
+  }
+
+  if (options.createDockerCentOS) {
+    // control w/ --docker-images or --skip-docker-centos or --skip-os-packages
+    await run(Tasks.CreateDockerCentOS);
+  }
+
+  if (options.createDockerContexts) {
+    // control w/ --docker-contexts or --skip-os-packages
+    await run(Tasks.CreateDockerContexts);
   }
 
   /**

src/dev/build/cli.ts

@@ -38,10 +38,12 @@ if (showHelp) {
         --skip-archives         {dim Don't produce tar/zip archives}
         --skip-os-packages      {dim Don't produce rpm/deb/docker packages}
         --all-platforms         {dim Produce archives for all platforms, not just this one}
-        --rpm                   {dim Only build the rpm package}
-        --deb                   {dim Only build the deb package}
-        --docker                {dim Only build the docker image}
+        --rpm                   {dim Only build the rpm packages}
+        --deb                   {dim Only build the deb packages}
+        --docker-images         {dim Only build the Docker images}
+        --docker-contexts       {dim Only build the Docker build contexts}
         --skip-docker-ubi       {dim Don't build the docker ubi image}
+        --skip-docker-centos    {dim Don't build the docker centos image}
         --release               {dim Produce a release-ready distributable}
         --version-qualifier     {dim Suffix version with a qualifier}
         --skip-node-download    {dim Reuse existing downloads of node.js}

src/dev/build/tasks/os_packages/create_os_package_tasks.ts

@@ -8,7 +8,7 @@
 
 import { Task } from '../../lib';
 import { runFpm } from './run_fpm';
-import { runDockerGenerator, runDockerGeneratorForUBI } from './docker_generator';
+import { runDockerGenerator } from './docker_generator';
 
 export const CreateDebPackage: Task = {
   description: 'Creating deb package',
@@ -49,20 +49,56 @@ export const CreateRpmPackage: Task = {
   },
 };
 
-export const CreateDockerPackage: Task = {
-  description: 'Creating docker package',
+export const CreateDockerCentOS: Task = {
+  description: 'Creating Docker CentOS image',
 
   async run(config, log, build) {
-    // Builds Docker targets for default and oss
-    await runDockerGenerator(config, log, build);
+    await runDockerGenerator(config, log, build, {
+      ubi: false,
+      context: false,
+      architecture: 'x64',
+      image: true,
+    });
+    await runDockerGenerator(config, log, build, {
+      ubi: false,
+      context: false,
+      architecture: 'aarch64',
+      image: true,
+    });
   },
 };
 
-export const CreateDockerUbiPackage: Task = {
-  description: 'Creating docker ubi package',
+export const CreateDockerUBI: Task = {
+  description: 'Creating Docker UBI image',
 
   async run(config, log, build) {
-    // Builds Docker target default with ubi7 base image
-    await runDockerGeneratorForUBI(config, log, build);
+    if (!build.isOss()) {
+      await runDockerGenerator(config, log, build, {
+        ubi: true,
+        context: false,
+        architecture: 'x64',
+        image: true,
+      });
+    }
+  },
+};
+
+export const CreateDockerContexts: Task = {
+  description: 'Creating Docker build contexts',
+
+  async run(config, log, build) {
+    await runDockerGenerator(config, log, build, {
+      ubi: false,
+      context: true,
+      image: false,
+    });
+
+    if (!build.isOss()) {
+      await runDockerGenerator(config, log, build, {
+        ubi: true,
+        context: true,
+        image: false,
+      });
+    }
   },
 };

src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts

@@ -18,7 +18,6 @@ export async function bundleDockerFiles(config: Config, log: ToolingLog, scope:
   log.info(
     `Generating kibana${scope.imageFlavor}${scope.ubiImageFlavor} docker build context bundle`
   );
-
   const dockerFilesDirName = `kibana${scope.imageFlavor}${scope.ubiImageFlavor}-${scope.version}-docker-build-context`;
   const dockerFilesBuildDir = resolve(scope.dockerBuildDir, dockerFilesDirName);
   const dockerFilesOutputDir = config.resolveFromTarget(`${dockerFilesDirName}.tar.gz`);

src/dev/build/tasks/os_packages/docker_generator/index.ts

@@ -6,4 +6,4 @@
  * Public License, v 1.
  */
 
-export { runDockerGenerator, runDockerGeneratorForUBI } from './run';
+export { runDockerGenerator } from './run';

src/dev/build/tasks/os_packages/docker_generator/run.ts

@@ -26,19 +26,26 @@ export async function runDockerGenerator(
   config: Config,
   log: ToolingLog,
   build: Build,
-  ubi: boolean = false
+  flags: {
+    architecture?: string;
+    context: boolean;
+    image: boolean;
+    ubi: boolean;
+  }
 ) {
   // UBI var config
-  const baseOSImage = ubi ? 'docker.elastic.co/ubi8/ubi-minimal:latest' : 'centos:8';
+  const baseOSImage = flags.ubi ? 'docker.elastic.co/ubi8/ubi-minimal:latest' : 'centos:8';
   const ubiVersionTag = 'ubi8';
-  const ubiImageFlavor = ubi ? `-${ubiVersionTag}` : '';
+  const ubiImageFlavor = flags.ubi ? `-${ubiVersionTag}` : '';
 
   // General docker var config
   const license = build.isOss() ? 'ASL 2.0' : 'Elastic License';
   const imageFlavor = build.isOss() ? '-oss' : '';
   const imageTag = 'docker.elastic.co/kibana/kibana';
   const version = config.getBuildVersion();
-  const artifactTarball = `kibana${imageFlavor}-${version}-linux-x86_64.tar.gz`;
+  const artifactArchitecture = flags.architecture === 'aarch64' ? 'aarch64' : 'x86_64';
+  const artifactPrefix = `kibana${imageFlavor}-${version}-linux`;
+  const artifactTarball = `${artifactPrefix}-${artifactArchitecture}.tar.gz`;
   const artifactsDir = config.resolveFromTarget('.');
   const dockerBuildDate = new Date().toISOString();
   // That would produce oss, default and default-ubi7
@@ -47,10 +54,12 @@ export async function runDockerGenerator(
     'kibana-docker',
     build.isOss() ? `oss` : `default${ubiImageFlavor}`
   );
+  const imageArchitecture = flags.architecture === 'aarch64' ? '-aarch64' : '';
   const dockerTargetFilename = config.resolveFromTarget(
-    `kibana${imageFlavor}${ubiImageFlavor}-${version}-docker-image.tar.gz`
+    `kibana${imageFlavor}${ubiImageFlavor}-${version}-docker-image${imageArchitecture}.tar.gz`
   );
   const scope: TemplateContext = {
+    artifactPrefix,
     artifactTarball,
     imageFlavor,
     version,
@@ -62,7 +71,8 @@ export async function runDockerGenerator(
     baseOSImage,
     ubiImageFlavor,
     dockerBuildDate,
-    ubi,
+    ubi: flags.ubi,
+    architecture: flags.architecture,
     revision: config.getBuildSha(),
   };
 
@@ -106,20 +116,23 @@ export async function runDockerGenerator(
   // created from the templates/build_docker_sh.template.js
   // and we just run that bash script
   await chmodAsync(`${resolve(dockerBuildDir, 'build_docker.sh')}`, '755');
-  await exec(log, `./build_docker.sh`, [], {
-    cwd: dockerBuildDir,
-    level: 'info',
-  });
 
-  // Pack Dockerfiles and create a target for them
-  await bundleDockerFiles(config, log, scope);
-}
-
-export async function runDockerGeneratorForUBI(config: Config, log: ToolingLog, build: Build) {
-  // Only run ubi docker image build for default distribution
-  if (build.isOss()) {
-    return;
+  // Only build images on native targets
+  type HostArchitectureToDocker = Record<string, string>;
+  const hostTarget: HostArchitectureToDocker = {
+    x64: 'x64',
+    arm64: 'aarch64',
+  };
+  const buildImage = hostTarget[process.arch] === flags.architecture && flags.image;
+  if (buildImage) {
+    await exec(log, `./build_docker.sh`, [], {
+      cwd: dockerBuildDir,
+      level: 'info',
+    });
   }
 
-  await runDockerGenerator(config, log, build, true);
+  // Pack Dockerfiles and create a target for them
+  if (flags.context) {
+    await bundleDockerFiles(config, log, scope);
+  }
 }

src/dev/build/tasks/os_packages/docker_generator/template_context.ts

@@ -7,6 +7,7 @@
  */
 
 export interface TemplateContext {
+  artifactPrefix: string;
   artifactTarball: string;
   imageFlavor: string;
   version: string;
@@ -21,4 +22,5 @@ export interface TemplateContext {
   usePublicArtifact?: boolean;
   ubi: boolean;
   revision: string;
+  architecture?: string;
 }

src/dev/build/tasks/os_packages/docker_generator/templates/Dockerfile

@@ -17,17 +17,19 @@ RUN {{packageManager}} install -y findutils tar gzip
 
 {{#usePublicArtifact}}
 RUN cd /opt && \
-  curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/kibana/{{artifactTarball}} && \
+  curl --retry 8 -s -L \
+    --output kibana.tar.gz \
+     https://artifacts.elastic.co/downloads/kibana/{{artifactPrefix}}-$(arch).tar.gz && \
   cd -
 {{/usePublicArtifact}}
 
 {{^usePublicArtifact}}
-COPY {{artifactTarball}} /opt
+COPY {{artifactTarball}} /opt/kibana.tar.gz
 {{/usePublicArtifact}}
 
 RUN mkdir /usr/share/kibana
 WORKDIR /usr/share/kibana
-RUN tar --strip-components=1 -zxf /opt/{{artifactTarball}}
+RUN tar --strip-components=1 -zxf /opt/kibana.tar.gz
 # Ensure that group permissions are the same as user permissions.
 # This will help when relying on GID-0 to run Kibana, rather than UID-1000.
 # OpenShift does this, for example.
@@ -51,7 +53,7 @@ EXPOSE 5601
 RUN for iter in {1..10}; do \
       {{packageManager}} update --setopt=tsflags=nodocs -y && \
       {{packageManager}} install --setopt=tsflags=nodocs -y \
-        fontconfig freetype shadow-utils libnss3.so {{#ubi}}findutils{{/ubi}} && \
+        fontconfig freetype shadow-utils nss {{#ubi}}findutils{{/ubi}} && \
       {{packageManager}} clean all && exit_code=0 && break || exit_code=$? && echo "{{packageManager}} error: retry $iter in 10s" && \
       sleep 10; \
     done; \
@@ -59,8 +61,17 @@ RUN for iter in {1..10}; do \
 
 # Add an init process, check the checksum to make sure it's a match
 RUN set -e ; \
+    TINI_BIN="" ; \
+    case "$(arch)" in \
+        aarch64) \
+            TINI_BIN='tini-arm64' ; \
+            ;; \
+        x86_64) \
+            TINI_BIN='tini-amd64' ; \
+            ;; \
+        *) echo >&2 "Unsupported architecture $(arch)" ; exit 1 ;; \
+    esac ; \
   TINI_VERSION='v0.19.0' ; \
-  TINI_BIN='tini-amd64' ; \
   curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN}" ; \
   curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN}.sha256sum" ; \
   sha256sum -c "${TINI_BIN}.sha256sum" ; \

src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts

@@ -17,7 +17,9 @@ function generator({
   dockerTargetFilename,
   baseOSImage,
   ubiImageFlavor,
+  architecture,
 }: TemplateContext) {
+  const fileArchitecture = architecture === 'aarch64' ? 'arm64' : 'amd64';
   return dedent(`
   #!/usr/bin/env bash
   #
@@ -54,9 +56,9 @@ function generator({
   retry_docker_pull ${baseOSImage}
 
   echo "Building: kibana${imageFlavor}${ubiImageFlavor}-docker"; \\
-  docker build -t ${imageTag}${imageFlavor}${ubiImageFlavor}:${version} -f Dockerfile . || exit 1;
+  docker build -t ${imageTag}${imageFlavor}${ubiImageFlavor}:${version}-${fileArchitecture} -f Dockerfile . || exit 1;
 
-  docker save ${imageTag}${imageFlavor}${ubiImageFlavor}:${version} | gzip -c > ${dockerTargetFilename}
+  docker save ${imageTag}${imageFlavor}${ubiImageFlavor}:${version}-${fileArchitecture} | gzip -c > ${dockerTargetFilename}
 
   exit 0
   `);

src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts

@@ -16,6 +16,7 @@ function generator(options: TemplateContext) {
   const template = readFileSync(resolve(__dirname, './Dockerfile'));
   return Mustache.render(template.toString(), {
     packageManager: options.ubiImageFlavor ? 'microdnf' : 'yum',
+    tiniBin: options.architecture === 'aarch64' ? 'tini-arm64' : 'tini-amd64',
     ...options,
   });
 }