Generate dockerfiles target to be used for elastic/dockerfiles (#32169) (#34704)

* feat(NA): generate bundles with dockerfiles as targets.

* chore(NA): change dockerfiles suffix to docker-build-context.

src/dev/build/lib/fs.js

@@ -17,9 +17,10 @@
  * under the License.
  */
 
+import archiver from 'archiver';
 import fs from 'fs';
 import { createHash } from 'crypto';
-import { resolve, dirname, isAbsolute } from 'path';
+import { resolve, dirname, isAbsolute, sep } from 'path';
 import { createGunzip } from 'zlib';
 import { inspect } from 'util';
 
@@ -203,3 +204,15 @@ export async function untar(source, destination, extractOptions = {}) {
     }),
   ]);
 }
+
+export async function compress(type, options = {}, source, destination) {
+  const output = fs.createWriteStream(destination);
+  const archive = archiver(type, options);
+  const name = source.split(sep).slice(-1)[0];
+
+  archive.pipe(output);
+
+  return archive
+    .directory(source, name)
+    .finalize();
+}

src/dev/build/lib/index.js

@@ -31,6 +31,7 @@ export {
   untar,
   deleteAll,
   deleteEmptyFolders,
+  compress,
 } from './fs';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/create_archives_task.js

@@ -18,20 +18,7 @@
  */
 
 import path from 'path';
-import { createWriteStream } from 'fs';
-import archiver from 'archiver';
-
-import { mkdirp } from '../lib';
-
-function compress(type, options = {}, source, destination) {
-  const output = createWriteStream(destination);
-  const archive = archiver(type, options);
-  const name = source.split(path.sep).slice(-1)[0];
-
-  archive.pipe(output);
-
-  return archive.directory(source, name).finalize();
-}
+import { mkdirp, compress } from '../lib';
 
 export const CreateArchivesTask = {
   description: 'Creating the archives for each platform',

src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.js

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+import { compress, copyAll, mkdirp, write } from '../../../lib';
+import { dockerfileTemplate } from './templates';
+
+export async function bundleDockerFiles(config, log, build, scope) {
+  log.info(`Generating kibana${ scope.imageFlavor } docker build context bundle`);
+
+  const dockerFilesDirName = `kibana${ scope.imageFlavor }-${ scope.versionTag }-docker-build-context`;
+  const dockerFilesBuildDir = resolve(scope.dockerBuildDir, dockerFilesDirName);
+  const dockerFilesOutputDir = config.resolveFromTarget(
+    `${ dockerFilesDirName }.tar.gz`
+  );
+
+  // Create dockerfiles dir inside docker build dir
+  await mkdirp(dockerFilesBuildDir);
+
+  // Create a release Dockerfile
+  await write(
+    resolve(dockerFilesBuildDir, dockerfileTemplate.name),
+    dockerfileTemplate.generator({
+      ...scope,
+      usePublicArtifact: true
+    })
+  );
+
+  // Move relevant docker build files inside
+  // dockerfiles folder
+  await copyAll(
+    resolve(scope.dockerBuildDir, 'bin'),
+    resolve(dockerFilesBuildDir, 'bin'),
+  );
+  await copyAll(
+    resolve(scope.dockerBuildDir, 'config'),
+    resolve(dockerFilesBuildDir, 'config'),
+  );
+
+  // Compress dockerfiles dir created inside
+  // docker build dir as output it as a target
+  // on targets folder
+  await compress(
+    'tar',
+    {
+      gzip: true,
+      gzipOptions: {
+        level: 9
+      }
+    },
+    dockerFilesBuildDir,
+    dockerFilesOutputDir
+  );
+}

src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker

@@ -1,4 +1,7 @@
 #!/bin/bash
+#
+# ** THIS IS AN AUTO-GENERATED FILE **
+#
 
 # Run Kibana, using environment variables to set longopts defining Kibana's
 # configuration.

src/dev/build/tasks/os_packages/docker_generator/run.js

@@ -22,6 +22,7 @@ import { resolve } from 'path';
 import { promisify } from 'util';
 import { write, copyAll, mkdirp, exec } from '../../../lib';
 import * as dockerTemplates from './templates';
+import { bundleDockerFiles } from './bundle_dockerfiles';
 
 const accessAsync = promisify(access);
 const linkAsync = promisify(link);
@@ -37,6 +38,16 @@ export async function runDockerGenerator(config, log, build) {
   const artifactsDir = config.resolveFromTarget('.');
   const dockerBuildDir = config.resolveFromRepo('build', 'kibana-docker', build.isOss() ? 'oss' : 'default');
   const dockerOutputDir = config.resolveFromTarget(`kibana${ imageFlavor }-${ versionTag }-docker.tar.gz`);
+  const scope = {
+    artifactTarball,
+    imageFlavor,
+    versionTag,
+    license,
+    artifactsDir,
+    imageTag,
+    dockerBuildDir,
+    dockerOutputDir
+  };
 
   // Verify if we have the needed kibana target in order
   // to build the kibana docker image.
@@ -65,16 +76,6 @@ export async function runDockerGenerator(config, log, build) {
 
   // Write all the needed docker config files
   // into kibana-docker folder
-  const scope = {
-    artifactTarball,
-    imageFlavor,
-    versionTag,
-    license,
-    artifactsDir,
-    imageTag,
-    dockerOutputDir
-  };
-
   for (const [, dockerTemplate] of Object.entries(dockerTemplates)) {
     await write(resolve(dockerBuildDir, dockerTemplate.name), dockerTemplate.generator(scope));
   }
@@ -96,4 +97,7 @@ export async function runDockerGenerator(config, log, build) {
     cwd: dockerBuildDir,
     level: 'info',
   });
+
+  // Pack Dockerfiles and create a target for them
+  await bundleDockerFiles(config, log, build, scope);
 }

src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.js

@@ -19,7 +19,15 @@
 
 import dedent from 'dedent';
 
-function generator({ artifactTarball, versionTag, license  }) {
+function generator({ artifactTarball, versionTag, license, usePublicArtifact  }) {
+  const copyArtifactTarballInsideDockerOptFolder = () => {
+    if (usePublicArtifact) {
+      return `RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/kibana/${ artifactTarball } && cd -`;
+    }
+
+    return `COPY ${ artifactTarball } /opt`;
+  };
+
   return dedent(`
   #
   # ** THIS IS AN AUTO-GENERATED FILE **
@@ -30,7 +38,7 @@ function generator({ artifactTarball, versionTag, license  }) {
   # Extract Kibana and make various file manipulations.
   ################################################################################
   FROM centos:7 AS prep_files
-  COPY ${ artifactTarball } /opt
+  ${copyArtifactTarballInsideDockerOptFolder()}
   RUN mkdir /usr/share/kibana
   WORKDIR /usr/share/kibana
   RUN tar --strip-components=1 -zxf /opt/${ artifactTarball }