[build] Add cloud docker images (#107949) (#111818)

Co-authored-by: Przemysław Hejman <przemyslaw.hejman@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Jonathan Budzenski <jon@budzenski.me>
Co-authored-by: Przemysław Hejman <przemyslaw.hejman@elastic.co>

.gitignore

@@ -5,6 +5,7 @@
 /.chromium
 .DS_Store
 .node_binaries
+/.beats
 .native_modules
 node_modules
 !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules

src/dev/build/args.test.ts

@@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -55,6 +56,7 @@ it('builds packages if --all-platforms is passed', () => {
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,
@@ -81,6 +83,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -107,6 +110,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -134,6 +138,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,
@@ -168,6 +173,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -195,6 +201,7 @@ it('limits packages if --all-platforms passed with --skip-docker-centos', () =>
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,

src/dev/build/args.ts

@@ -26,6 +26,7 @@ export function readCliArgs(argv: string[]) {
       'skip-docker-contexts',
       'skip-docker-ubi',
       'skip-docker-centos',
+      'docker-cloud',
       'release',
       'skip-node-download',
       'verbose',
@@ -103,6 +104,7 @@ export function readCliArgs(argv: string[]) {
     createDebPackage: isOsPackageDesired('deb'),
     createDockerCentOS:
       isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-centos']),
+    createDockerCloud: isOsPackageDesired('docker-images') && Boolean(flags['docker-cloud']),
     createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']),
     createDockerContexts: !Boolean(flags['skip-docker-contexts']),
     targetAllPlatforms: Boolean(flags['all-platforms']),

src/dev/build/build_distributables.ts

@@ -22,6 +22,7 @@ export interface BuildOptions {
   createDebPackage: boolean;
   createDockerUBI: boolean;
   createDockerCentOS: boolean;
+  createDockerCloud: boolean;
   createDockerContexts: boolean;
   versionQualifier: string | undefined;
   targetAllPlatforms: boolean;
@@ -127,6 +128,11 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await run(Tasks.CreateDockerCentOS);
   }
 
+  if (options.createDockerCloud) {
+    // control w/ --docker-images and --docker-cloud
+    await run(Tasks.CreateDockerCloud);
+  }
+
   if (options.createDockerContexts) {
     // control w/ --skip-docker-contexts
     await run(Tasks.CreateDockerContexts);

src/dev/build/tasks/os_packages/create_os_package_tasks.ts

@@ -91,6 +91,25 @@ export const CreateDockerUBI: Task = {
   },
 };
 
+export const CreateDockerCloud: Task = {
+  description: 'Creating Docker Cloud image',
+
+  async run(config, log, build) {
+    await runDockerGenerator(config, log, build, {
+      architecture: 'x64',
+      context: false,
+      cloud: true,
+      image: true,
+    });
+    await runDockerGenerator(config, log, build, {
+      architecture: 'aarch64',
+      context: false,
+      cloud: true,
+      image: true,
+    });
+  },
+};
+
 export const CreateDockerContexts: Task = {
   description: 'Creating Docker build contexts',
 
@@ -111,5 +130,10 @@ export const CreateDockerContexts: Task = {
       context: true,
       image: false,
     });
+    await runDockerGenerator(config, log, build, {
+      cloud: true,
+      context: true,
+      image: false,
+    });
   },
 };

src/dev/build/tasks/os_packages/docker_generator/run.ts

@@ -7,7 +7,7 @@
  */
 
 import { access, link, unlink, chmod } from 'fs';
-import { resolve } from 'path';
+import { resolve, basename } from 'path';
 import { promisify } from 'util';
 
 import { ToolingLog, kibanaPackageJson } from '@kbn/dev-utils';
@@ -32,6 +32,7 @@ export async function runDockerGenerator(
     image: boolean;
     ubi?: boolean;
     ironbank?: boolean;
+    cloud?: boolean;
     dockerBuildDate?: string;
   }
 ) {
@@ -42,6 +43,7 @@ export async function runDockerGenerator(
   let imageFlavor = '';
   if (flags.ubi) imageFlavor += `-${ubiVersionTag}`;
   if (flags.ironbank) imageFlavor += '-ironbank';
+  if (flags.cloud) imageFlavor += '-cloud';
 
   // General docker var config
   const license = 'Elastic License';
@@ -50,7 +52,10 @@ export async function runDockerGenerator(
   const artifactArchitecture = flags.architecture === 'aarch64' ? 'aarch64' : 'x86_64';
   const artifactPrefix = `kibana-${version}-linux`;
   const artifactTarball = `${artifactPrefix}-${artifactArchitecture}.tar.gz`;
+  const metricbeatTarball = `metricbeat-${version}-linux-${artifactArchitecture}.tar.gz`;
+  const filebeatTarball = `filebeat-${version}-linux-${artifactArchitecture}.tar.gz`;
   const artifactsDir = config.resolveFromTarget('.');
+  const beatsDir = config.resolveFromRepo('.beats');
   const dockerBuildDate = flags.dockerBuildDate || new Date().toISOString();
   // That would produce oss, default and default-ubi7
   const dockerBuildDir = config.resolveFromRepo('build', 'kibana-docker', `default${imageFlavor}`);
@@ -58,6 +63,13 @@ export async function runDockerGenerator(
   const dockerTargetFilename = config.resolveFromTarget(
     `kibana${imageFlavor}-${version}-docker-image${imageArchitecture}.tar.gz`
   );
+  const dependencies = [
+    resolve(artifactsDir, artifactTarball),
+    ...(flags.cloud
+      ? [resolve(beatsDir, metricbeatTarball), resolve(beatsDir, filebeatTarball)]
+      : []),
+  ];
+
   const scope: TemplateContext = {
     artifactPrefix,
     artifactTarball,
@@ -72,6 +84,9 @@ export async function runDockerGenerator(
     baseOSImage,
     dockerBuildDate,
     ubi: flags.ubi,
+    cloud: flags.cloud,
+    metricbeatTarball,
+    filebeatTarball,
     ironbank: flags.ironbank,
     architecture: flags.architecture,
     revision: config.getBuildSha(),
@@ -87,26 +102,8 @@ export async function runDockerGenerator(
     return;
   }
 
-  // Verify if we have the needed kibana target in order
-  // to build the kibana docker image.
-  // Also create the docker build target folder
-  // and  delete the current linked target into the
-  // kibana docker build folder if we have one.
-  try {
-    await accessAsync(resolve(artifactsDir, artifactTarball));
-    await mkdirp(dockerBuildDir);
-    await unlinkAsync(resolve(dockerBuildDir, artifactTarball));
-  } catch (e) {
-    if (e && e.code === 'ENOENT' && e.syscall === 'access') {
-      throw new Error(
-        `Kibana linux target (${artifactTarball}) is needed in order to build ${''}the docker image. None was found at ${artifactsDir}`
-      );
-    }
-  }
-
-  // Create the kibana linux target inside the
-  // Kibana docker build
-  await linkAsync(resolve(artifactsDir, artifactTarball), resolve(dockerBuildDir, artifactTarball));
+  // Create the docker build target folder
+  await mkdirp(dockerBuildDir);
 
   // Write all the needed docker config files
   // into kibana-docker folder
@@ -137,6 +134,21 @@ export async function runDockerGenerator(
 
   // Only build images on native targets
   if (flags.image) {
+    // Link dependencies
+    for (const src of dependencies) {
+      const file = basename(src);
+      const dest = resolve(dockerBuildDir, file);
+      try {
+        await accessAsync(src);
+        await unlinkAsync(dest);
+      } catch (e) {
+        if (e && e.code === 'ENOENT' && e.syscall === 'access') {
+          throw new Error(`${src} is needed in order to build the docker image.`);
+        }
+      }
+      await linkAsync(src, dest);
+    }
+
     await exec(log, `./build_docker.sh`, [], {
       cwd: dockerBuildDir,
       level: 'info',

src/dev/build/tasks/os_packages/docker_generator/template_context.ts

@@ -21,6 +21,9 @@ export interface TemplateContext {
   dockerBuildDate: string;
   usePublicArtifact?: boolean;
   ubi?: boolean;
+  cloud?: boolean;
+  metricbeatTarball?: string;
+  filebeatTarball?: string;
   ironbank?: boolean;
   revision: string;
   architecture?: string;

src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile

@@ -24,18 +24,27 @@ RUN cd /opt && \
 {{/usePublicArtifact}}
 
 {{^usePublicArtifact}}
-COPY {{artifactTarball}} /opt/kibana.tar.gz
+COPY {{artifactTarball}} /tmp/kibana.tar.gz
 {{/usePublicArtifact}}
 
 RUN mkdir /usr/share/kibana
 WORKDIR /usr/share/kibana
-RUN tar --strip-components=1 -zxf /opt/kibana.tar.gz
+RUN tar --strip-components=1 -zxf /tmp/kibana.tar.gz
 # Ensure that group permissions are the same as user permissions.
 # This will help when relying on GID-0 to run Kibana, rather than UID-1000.
 # OpenShift does this, for example.
 # REF: https://docs.openshift.org/latest/creating_images/guidelines.html
 RUN chmod -R g=u /usr/share/kibana
 
+{{#cloud}}
+COPY {{filebeatTarball}} /tmp/filebeat.tar.gz
+COPY {{metricbeatTarball}} /tmp/metricbeat.tar.gz
+
+RUN mkdir -p /opt/filebeat /opt/metricbeat && \
+  tar xf /tmp/filebeat.tar.gz -C /opt/filebeat --strip-components=1 && \
+  tar xf /tmp/metricbeat.tar.gz -C /opt/metricbeat --strip-components=1
+{{/cloud}}
+
 ################################################################################
 # Build stage 1 (the actual Kibana image):
 #
@@ -86,6 +95,9 @@ RUN fc-cache -v
 
 # Bring in Kibana from the initial stage.
 COPY --from=builder --chown=1000:0 /usr/share/kibana /usr/share/kibana
+{{#cloud}}
+COPY --from=builder --chown=0:0 /opt /opt
+{{/cloud}}
 WORKDIR /usr/share/kibana
 RUN ln -s /usr/share/kibana /opt/kibana
 
@@ -145,8 +157,19 @@ RUN mkdir /licenses && \
     cp LICENSE.txt /licenses/LICENSE
 {{/ubi}}
 
-USER kibana
-
 ENTRYPOINT ["/bin/tini", "--"]
 
+{{#cloud}}
+CMD ["/app/kibana.sh"]
+# Generate a stub command that will be overwritten at runtime
+RUN mkdir /app && \
+    echo -e '#!/bin/sh\nexec /usr/local/bin/kibana-docker' > /app/kibana.sh && \
+    chmod 0555 /app/kibana.sh
+{{/cloud}}
+
+{{^cloud}}
 CMD ["/usr/local/bin/kibana-docker"]
+{{/cloud}}
+
+
+USER kibana