[build] Create CDN assets (#169707)

Closes https://github.com/elastic/kibana/issues/169427

Adds a new build step `createCdnAssets` that will create an archive
`kibana-<version>-cdn-assets.tar.gz` with static assets organized using
the request structure of the kibana client.

- By default CDN assets are created
- Adding the flag `node scripts/build --skip-cdn-assets` will skip
creation
- `ci:build-cdn-assets` can be used to create and upload the archive for
testing

Testing: see https://github.com/elastic/kibana/pull/169408. Builds are
available in the artifacts tab on the `Build Distribution` step.

1) Extract builds
2) ```
python3 -m http.server -b localhost -d kibana-8.12.0-SNAPSHOT-cdn-assets
8000
    ```
3) ```
echo 'server.cdn.url: http://localhost:8000' >>
kibana-8.12.0-SNAPSHOT/config/kibana.yml
    ```
4) ```
   kibana-8.12.0-SNAPSHOT/bin/kibana
   ```

.buildkite/scripts/build_kibana.sh

@@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross
 is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages")
 is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime")
 is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts")
+is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets")
 
 echo "> node scripts/build" "${BUILD_ARGS[@]}"
 node scripts/build "${BUILD_ARGS[@]}"
@@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-initialize \
   --skip-generic-folders \
   --skip-platform-folders \
+  --skip-cdn-assets \
   --skip-archives \
   --docker-images \
   --docker-tag-qualifier="$GIT_COMMIT" \

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -37,7 +37,8 @@ node scripts/build \
   --skip-docker-ubuntu \
   --skip-docker-ubi \
   --skip-docker-cloud \
-  --skip-docker-contexts
+  --skip-docker-contexts \
+  --skip-cdn-assets
 
 echo "--- Tag images"
 docker rmi "$KIBANA_IMAGE"

.buildkite/scripts/steps/package_testing/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts
+node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts --skip-cdn-assets
 
 DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"
 

src/dev/build/args.test.ts

@@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": false,
         "createDockerCloud": false,
         "createDockerContexts": true,
@@ -67,6 +68,7 @@ it('builds packages if --all-platforms is passed', () => {
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": true,
         "createDockerCloud": true,
         "createDockerContexts": true,
@@ -105,6 +107,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": false,
         "createDockerCloud": false,
         "createDockerContexts": true,
@@ -143,6 +146,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": true,
         "createDockerCloud": false,
         "createDockerContexts": true,
@@ -182,6 +186,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": false,
         "createDockerCloud": true,
         "createDockerContexts": true,
@@ -228,6 +233,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": false,
         "createDockerCloud": true,
         "createDockerContexts": true,
@@ -267,6 +273,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
       "buildOptions": Object {
         "buildCanvasShareableRuntime": true,
         "createArchives": true,
+        "createCdnAssets": true,
         "createDebPackage": true,
         "createDockerCloud": true,
         "createDockerContexts": true,

src/dev/build/args.ts

@@ -16,6 +16,7 @@ export function readCliArgs(argv: string[]) {
   const flags = getopts(argv, {
     boolean: [
       'skip-archives',
+      'skip-cdn-assets',
       'skip-initialize',
       'skip-generic-folders',
       'skip-platform-folders',
@@ -132,6 +133,7 @@ export function readCliArgs(argv: string[]) {
     createGenericFolders: !Boolean(flags['skip-generic-folders']),
     createPlatformFolders: !Boolean(flags['skip-platform-folders']),
     createArchives: !Boolean(flags['skip-archives']),
+    createCdnAssets: !Boolean(flags['skip-cdn-assets']),
     createRpmPackage: isOsPackageDesired('rpm'),
     createDebPackage: isOsPackageDesired('deb'),
     createDockerUbuntu:

src/dev/build/build_distributables.ts

@@ -26,6 +26,7 @@ export interface BuildOptions {
   createGenericFolders: boolean;
   createPlatformFolders: boolean;
   createArchives: boolean;
+  createCdnAssets: boolean;
   createRpmPackage: boolean;
   createDebPackage: boolean;
   createDockerUBI: boolean;
@@ -113,6 +114,10 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await run(Tasks.AssertPathLength);
     await run(Tasks.AssertNoUUID);
   }
+  // control w/ --skip-cdn-assets
+  if (options.createCdnAssets) {
+    await run(Tasks.CreateCdnAssets);
+  }
 
   /**
    * package platform-specific builds into archives

src/dev/build/tasks/create_cdn_assets_task.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { readFileSync } from 'fs';
+import { access } from 'fs/promises';
+
+import { resolve, dirname } from 'path';
+import { asyncForEach } from '@kbn/std';
+import { Jsonc } from '@kbn/repo-packages';
+
+import del from 'del';
+import globby from 'globby';
+
+import { mkdirp, compressTar, Task, copyAll } from '../lib';
+
+export const CreateCdnAssets: Task = {
+  description: 'Creating CDN assets',
+
+  async run(config, log, build) {
+    const buildSource = build.resolvePath();
+    const buildNum = config.getBuildNumber();
+    const buildVersion = config.getBuildVersion();
+    const assets = config.resolveFromRepo('build', 'cdn-assets');
+    const bundles = resolve(assets, String(buildNum), 'bundles');
+
+    await del(assets);
+    await mkdirp(assets);
+
+    // Plugins
+
+    const plugins = globby.sync([`${buildSource}/node_modules/@kbn/**/*/kibana.jsonc`]);
+    await asyncForEach(plugins, async (path) => {
+      const manifest = Jsonc.parse(readFileSync(path, 'utf8')) as any;
+      if (manifest?.plugin?.id) {
+        const pluginRoot = resolve(dirname(path));
+
+        try {
+          // packages/core/plugins/core-plugins-server-internal/src/plugins_service.ts
+          const assetsSource = resolve(pluginRoot, 'assets');
+          const assetsDest = resolve('plugins', manifest.plugin.id, 'assets');
+          await access(assetsSource);
+          await mkdirp(assetsDest);
+          await copyAll(assetsSource, assetsDest);
+        } catch (e) {
+          // assets are optional
+          if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e;
+        }
+
+        try {
+          // packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts
+          const bundlesSource = resolve(pluginRoot, 'target', 'public');
+          const bundlesDest = resolve(bundles, 'plugin', manifest.plugin.id, '1.0.0');
+          await access(bundlesSource);
+          await mkdirp(bundlesDest);
+          await copyAll(bundlesSource, bundlesDest);
+        } catch (e) {
+          // bundles are optional
+          if (!(e.code === 'ENOENT' && e.syscall === 'access')) throw e;
+        }
+      }
+    });
+
+    // packages/core/apps/core-apps-server-internal/src/bundle_routes/register_bundle_routes.ts
+    await copyAll(
+      resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-npm/shared_built_assets'),
+      resolve(bundles, 'kbn-ui-shared-deps-npm')
+    );
+    await copyAll(
+      resolve(buildSource, 'node_modules/@kbn/ui-shared-deps-src/shared_built_assets'),
+      resolve(bundles, 'kbn-ui-shared-deps-src')
+    );
+    await copyAll(
+      resolve(buildSource, 'node_modules/@kbn/core/target/public'),
+      resolve(bundles, 'core')
+    );
+    await copyAll(resolve(buildSource, 'node_modules/@kbn/monaco'), resolve(bundles, 'kbn-monaco'));
+
+    // packages/core/apps/core-apps-server-internal/src/core_app.ts
+    await copyAll(
+      resolve(buildSource, 'node_modules/@kbn/core-apps-server-internal/assets'),
+      resolve(assets, 'ui')
+    );
+
+    await compressTar({
+      source: assets,
+      destination: config.resolveFromTarget(`kibana-${buildVersion}-cdn-assets.tar.gz`),
+      archiverOptions: {
+        gzip: true,
+        gzipOptions: {
+          level: 9,
+        },
+      },
+      createRootDirectory: true,
+      rootDirectoryName: `kibana-${buildVersion}-cdn-assets`,
+    });
+  },
+};

src/dev/build/tasks/index.ts

@@ -14,6 +14,7 @@ export * from './clean_tasks';
 export * from './copy_legacy_source_task';
 export * from './create_archives_sources_task';
 export * from './create_archives_task';
+export * from './create_cdn_assets_task';
 export * from './create_empty_dirs_and_files_task';
 export * from './create_readme_task';
 export * from './download_cloud_dependencies';