[kbn/optimizer] include used dll refs in cache key (#129928)

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -8,6 +8,7 @@
 
 import { Bundle, BundleSpec, parseBundles } from './bundle';
 import { Hashes } from './hashes';
+import { parseDllManifest } from './dll_manifest';
 
 jest.mock('fs');
 
@@ -31,13 +32,31 @@ it('creates cache keys', () => {
   ].sort(() => (Math.random() > 0.5 ? 1 : -1));
 
   const hashes = new Hashes(new Map(hashEntries));
+  const dllManifest = parseDllManifest({
+    name: 'manifest-name',
+    content: {
+      './some-foo.ts': {
+        id: 1,
+        buildMeta: {
+          a: 'b',
+        },
+        unknownField: 'hi',
+      },
+    },
+  });
+  const dllRefKeys = ['./some-foo.ts'];
 
-  expect(bundle.createCacheKey(['/foo/bar/a', '/foo/bar/c'], hashes)).toMatchInlineSnapshot(`
+  expect(bundle.createCacheKey(['/foo/bar/a', '/foo/bar/c'], hashes, dllManifest, dllRefKeys))
+    .toMatchInlineSnapshot(`
     Object {
       "checksums": Object {
         "/foo/bar/a": "123",
         "/foo/bar/c": "789",
       },
+      "dllName": "manifest-name",
+      "dllRefs": Object {
+        "./some-foo.ts": "1:ku/53aRMuAA+4TmQeCWA/w:GtuPW9agF2yecW0xAIHtUQ",
+      },
       "spec": Object {
         "banner": undefined,
         "contextDir": "/foo/bar",

packages/kbn-optimizer/src/common/bundle.ts

@@ -14,6 +14,7 @@ import { UnknownVals } from './ts_helpers';
 import { omit } from './obj_helpers';
 import { includes } from './array_helpers';
 import type { Hashes } from './hashes';
+import { ParsedDllManifest } from './dll_manifest';
 
 const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
@@ -88,12 +89,19 @@ export class Bundle {
 
   /**
    * Calculate the cache key for this bundle based from current
-   * mtime values.
+   * state determined by looking at files on disk.
    */
-  createCacheKey(paths: string[], hashes: Hashes): unknown {
+  createCacheKey(
+    paths: string[],
+    hashes: Hashes,
+    dllManifest: ParsedDllManifest,
+    dllRefKeys: string[]
+  ): unknown {
     return {
       spec: omit(this.toSpec(), ['pageLoadAssetSizeLimit']),
       checksums: Object.fromEntries(paths.map((p) => [p, hashes.getCached(p)] as const)),
+      dllName: dllManifest.name,
+      dllRefs: Object.fromEntries(dllRefKeys.map((k) => [k, dllManifest.content[k]] as const)),
     };
   }
 

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -19,6 +19,7 @@ export interface State {
   workUnits?: number;
   referencedPaths?: string[];
   bundleRefExportIds?: string[];
+  dllRefKeys?: string[];
 }
 
 const DEFAULT_STATE: State = {};
@@ -90,6 +91,10 @@ export class BundleCache {
     return this.get().bundleRefExportIds;
   }
 
+  public getDllRefKeys() {
+    return this.get().dllRefKeys;
+  }
+
   public getCacheKey() {
     return this.get().cacheKey;
   }

packages/kbn-optimizer/src/common/dll_manifest.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Crypto from 'crypto';
+
+export interface DllManifest {
+  name: string;
+  content: Record<string, any>;
+}
+
+export interface ParsedDllManifest {
+  name: string;
+  content: Record<string, any>;
+}
+
+const hash = (s: string) => {
+  return Crypto.createHash('md5').update(s).digest('base64').replace(/=+$/, '');
+};
+
+export function parseDllManifest(manifest: DllManifest): ParsedDllManifest {
+  return {
+    name: manifest.name,
+    content: Object.fromEntries(
+      Object.entries(manifest.content).map(([k, v]) => {
+        const { id, buildMeta, ...other } = v;
+        const metaJson = JSON.stringify(buildMeta) || '{}';
+        const otherJson = JSON.stringify(other) || '{}';
+
+        return [
+          k,
+          [
+            v.id,
+            ...(metaJson !== '{}' ? [hash(metaJson)] : []),
+            ...(otherJson !== '{}' ? [hash(otherJson)] : []),
+          ].join(':'),
+        ];
+      })
+    ),
+  };
+}

packages/kbn-optimizer/src/common/index.ts

@@ -20,3 +20,4 @@ export * from './parse_path';
 export * from './theme_tags';
 export * from './obj_helpers';
 export * from './hashes';
+export * from './dll_manifest';

packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts

@@ -10,20 +10,31 @@ import Path from 'path';
 
 import cpy from 'cpy';
 import del from 'del';
-import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+import { createAbsolutePathSerializer, createStripAnsiSerializer } from '@kbn/dev-utils';
 
 import { OptimizerConfig } from '../optimizer/optimizer_config';
-import { allValuesFrom, Bundle, Hashes } from '../common';
+import { allValuesFrom, Bundle, Hashes, ParsedDllManifest } from '../common';
 import { getBundleCacheEvent$ } from '../optimizer/bundle_cache';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
 const MOCK_REPO_DIR = Path.resolve(TMP_DIR, 'mock_repo');
 
+jest.mock('../common/dll_manifest', () => ({
+  parseDllManifest: jest.fn(),
+}));
+
+const EMPTY_DLL_MANIFEST: ParsedDllManifest = {
+  name: 'foo',
+  content: {},
+};
+jest.requireMock('../common/dll_manifest').parseDllManifest.mockReturnValue(EMPTY_DLL_MANIFEST);
+
 expect.addSnapshotSerializer({
   print: () => '<Bundle>',
   test: (v) => v instanceof Bundle,
 });
+expect.addSnapshotSerializer(createStripAnsiSerializer());
 expect.addSnapshotSerializer(createAbsolutePathSerializer(MOCK_REPO_DIR));
 
 beforeEach(async () => {
@@ -55,7 +66,7 @@ it('emits "bundle cached" event when everything is updated', async () => {
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
@@ -63,6 +74,7 @@ it('emits "bundle cached" event when everything is updated', async () => {
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -94,7 +106,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
@@ -102,6 +114,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -133,7 +146,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async ()
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
@@ -141,6 +154,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async ()
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -172,7 +186,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
@@ -180,6 +194,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -188,11 +203,11 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "- Expected
-    + Received
+        "diff": "- Expected
+    + Received
 
-    - \\"old\\"
-    + \\"optimizerCacheKey\\"",
+    - \\"old\\"
+    + \\"optimizerCacheKey\\"",
         "reason": "optimizer cache key mismatch",
         "type": "bundle not cached",
       },
@@ -216,7 +231,7 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
@@ -224,6 +239,7 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: ['plugin/bar/public'],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -232,12 +248,12 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "- Expected
-    + Received
+        "diff": "- Expected
+    + Received
 
-      [
-    +   \\"plugin/bar/public\\"
-      ]",
+      [
+    +   \\"plugin/bar/public\\"
+      ]",
         "reason": "bundle references outdated",
         "type": "bundle not cached",
       },
@@ -267,6 +283,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => {
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -304,6 +321,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'new');
@@ -314,14 +332,53 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "- Expected
-    + Received
+        "diff": "- Expected
+    + Received
 
-    - \\"old\\"
-    + \\"new\\"",
+    - \\"old\\"
+    + \\"new\\"",
         "reason": "cache key mismatch",
         "type": "bundle not cached",
       },
     ]
   `);
 });
+
+it('emits "dll references missing" when cacheKey has no dllRefs', async () => {
+  const config = OptimizerConfig.create({
+    repoRoot: MOCK_REPO_DIR,
+    pluginScanDirs: [],
+    pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
+    maxWorkerCount: 1,
+  });
+  const [bundle] = config.bundles;
+
+  const optimizerCacheKey = 'optimizerCacheKey';
+  const referencedPaths = [
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
+  ];
+
+  bundle.cache.set({
+    cacheKey: 'correct',
+    optimizerCacheKey,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
+    bundleRefExportIds: [],
+  });
+
+  jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'correct');
+
+  const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
+
+  expect(cacheEvents).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "bundle": <Bundle>,
+        "reason": "dll references missing",
+        "type": "bundle not cached",
+      },
+    ]
+  `);
+});

packages/kbn-optimizer/src/integration_tests/optimizer_built_paths.test.ts

@@ -22,6 +22,7 @@ it(`finds all the optimizer files relative to it's path`, async () => {
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle_refs.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/compiler_messages.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/dll_manifest.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/event_stream_helpers.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/hashes.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/index.js,

packages/kbn-optimizer/src/optimizer/bundle_cache.ts

@@ -6,10 +6,13 @@
  * Side Public License, v 1.
  */
 
+import Fs from 'fs';
+
 import * as Rx from 'rxjs';
 import { mergeAll } from 'rxjs/operators';
+import { dllManifestPath } from '@kbn/ui-shared-deps-npm';
 
-import { Bundle, BundleRefs, Hashes } from '../common';
+import { Bundle, BundleRefs, Hashes, parseDllManifest } from '../common';
 
 import { OptimizerConfig } from './optimizer_config';
 import { diffCacheKey } from './diff_cache_key';
@@ -25,7 +28,8 @@ export interface BundleNotCachedEvent {
     | 'cache key mismatch'
     | 'cache disabled'
     | 'bundle references missing'
-    | 'bundle references outdated';
+    | 'bundle references outdated'
+    | 'dll references missing';
   diff?: string;
   bundle: Bundle;
 }
@@ -95,7 +99,6 @@ export function getBundleCacheEvent$(
       }
 
       const refs = bundleRefs.filterByExportIds(bundleRefExportIds);
-
       const bundleRefsDiff = diffCacheKey(
         refs.map((r) => r.exportId).sort((a, b) => a.localeCompare(b)),
         bundleRefExportIds
@@ -110,15 +113,28 @@ export function getBundleCacheEvent$(
         continue;
       }
 
+      if (!bundle.cache.getDllRefKeys()) {
+        events.push({
+          type: 'bundle not cached',
+          reason: 'dll references missing',
+          bundle,
+        });
+        continue;
+      }
+
       eligibleBundles.push(bundle);
     }
 
+    const dllManifest = parseDllManifest(JSON.parse(Fs.readFileSync(dllManifestPath, 'utf8')));
     const hashes = new Hashes();
     for (const bundle of eligibleBundles) {
       const paths = bundle.cache.getReferencedPaths() ?? [];
       await hashes.populate(paths);
 
-      const diff = diffCacheKey(bundle.cache.getCacheKey(), bundle.createCacheKey(paths, hashes));
+      const diff = diffCacheKey(
+        bundle.cache.getCacheKey(),
+        bundle.createCacheKey(paths, hashes, dllManifest, bundle.cache.getDllRefKeys() ?? [])
+      );
 
       if (diff) {
         events.push({

packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts

@@ -12,7 +12,14 @@ import { inspect } from 'util';
 
 import webpack from 'webpack';
 
-import { Bundle, WorkerConfig, ascending, parseFilePath, Hashes } from '../common';
+import {
+  Bundle,
+  WorkerConfig,
+  ascending,
+  parseFilePath,
+  Hashes,
+  ParsedDllManifest,
+} from '../common';
 import { BundleRefModule } from './bundle_ref_module';
 import {
   isExternalModule,
@@ -47,7 +54,11 @@ function isBazelPackage(pkgJsonPath: string) {
 }
 
 export class PopulateBundleCachePlugin {
-  constructor(private readonly workerConfig: WorkerConfig, private readonly bundle: Bundle) {}
+  constructor(
+    private readonly workerConfig: WorkerConfig,
+    private readonly bundle: Bundle,
+    private readonly dllManifest: ParsedDllManifest
+  ) {}
 
   public apply(compiler: webpack.Compiler) {
     const { bundle, workerConfig } = this;
@@ -80,6 +91,8 @@ export class PopulateBundleCachePlugin {
           return rawHashes.set(path, Hashes.hash(content));
         };
 
+        const dllRefKeys = new Set<string>();
+
         if (bundle.manifestPath) {
           addReferenced(bundle.manifestPath);
         }
@@ -136,7 +149,13 @@ export class PopulateBundleCachePlugin {
             continue;
           }
 
-          if (isExternalModule(module) || isIgnoredModule(module) || isDelegatedModule(module)) {
+          if (isDelegatedModule(module)) {
+            // delegated modules are the references to the ui-shared-deps-npm dll
+            dllRefKeys.add(module.userRequest);
+            continue;
+          }
+
+          if (isExternalModule(module) || isIgnoredModule(module)) {
             continue;
           }
 
@@ -144,14 +163,21 @@ export class PopulateBundleCachePlugin {
         }
 
         const referencedPaths = Array.from(paths).sort(ascending((p) => p));
+        const sortedDllRefKeys = Array.from(dllRefKeys).sort(ascending((p) => p));
 
         bundle.cache.set({
           bundleRefExportIds: bundleRefExportIds.sort(ascending((p) => p)),
           optimizerCacheKey: workerConfig.optimizerCacheKey,
-          cacheKey: bundle.createCacheKey(referencedPaths, new Hashes(rawHashes)),
+          cacheKey: bundle.createCacheKey(
+            referencedPaths,
+            new Hashes(rawHashes),
+            this.dllManifest,
+            sortedDllRefKeys
+          ),
           moduleCount,
           workUnits,
           referencedPaths,
+          dllRefKeys: sortedDllRefKeys,
         });
 
         // write the cache to the compilation so that it isn't cleaned by clean-webpack-plugin

packages/kbn-optimizer/src/worker/webpack.config.ts

@@ -7,6 +7,7 @@
  */
 
 import Path from 'path';
+import Fs from 'fs';
 
 import { stringifyRequest } from 'loader-utils';
 import webpack from 'webpack';
@@ -18,7 +19,7 @@ import CompressionPlugin from 'compression-webpack-plugin';
 import UiSharedDepsNpm from '@kbn/ui-shared-deps-npm';
 import * as UiSharedDepsSrc from '@kbn/ui-shared-deps-src';
 
-import { Bundle, BundleRefs, WorkerConfig } from '../common';
+import { Bundle, BundleRefs, WorkerConfig, parseDllManifest } from '../common';
 import { BundleRefsPlugin } from './bundle_refs_plugin';
 import { BundleMetricsPlugin } from './bundle_metrics_plugin';
 import { EmitStatsPlugin } from './emit_stats_plugin';
@@ -27,6 +28,7 @@ import { PopulateBundleCachePlugin } from './populate_bundle_cache_plugin';
 const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE;
 const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset');
 const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
+const DLL_MANIFEST = JSON.parse(Fs.readFileSync(UiSharedDepsNpm.dllManifestPath, 'utf8'));
 
 const nodeModulesButNotKbnPackages = (path: string) => {
   if (!path.includes('node_modules')) {
@@ -79,11 +81,11 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
     plugins: [
       new CleanWebpackPlugin(),
       new BundleRefsPlugin(bundle, bundleRefs),
-      new PopulateBundleCachePlugin(worker, bundle),
+      new PopulateBundleCachePlugin(worker, bundle, parseDllManifest(DLL_MANIFEST)),
       new BundleMetricsPlugin(bundle),
       new webpack.DllReferencePlugin({
         context: worker.repoRoot,
-        manifest: require(UiSharedDepsNpm.dllManifestPath),
+        manifest: DLL_MANIFEST,
       }),
       ...(worker.profileWebpack ? [new EmitStatsPlugin(bundle)] : []),
       ...(bundle.banner ? [new webpack.BannerPlugin({ banner: bundle.banner, raw: true })] : []),

packages/kbn-optimizer/src/worker/webpack_helpers.ts

@@ -156,6 +156,8 @@ export interface WebpackDelegatedModule {
   type: string;
   id: number;
   dependencies: unknown[];
+  /** The ID of the module in the DLL manifest */
+  userRequest: string;
 }
 
 export function isDelegatedModule(module: any): module is WebpackDelegatedModule {