[optimizer] harden cache to bazel output changing mtime (#127609)

packages/kbn-optimizer/src/common/array_helpers.ts

@@ -62,14 +62,3 @@ export const descending = <T>(...getters: Array<SortPropGetter<T>>): Comparator<
  * Alternate Array#includes() implementation with sane types, functions as a type guard
  */
 export const includes = <T>(array: T[], value: any): value is T => array.includes(value);
-
-/**
- * Ponyfill for Object.fromEntries()
- */
-export const entriesToObject = <T>(entries: Array<readonly [string, T]>): Record<string, T> => {
-  const object: Record<string, T> = {};
-  for (const [key, value] of entries) {
-    object[key] = value;
-  }
-  return object;
-};

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -7,6 +7,7 @@
  */
 
 import { Bundle, BundleSpec, parseBundles } from './bundle';
+import { Hashes } from './hashes';
 
 jest.mock('fs');
 
@@ -21,20 +22,21 @@ const SPEC: BundleSpec = {
 
 it('creates cache keys', () => {
   const bundle = new Bundle(SPEC);
-  expect(
-    bundle.createCacheKey(
-      ['/foo/bar/a', '/foo/bar/c'],
-      new Map([
-        ['/foo/bar/a', 123],
-        ['/foo/bar/b', 456],
-        ['/foo/bar/c', 789],
-      ])
-    )
-  ).toMatchInlineSnapshot(`
+
+  // randomly sort the hash entries to make sure that the cache key never changes based on the order of the hash cache
+  const hashEntries = [
+    ['/foo/bar/a', '123'] as const,
+    ['/foo/bar/b', '456'] as const,
+    ['/foo/bar/c', '789'] as const,
+  ].sort(() => (Math.random() > 0.5 ? 1 : -1));
+
+  const hashes = new Hashes(new Map(hashEntries));
+
+  expect(bundle.createCacheKey(['/foo/bar/a', '/foo/bar/c'], hashes)).toMatchInlineSnapshot(`
     Object {
-      "mtimes": Object {
-        "/foo/bar/a": 123,
-        "/foo/bar/c": 789,
+      "checksums": Object {
+        "/foo/bar/a": "123",
+        "/foo/bar/c": "789",
       },
       "spec": Object {
         "banner": undefined,

packages/kbn-optimizer/src/common/bundle.ts

@@ -12,7 +12,8 @@ import Fs from 'fs';
 import { BundleCache } from './bundle_cache';
 import { UnknownVals } from './ts_helpers';
 import { omit } from './obj_helpers';
-import { includes, ascending, entriesToObject } from './array_helpers';
+import { includes } from './array_helpers';
+import type { Hashes } from './hashes';
 
 const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
@@ -89,12 +90,10 @@ export class Bundle {
    * Calculate the cache key for this bundle based from current
    * mtime values.
    */
-  createCacheKey(files: string[], mtimes: Map<string, number | undefined>): unknown {
+  createCacheKey(paths: string[], hashes: Hashes): unknown {
     return {
       spec: omit(this.toSpec(), ['pageLoadAssetSizeLimit']),
-      mtimes: entriesToObject(
-        files.map((p) => [p, mtimes.get(p)] as const).sort(ascending((e) => e[0]))
-      ),
+      checksums: Object.fromEntries(paths.map((p) => [p, hashes.getCached(p)] as const)),
     };
   }
 

packages/kbn-optimizer/src/common/bundle_cache.test.ts

@@ -15,7 +15,7 @@ const mockWriteFileSync: jest.Mock = jest.requireMock('fs').writeFileSync;
 
 const SOME_STATE: State = {
   cacheKey: 'abc',
-  files: ['123'],
+  referencedPaths: ['123'],
   moduleCount: 123,
   optimizerCacheKey: 'abc',
 };
@@ -49,7 +49,7 @@ it(`updates files on disk when calling set()`, () => {
         "/foo/.kbn-optimizer-cache",
         "{
       \\"cacheKey\\": \\"abc\\",
-      \\"files\\": [
+      \\"referencedPaths\\": [
         \\"123\\"
       ],
       \\"moduleCount\\": 123,
@@ -94,14 +94,14 @@ it('provides accessors to specific state properties', () => {
   const cache = new BundleCache('/foo');
 
   expect(cache.getModuleCount()).toBe(undefined);
-  expect(cache.getReferencedFiles()).toEqual(undefined);
+  expect(cache.getReferencedPaths()).toEqual(undefined);
   expect(cache.getCacheKey()).toEqual(undefined);
   expect(cache.getOptimizerCacheKey()).toEqual(undefined);
 
   cache.set(SOME_STATE);
 
   expect(cache.getModuleCount()).toBe(123);
-  expect(cache.getReferencedFiles()).toEqual(['123']);
+  expect(cache.getReferencedPaths()).toEqual(['123']);
   expect(cache.getCacheKey()).toEqual('abc');
   expect(cache.getOptimizerCacheKey()).toEqual('abc');
 });

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -17,7 +17,7 @@ export interface State {
   cacheKey?: unknown;
   moduleCount?: number;
   workUnits?: number;
-  files?: string[];
+  referencedPaths?: string[];
   bundleRefExportIds?: string[];
 }
 
@@ -82,8 +82,8 @@ export class BundleCache {
     return this.get().moduleCount;
   }
 
-  public getReferencedFiles() {
-    return this.get().files;
+  public getReferencedPaths() {
+    return this.get().referencedPaths;
   }
 
   public getBundleRefExportIds() {

packages/kbn-optimizer/src/common/hashes.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fs from 'fs';
+import { pipeline } from 'stream/promises';
+import { createHash } from 'crypto';
+import { asyncMapWithLimit, asyncForEachWithLimit } from '@kbn/std';
+
+export class Hashes {
+  static async hashFile(path: string) {
+    const hash = createHash('sha256');
+    try {
+      await pipeline(Fs.createReadStream(path), hash);
+      return hash.digest('hex');
+    } catch (error) {
+      if (error && error.code === 'ENOENT') {
+        return null;
+      }
+
+      throw error;
+    }
+  }
+
+  static hash(content: Buffer) {
+    return createHash('sha256').update(content).digest('hex');
+  }
+
+  static async ofFiles(paths: string[]) {
+    return new Hashes(
+      new Map(
+        await asyncMapWithLimit(paths, 100, async (path) => {
+          return [path, await Hashes.hashFile(path)];
+        })
+      )
+    );
+  }
+
+  constructor(public readonly cache = new Map<string, string | null>()) {}
+
+  async populate(paths: string[]) {
+    await asyncForEachWithLimit(paths, 100, async (path) => {
+      if (!this.cache.has(path)) {
+        this.cache.set(path, await Hashes.hashFile(path));
+      }
+    });
+  }
+
+  getCached(path: string) {
+    const cached = this.cache.get(path);
+    if (cached === undefined) {
+      throw new Error(`hash for path [${path}] is not cached`);
+    }
+    return cached;
+  }
+
+  cacheToJson() {
+    return Object.fromEntries(
+      Array.from(this.cache.entries()).filter((e): e is [string, string] => e[1] !== null)
+    );
+  }
+}

packages/kbn-optimizer/src/common/index.ts

@@ -19,3 +19,4 @@ export * from './event_stream_helpers';
 export * from './parse_path';
 export * from './theme_tags';
 export * from './obj_helpers';
+export * from './hashes';

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -131,7 +131,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   expect(foo).toBeTruthy();
   foo.cache.refresh();
   expect(foo.cache.getModuleCount()).toBe(6);
-  expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(`
+  expect(foo.cache.getReferencedPaths()).toMatchInlineSnapshot(`
     Array [
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/bazel-out/<platform>-fastbuild/bin/packages/kbn-ui-shared-deps-npm/target_node/public_path_module_creator.js,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json,
@@ -151,7 +151,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     16
   );
 
-  expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(`
+  expect(bar.cache.getReferencedPaths()).toMatchInlineSnapshot(`
     Array [
       <absolute path>/node_modules/@kbn/optimizer/postcss.config.js,
       <absolute path>/node_modules/css-loader/package.json,
@@ -174,7 +174,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   baz.cache.refresh();
   expect(baz.cache.getModuleCount()).toBe(3);
 
-  expect(baz.cache.getReferencedFiles()).toMatchInlineSnapshot(`
+  expect(baz.cache.getReferencedPaths()).toMatchInlineSnapshot(`
     Array [
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/bazel-out/<platform>-fastbuild/bin/packages/kbn-ui-shared-deps-npm/target_node/public_path_module_creator.js,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/x-pack/baz/kibana.json,

packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts

@@ -12,9 +12,8 @@ import cpy from 'cpy';
 import del from 'del';
 import { createAbsolutePathSerializer } from '@kbn/dev-utils';
 
-import { getMtimes } from '../optimizer/get_mtimes';
 import { OptimizerConfig } from '../optimizer/optimizer_config';
-import { allValuesFrom, Bundle } from '../common';
+import { allValuesFrom, Bundle, Hashes } from '../common';
 import { getBundleCacheEvent$ } from '../optimizer/bundle_cache';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
@@ -50,19 +49,19 @@ it('emits "bundle cached" event when everything is updated', async () => {
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
-  const mtimes = await getMtimes(files);
-  const cacheKey = bundle.createCacheKey(files, mtimes);
+  const hashes = await Hashes.ofFiles(referencedPaths);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 
@@ -89,19 +88,19 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
-  const mtimes = await getMtimes(files);
-  const cacheKey = bundle.createCacheKey(files, mtimes);
+  const hashes = await Hashes.ofFiles(referencedPaths);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 
@@ -128,19 +127,19 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async ()
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
-  const mtimes = await getMtimes(files);
-  const cacheKey = bundle.createCacheKey(files, mtimes);
+  const hashes = await Hashes.ofFiles(referencedPaths);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey: undefined,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 
@@ -167,19 +166,19 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
-  const mtimes = await getMtimes(files);
-  const cacheKey = bundle.createCacheKey(files, mtimes);
+  const hashes = await Hashes.ofFiles(referencedPaths);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey: 'old',
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 
@@ -211,19 +210,19 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
-  const mtimes = await getMtimes(files);
-  const cacheKey = bundle.createCacheKey(files, mtimes);
+  const hashes = await Hashes.ofFiles(referencedPaths);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: ['plugin/bar/public'],
   });
 
@@ -256,7 +255,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => {
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
@@ -265,8 +264,8 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => {
   bundle.cache.set({
     cacheKey: undefined,
     optimizerCacheKey,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 
@@ -293,7 +292,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
   const [bundle] = config.bundles;
 
   const optimizerCacheKey = 'optimizerCacheKey';
-  const files = [
+  const referencedPaths = [
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
@@ -302,8 +301,8 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
   bundle.cache.set({
     cacheKey: 'old',
     optimizerCacheKey,
-    files,
-    moduleCount: files.length,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
   });
 

packages/kbn-optimizer/src/integration_tests/optimizer_built_paths.test.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// @ts-expect-error
+import { getOptimizerBuiltPaths } from '@kbn/optimizer/target_node/optimizer/optimizer_built_paths';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
+
+it(`finds all the optimizer files relative to it's path`, async () => {
+  const paths = await getOptimizerBuiltPaths();
+  expect(paths).toMatchInlineSnapshot(`
+    Array [
+      <absolute path>/node_modules/@kbn/optimizer/target_node/cli.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/array_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle_cache.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle_refs.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/compiler_messages.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/event_stream_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/hashes.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/index.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/obj_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/parse_path.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/rxjs_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/theme_tags.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/ts_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/worker_config.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/worker_messages.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/index.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/limits.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/log_optimizer_progress.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/log_optimizer_state.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/assign_bundles_to_workers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/bundle_cache.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/diff_cache_key.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/filter_by_id.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/focus_bundles.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/get_plugin_bundles.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/handle_optimizer_completion.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/index.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/kibana_platform_plugins.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/observe_stdio.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/observe_worker.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/optimizer_built_paths.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/optimizer_cache_key.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/optimizer_config.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/optimizer_state.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/run_workers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/watch_bundles_for_changes.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/optimizer/watcher.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/report_optimizer_timings.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/run_optimizer.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/bundle_metrics_plugin.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/bundle_ref_module.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/bundle_refs_plugin.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/emit_stats_plugin.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/entry_point_creator.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/populate_bundle_cache_plugin.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/run_compilers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/run_worker.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/theme_loader.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/webpack_helpers.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/worker/webpack.config.js,
+    ]
+  `);
+});

packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts

@@ -35,7 +35,7 @@ const makeTestBundle = (id: string) => {
     cacheKey: 'abc',
     moduleCount: 1,
     optimizerCacheKey: 'abc',
-    files: [bundleEntryPath(bundle)],
+    referencedPaths: [bundleEntryPath(bundle)],
   });
 
   return bundle;
@@ -83,7 +83,7 @@ it('notifies of changes and completes once all bundles have changed', async () =
           return;
         }
 
-        // first we change foo and bar, and after 1 second get that change comes though
+        // first we change foo and bar, after 1 second that change comes though
         if (i === 1) {
           expect(event.bundles).toHaveLength(2);
           const [bar, foo] = event.bundles.sort(ascending((b) => b.id));

packages/kbn-optimizer/src/optimizer/bundle_cache.ts

@@ -9,11 +9,10 @@
 import * as Rx from 'rxjs';
 import { mergeAll } from 'rxjs/operators';
 
-import { Bundle, BundleRefs } from '../common';
+import { Bundle, BundleRefs, Hashes } from '../common';
 
 import { OptimizerConfig } from './optimizer_config';
-import { getMtimes } from './get_mtimes';
-import { diffCacheKey } from './cache_keys';
+import { diffCacheKey } from './diff_cache_key';
 
 export type BundleCacheEvent = BundleNotCachedEvent | BundleCachedEvent;
 
@@ -114,20 +113,12 @@ export function getBundleCacheEvent$(
       eligibleBundles.push(bundle);
     }
 
-    const mtimes = await getMtimes(
-      new Set<string>(
-        eligibleBundles.reduce(
-          (acc: string[], bundle) => [...acc, ...(bundle.cache.getReferencedFiles() || [])],
-          []
-        )
-      )
-    );
-
+    const hashes = new Hashes();
     for (const bundle of eligibleBundles) {
-      const diff = diffCacheKey(
-        bundle.cache.getCacheKey(),
-        bundle.createCacheKey(bundle.cache.getReferencedFiles() || [], mtimes)
-      );
+      const paths = bundle.cache.getReferencedPaths() ?? [];
+      await hashes.populate(paths);
+
+      const diff = diffCacheKey(bundle.cache.getCacheKey(), bundle.createCacheKey(paths, hashes));
 
       if (diff) {
         events.push({

packages/kbn-optimizer/src/optimizer/cache_keys.test.ts

@@ -1,102 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Path from 'path';
-
-import { REPO_ROOT } from '@kbn/utils';
-import { createAbsolutePathSerializer } from '@kbn/dev-utils';
-
-import { getOptimizerCacheKey } from './cache_keys';
-import { OptimizerConfig } from './optimizer_config';
-
-jest.mock('./get_changes.ts', () => ({
-  getChanges: async () =>
-    new Map([
-      ['/foo/bar/a', 'modified'],
-      ['/foo/bar/b', 'modified'],
-      ['/foo/bar/c', 'deleted'],
-    ]),
-}));
-
-jest.mock('./get_mtimes.ts', () => ({
-  getMtimes: async (paths: string[]) => new Map(paths.map((path) => [path, 12345])),
-}));
-
-jest.mock('execa');
-
-jest.mock('fs', () => {
-  const realFs = jest.requireActual('fs');
-  return {
-    ...realFs,
-    readFile: jest.fn(realFs.readFile),
-  };
-});
-
-expect.addSnapshotSerializer(createAbsolutePathSerializer());
-
-jest.requireMock('execa').mockImplementation(async (cmd: string, args: string[], opts: object) => {
-  expect(cmd).toBe('git');
-  expect(args).toEqual([
-    'log',
-    '-n',
-    '1',
-    '--pretty=format:%H',
-    '--',
-    expect.stringContaining('kbn-optimizer'),
-  ]);
-  expect(opts).toEqual({
-    cwd: REPO_ROOT,
-  });
-
-  return {
-    stdout: '<last commit sha>',
-  };
-});
-
-describe('getOptimizerCacheKey()', () => {
-  it('uses latest commit, bootstrap cache, and changed files to create unique value', async () => {
-    jest
-      .requireMock('fs')
-      .readFile.mockImplementation(
-        (path: string, enc: string, cb: (err: null, file: string) => void) => {
-          expect(path).toBe(
-            Path.resolve(REPO_ROOT, 'packages/kbn-optimizer/target/.bootstrap-cache')
-          );
-          expect(enc).toBe('utf8');
-          cb(null, '<bootstrap cache>');
-        }
-      );
-
-    const config = OptimizerConfig.create({
-      repoRoot: REPO_ROOT,
-    });
-
-    await expect(getOptimizerCacheKey(config)).resolves.toMatchInlineSnapshot(`
-            Object {
-              "deletedPaths": Array [
-                "/foo/bar/c",
-              ],
-              "lastCommit": "<last commit sha>",
-              "modifiedTimes": Object {
-                "/foo/bar/a": 12345,
-                "/foo/bar/b": 12345,
-              },
-              "workerConfig": Object {
-                "browserslistEnv": "dev",
-                "dist": false,
-                "optimizerCacheKey": "♻",
-                "repoRoot": <absolute path>,
-                "themeTags": Array [
-                  "v8dark",
-                  "v8light",
-                ],
-              },
-            }
-          `);
-  });
-});

packages/kbn-optimizer/src/optimizer/cache_keys.ts

@@ -1,95 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Path from 'path';
-import Fs from 'fs';
-
-import execa from 'execa';
-import { REPO_ROOT } from '@kbn/utils';
-import { diffStrings } from '@kbn/dev-utils';
-
-import jsonStable from 'json-stable-stringify';
-import { ascending, CacheableWorkerConfig } from '../common';
-
-import { getMtimes } from './get_mtimes';
-import { getChanges } from './get_changes';
-import { OptimizerConfig } from './optimizer_config';
-
-const RELATIVE_DIR = 'packages/kbn-optimizer';
-
-export function diffCacheKey(expected?: unknown, actual?: unknown) {
-  const expectedJson = jsonStable(expected, {
-    space: '  ',
-  });
-  const actualJson = jsonStable(actual, {
-    space: '  ',
-  });
-
-  if (expectedJson === actualJson) {
-    return;
-  }
-
-  return diffStrings(expectedJson, actualJson);
-}
-
-export interface OptimizerCacheKey {
-  readonly lastCommit: string | undefined;
-  readonly workerConfig: CacheableWorkerConfig;
-  readonly deletedPaths: string[];
-  readonly modifiedTimes: Record<string, number>;
-}
-
-async function getLastCommit() {
-  const { stdout } = await execa(
-    'git',
-    ['log', '-n', '1', '--pretty=format:%H', '--', RELATIVE_DIR],
-    {
-      cwd: REPO_ROOT,
-    }
-  );
-
-  return stdout.trim() || undefined;
-}
-
-export async function getOptimizerCacheKey(config: OptimizerConfig): Promise<OptimizerCacheKey> {
-  if (!Fs.existsSync(Path.resolve(REPO_ROOT, '.git'))) {
-    return {
-      lastCommit: undefined,
-      modifiedTimes: {},
-      workerConfig: config.getCacheableWorkerConfig(),
-      deletedPaths: [],
-    };
-  }
-
-  const [changes, lastCommit] = await Promise.all([
-    getChanges(RELATIVE_DIR),
-    getLastCommit(),
-  ] as const);
-
-  const deletedPaths: string[] = [];
-  const modifiedPaths: string[] = [];
-  for (const [path, type] of changes) {
-    (type === 'deleted' ? deletedPaths : modifiedPaths).push(path);
-  }
-
-  const cacheKeys: OptimizerCacheKey = {
-    lastCommit,
-    deletedPaths,
-    modifiedTimes: {} as Record<string, number>,
-    workerConfig: config.getCacheableWorkerConfig(),
-  };
-
-  const mtimes = await getMtimes(modifiedPaths);
-  for (const [path, mtime] of Array.from(mtimes.entries()).sort(ascending((e) => e[0]))) {
-    if (typeof mtime === 'number') {
-      cacheKeys.modifiedTimes[path] = mtime;
-    }
-  }
-
-  return cacheKeys;
-}

packages/kbn-optimizer/src/optimizer/diff_cache_key.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { diffStrings } from '@kbn/dev-utils';
+import jsonStable from 'json-stable-stringify';
+
+export function diffCacheKey(expected?: unknown, actual?: unknown) {
+  const expectedJson = jsonStable(expected, {
+    space: '  ',
+  });
+  const actualJson = jsonStable(actual, {
+    space: '  ',
+  });
+
+  if (expectedJson === actualJson) {
+    return;
+  }
+
+  return diffStrings(expectedJson, actualJson);
+}

packages/kbn-optimizer/src/optimizer/get_changes.test.ts

@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-jest.mock('execa');
-
-import { getChanges } from './get_changes';
-import { createAbsolutePathSerializer } from '@kbn/dev-utils';
-import { REPO_ROOT } from '@kbn/utils';
-
-const execa: jest.Mock = jest.requireMock('execa');
-
-expect.addSnapshotSerializer(createAbsolutePathSerializer());
-
-it('parses git ls-files output', async () => {
-  expect.assertions(4);
-
-  execa.mockImplementation((cmd, args, options) => {
-    expect(cmd).toBe('git');
-    expect(args).toEqual(['ls-files', '-dmt', '--', 'foo/bar/x']);
-    expect(options).toEqual({
-      cwd: REPO_ROOT,
-    });
-
-    return {
-      stdout: [
-        'C kbn-optimizer/package.json',
-        'C kbn-optimizer/src/common/bundle.ts',
-        'R kbn-optimizer/src/common/bundles.ts',
-        'C kbn-optimizer/src/common/bundles.ts',
-        'R kbn-optimizer/src/get_bundle_definitions.test.ts',
-        'C kbn-optimizer/src/get_bundle_definitions.test.ts',
-      ].join('\n'),
-    };
-  });
-
-  const changes = await getChanges('foo/bar/x');
-
-  expect(changes).toMatchInlineSnapshot(`
-    Map {
-      <absolute path>/kbn-optimizer/package.json => "modified",
-      <absolute path>/kbn-optimizer/src/common/bundle.ts => "modified",
-      <absolute path>/kbn-optimizer/src/common/bundles.ts => "deleted",
-      <absolute path>/kbn-optimizer/src/get_bundle_definitions.test.ts => "deleted",
-    }
-  `);
-});

packages/kbn-optimizer/src/optimizer/get_changes.ts

@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Path from 'path';
-
-import execa from 'execa';
-
-import { REPO_ROOT } from '@kbn/utils';
-
-export type Changes = Map<string, 'modified' | 'deleted'>;
-
-/**
- * get the changes in all the context directories (plugin public paths)
- */
-export async function getChanges(relativeDir: string) {
-  const changes: Changes = new Map();
-
-  const { stdout } = await execa('git', ['ls-files', '-dmt', '--', relativeDir], {
-    cwd: REPO_ROOT,
-  });
-
-  const output = stdout.trim();
-
-  if (output) {
-    for (const line of output.split('\n')) {
-      const [tag, ...pathParts] = line.trim().split(' ');
-      const path = Path.resolve(REPO_ROOT, pathParts.join(' '));
-      switch (tag) {
-        case 'M':
-        case 'C':
-          // for some reason ls-files returns deleted files as both deleted
-          // and modified, so make sure not to overwrite changes already
-          // tracked as "deleted"
-          if (changes.get(path) !== 'deleted') {
-            changes.set(path, 'modified');
-          }
-          break;
-
-        case 'R':
-          changes.set(path, 'deleted');
-          break;
-
-        default:
-          throw new Error(`unexpected path status ${tag} for path ${path}`);
-      }
-    }
-  }
-
-  return changes;
-}

packages/kbn-optimizer/src/optimizer/get_mtimes.test.ts

@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-jest.mock('fs');
-
-import { getMtimes } from './get_mtimes';
-
-const { stat }: { stat: jest.Mock } = jest.requireMock('fs');
-
-it('returns mtimes Map', async () => {
-  stat.mockImplementation((path, cb) => {
-    if (path.includes('missing')) {
-      const error = new Error('file not found');
-      (error as any).code = 'ENOENT';
-      cb(error);
-    } else {
-      cb(null, {
-        mtimeMs: 1234,
-      });
-    }
-  });
-
-  await expect(getMtimes(['/foo/bar', '/foo/missing', '/foo/baz', '/foo/bar'])).resolves
-    .toMatchInlineSnapshot(`
-    Map {
-      "/foo/bar" => 1234,
-      "/foo/baz" => 1234,
-    }
-  `);
-});

packages/kbn-optimizer/src/optimizer/get_mtimes.ts

@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import Fs from 'fs';
-
-import * as Rx from 'rxjs';
-import { mergeMap, map, catchError } from 'rxjs/operators';
-import { allValuesFrom } from '../common';
-
-const stat$ = Rx.bindNodeCallback<Fs.PathLike, Fs.Stats>(Fs.stat);
-
-/**
- * get mtimes of referenced paths concurrently, limit concurrency to 100
- */
-export async function getMtimes(paths: Iterable<string>) {
-  return new Map(
-    await allValuesFrom(
-      Rx.from(paths).pipe(
-        // map paths to [path, mtimeMs] entries with concurrency of
-        // 100 at a time, ignoring missing paths
-        mergeMap(
-          (path) =>
-            stat$(path).pipe(
-              map((stat) => [path, stat.mtimeMs] as const),
-              catchError((error: any) =>
-                error?.code === 'ENOENT' ? Rx.EMPTY : Rx.throwError(error)
-              )
-            ),
-          100
-        )
-      )
-    )
-  );
-}

packages/kbn-optimizer/src/optimizer/index.ts

@@ -9,7 +9,8 @@
 export * from './optimizer_config';
 export type { WorkerStdio } from './observe_worker';
 export * from './optimizer_state';
-export * from './cache_keys';
+export * from './diff_cache_key';
+export * from './optimizer_cache_key';
 export * from './watch_bundles_for_changes';
 export * from './run_workers';
 export * from './bundle_cache';

packages/kbn-optimizer/src/optimizer/optimizer_built_paths.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import globby from 'globby';
+
+import { ascending } from '../common';
+
+export async function getOptimizerBuiltPaths() {
+  return (
+    await globby(
+      ['**/*', '!**/{__fixtures__,__snapshots__,integration_tests,babel_runtime_helpers,node}/**'],
+      {
+        cwd: Path.resolve(__dirname, '../'),
+        absolute: true,
+      }
+    )
+  )
+    .slice()
+    .sort(ascending((p) => p));
+}

packages/kbn-optimizer/src/optimizer/optimizer_cache_key.test.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { REPO_ROOT } from '@kbn/utils';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+
+import { getOptimizerCacheKey } from './optimizer_cache_key';
+import { OptimizerConfig } from './optimizer_config';
+
+jest.mock('../common/hashes', () => {
+  return {
+    Hashes: class MockHashes {
+      static ofFiles = jest.fn(() => {
+        return new MockHashes();
+      });
+
+      cacheToJson() {
+        return { foo: 'bar' };
+      }
+    },
+  };
+});
+
+jest.mock('./optimizer_built_paths', () => {
+  return {
+    getOptimizerBuiltPaths: () => ['/built/foo.js', '/built/bar.js'],
+  };
+});
+
+const { Hashes: MockHashes } = jest.requireMock('../common/hashes');
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
+
+describe('getOptimizerCacheKey()', () => {
+  it('determines checksums of all optimizer files', async () => {
+    const config = OptimizerConfig.create({
+      repoRoot: REPO_ROOT,
+    });
+
+    const key = await getOptimizerCacheKey(config);
+
+    expect(MockHashes.ofFiles).toMatchInlineSnapshot(`
+      [MockFunction] {
+        "calls": Array [
+          Array [
+            Array [
+              "/built/foo.js",
+              "/built/bar.js",
+            ],
+          ],
+        ],
+        "results": Array [
+          Object {
+            "type": "return",
+            "value": MockHashes {},
+          },
+        ],
+      }
+    `);
+
+    expect(key).toMatchInlineSnapshot(`
+      Object {
+        "checksums": Object {
+          "foo": "bar",
+        },
+        "workerConfig": Object {
+          "browserslistEnv": "dev",
+          "dist": false,
+          "optimizerCacheKey": "♻",
+          "repoRoot": <absolute path>,
+          "themeTags": Array [
+            "v8dark",
+            "v8light",
+          ],
+        },
+      }
+    `);
+  });
+});

packages/kbn-optimizer/src/optimizer/optimizer_cache_key.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { CacheableWorkerConfig, Hashes } from '../common';
+import { OptimizerConfig } from './optimizer_config';
+import { getOptimizerBuiltPaths } from './optimizer_built_paths';
+
+export interface OptimizerCacheKey {
+  readonly workerConfig: CacheableWorkerConfig;
+  readonly checksums: Record<string, string>;
+}
+
+/**
+ * Hash the contents of the built files that the optimizer is currently running. This allows us to
+ * invalidate the optimizer results if someone forgets to bootstrap, or changes the optimizer source files
+ */
+export async function getOptimizerCacheKey(config: OptimizerConfig): Promise<OptimizerCacheKey> {
+  const hashes = await Hashes.ofFiles(await getOptimizerBuiltPaths());
+
+  return {
+    checksums: hashes.cacheToJson(),
+    workerConfig: config.getCacheableWorkerConfig(),
+  };
+}

packages/kbn-optimizer/src/optimizer/watcher.ts

@@ -63,7 +63,7 @@ export class Watcher {
           (changes): Changes => ({
             type: 'changes',
             bundles: bundles.filter((bundle) => {
-              const referencedFiles = bundle.cache.getReferencedFiles();
+              const referencedFiles = bundle.cache.getReferencedPaths();
               return changes.some((change) => referencedFiles?.includes(change));
             }),
           })
@@ -73,15 +73,15 @@ export class Watcher {
 
       // call watchpack.watch after listerners are setup
       Rx.defer(() => {
-        const watchPaths: string[] = [];
+        const watchPaths = new Set<string>();
 
         for (const bundle of bundles) {
-          for (const path of bundle.cache.getReferencedFiles() || []) {
-            watchPaths.push(path);
+          for (const path of bundle.cache.getReferencedPaths() || []) {
+            watchPaths.add(path);
           }
         }
 
-        this.watchpack.watch(watchPaths, [], startTime);
+        this.watchpack.watch(Array.from(watchPaths), [], startTime);
         return Rx.EMPTY;
       })
     );

packages/kbn-optimizer/src/worker/populate_bundle_cache_plugin.ts

@@ -12,7 +12,7 @@ import { inspect } from 'util';
 
 import webpack from 'webpack';
 
-import { Bundle, WorkerConfig, ascending, parseFilePath } from '../common';
+import { Bundle, WorkerConfig, ascending, parseFilePath, Hashes } from '../common';
 import { BundleRefModule } from './bundle_ref_module';
 import {
   isExternalModule,
@@ -59,12 +59,29 @@ export class PopulateBundleCachePlugin {
       },
       (compilation) => {
         const bundleRefExportIds: string[] = [];
-        const referencedFiles = new Set<string>();
         let moduleCount = 0;
         let workUnits = compilation.fileDependencies.size;
 
+        const paths = new Set<string>();
+        const rawHashes = new Map<string, string | null>();
+        const addReferenced = (path: string) => {
+          if (paths.has(path)) {
+            return;
+          }
+
+          paths.add(path);
+          let content: Buffer;
+          try {
+            content = compiler.inputFileSystem.readFileSync(path);
+          } catch {
+            return rawHashes.set(path, null);
+          }
+
+          return rawHashes.set(path, Hashes.hash(content));
+        };
+
         if (bundle.manifestPath) {
-          referencedFiles.add(bundle.manifestPath);
+          addReferenced(bundle.manifestPath);
         }
 
         for (const module of compilation.modules) {
@@ -84,13 +101,13 @@ export class PopulateBundleCachePlugin {
             }
 
             if (!parsedPath.dirs.includes('node_modules')) {
-              referencedFiles.add(path);
+              addReferenced(path);
 
               if (path.endsWith('.scss')) {
                 workUnits += EXTRA_SCSS_WORK_UNITS;
 
                 for (const depPath of module.buildInfo.fileDependencies) {
-                  referencedFiles.add(depPath);
+                  addReferenced(depPath);
                 }
               }
 
@@ -105,7 +122,7 @@ export class PopulateBundleCachePlugin {
               'package.json'
             );
 
-            referencedFiles.add(isBazelPackage(pkgJsonPath) ? path : pkgJsonPath);
+            addReferenced(isBazelPackage(pkgJsonPath) ? path : pkgJsonPath);
             continue;
           }
 
@@ -126,28 +143,15 @@ export class PopulateBundleCachePlugin {
           throw new Error(`Unexpected module type: ${inspect(module)}`);
         }
 
-        const files = Array.from(referencedFiles).sort(ascending((p) => p));
-        const mtimes = new Map(
-          files.map((path): [string, number | undefined] => {
-            try {
-              return [path, compiler.inputFileSystem.statSync(path)?.mtimeMs];
-            } catch (error) {
-              if (error?.code === 'ENOENT') {
-                return [path, undefined];
-              }
-
-              throw error;
-            }
-          })
-        );
+        const referencedPaths = Array.from(paths).sort(ascending((p) => p));
 
         bundle.cache.set({
           bundleRefExportIds: bundleRefExportIds.sort(ascending((p) => p)),
           optimizerCacheKey: workerConfig.optimizerCacheKey,
-          cacheKey: bundle.createCacheKey(files, mtimes),
+          cacheKey: bundle.createCacheKey(referencedPaths, new Hashes(rawHashes)),
           moduleCount,
           workUnits,
-          files,
+          referencedPaths,
         });
 
         // write the cache to the compilation so that it isn't cleaned by clean-webpack-plugin