[7.x] [kbn/optimizer] report limits with ci metrics (#78205) (#79950)

* [kbn/optimizer] report limits with ci metrics (#78205)

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

* update bundle limits for 7.x branch

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-dev-utils/src/ci_stats_reporter/README.md

@@ -8,10 +8,23 @@ This class integrates with the `ciStats.trackBuild {}` Jenkins Pipeline function
 
 To create an instance of the reporter, import the class and call `CiStatsReporter.fromEnv(log)` (passing it a tooling log).
 
-#### `CiStatsReporter#metrics(metrics: Array<{ group: string, id: string, value: number }>)`
+#### `CiStatsReporter#metrics(metrics: Metric[])`
 
 Use this method to record metrics in the Kibana CI Stats service.
 
+```ts
+interface Metric {
+  group: string,
+  id: string,
+  value: number,
+  // optional limit, values which exceed the limit will fail PRs
+  limit?: number
+  // optional path, relative to the root of the repo, where config values
+  // are defined. Will be linked to in PRs which have overages.
+  limitConfigPath?: string
+}
+```
+
 Example:
 
 ```ts

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -29,7 +29,13 @@ interface Config {
   buildId: string;
 }
 
-export type CiStatsMetrics = Array<{ group: string; id: string; value: number }>;
+export type CiStatsMetrics = Array<{
+  group: string;
+  id: string;
+  value: number;
+  limit?: number;
+  limitConfigPath?: string;
+}>;
 
 function parseConfig(log: ToolingLog) {
   const configJson = process.env.KIBANA_CI_STATS_CONFIG;

packages/kbn-optimizer/limits.yml

@@ -0,0 +1,100 @@
+pageLoadAssetSize:
+  advancedSettings: 27_596
+  alerts: 106_936
+  apm: 64_385
+  apmOss: 18_996
+  beatsManagement: 188_135
+  bfetch: 41_874
+  canvas: 1_065_624
+  charts: 159_211
+  cloud: 21_076
+  console: 46_235
+  core: 692_684
+  crossClusterReplication: 65_408
+  dashboard: 374_267
+  dashboardEnhanced: 65_646
+  dashboardMode: 22_716
+  data: 1_174_121
+  dataEnhanced: 50_420
+  devTools: 38_781
+  discover: 105_147
+  discoverEnhanced: 42_730
+  embeddable: 242_753
+  embeddableEnhanced: 41_145
+  enterpriseSearch: 35_741
+  esUiShared: 326_798
+  expressions: 224_120
+  features: 31_211
+  fileUpload: 24_717
+  globalSearch: 43_548
+  globalSearchBar: 62_888
+  globalSearchProviders: 25_554
+  graph: 31_504
+  grokdebugger: 26_779
+  home: 41_661
+  indexLifecycleManagement: 107_090
+  indexManagement: 662_506
+  indexPatternManagement: 154_366
+  infra: 197_873
+  ingestManager: 415_829
+  ingestPipelines: 58_003
+  inputControlVis: 172_819
+  inspector: 148_999
+  kibanaLegacy: 107_855
+  kibanaOverview: 56_426
+  kibanaReact: 162_353
+  kibanaUtils: 198_829
+  lens: 96_624
+  licenseManagement: 41_961
+  licensing: 39_008
+  lists: 183_665
+  logstash: 53_548
+  management: 46_112
+  maps: 183_754
+  mapsLegacy: 116_961
+  mapsLegacyLicensing: 20_214
+  ml: 82_187
+  monitoring: 268_758
+  navigation: 37_413
+  newsfeed: 42_228
+  observability: 89_709
+  painlessLab: 179_892
+  regionMap: 66_098
+  remoteClusters: 51_327
+  reporting: 183_418
+  rollup: 97_204
+  savedObjects: 108_662
+  savedObjectsManagement: 100_503
+  searchprofiler: 67_224
+  security: 189_428
+  securityOss: 30_806
+  securitySolution: 622_387
+  share: 99_205
+  snapshotRestore: 79_176
+  spaces: 389_643
+  telemetry: 91_832
+  telemetryManagementSection: 52_443
+  tileMap: 65_337
+  timelion: 29_920
+  transform: 41_151
+  triggersActionsUi: 170_145
+  uiActions: 95_074
+  uiActionsEnhanced: 349_799
+  upgradeAssistant: 80_966
+  uptime: 40_825
+  urlDrilldown: 34_174
+  urlForwarding: 32_579
+  usageCollection: 39_762
+  visDefaultEditor: 50_178
+  visTypeMarkdown: 30_896
+  visTypeMetric: 42_790
+  visTypeTable: 95_078
+  visTypeTagcloud: 37_575
+  visTypeTimelion: 51_933
+  visTypeTimeseries: 155_347
+  visTypeVega: 153_861
+  visTypeVislib: 242_982
+  visTypeXy: 20_255
+  visualizations: 295_169
+  visualize: 57_433
+  watcher: 43_742

packages/kbn-optimizer/package.json

@@ -22,6 +22,7 @@
     "cpy": "^8.0.0",
     "core-js": "^3.6.5",
     "css-loader": "^3.4.2",
+    "dedent": "^0.7.0",
     "del": "^5.1.0",
     "execa": "^4.0.2",
     "file-loader": "^4.2.0",
@@ -38,6 +39,7 @@
     "postcss-loader": "^3.0.0",
     "raw-loader": "^3.1.0",
     "rxjs": "^6.5.5",
+    "js-yaml": "^3.14.0",
     "sass-loader": "^8.0.2",
     "source-map-support": "^0.5.19",
     "style-loader": "^1.1.3",

packages/kbn-optimizer/src/cli.ts

@@ -28,6 +28,7 @@ import { logOptimizerState } from './log_optimizer_state';
 import { OptimizerConfig } from './optimizer';
 import { reportOptimizerStats } from './report_optimizer_stats';
 import { runOptimizer } from './run_optimizer';
+import { validateLimitsForAllBundles, updateBundleLimits } from './limits';
 
 run(
   async ({ log, flags }) => {
@@ -93,14 +94,24 @@ run(
       throw createFlagError('expected --filter to be one or more strings');
     }
 
+    const validateLimits = flags['validate-limits'] ?? false;
+    if (typeof validateLimits !== 'boolean') {
+      throw createFlagError('expected --validate-limits to have no value');
+    }
+
+    const updateLimits = flags['update-limits'] ?? false;
+    if (typeof updateLimits !== 'boolean') {
+      throw createFlagError('expected --update-limits to have no value');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
       maxWorkerCount,
-      oss,
-      dist,
+      oss: oss && !(validateLimits || updateLimits),
+      dist: dist || updateLimits,
       cache,
-      examples,
+      examples: examples && !(validateLimits || updateLimits),
       profileWebpack,
       extraPluginScanDirs,
       inspectWorkers,
@@ -108,6 +119,11 @@ run(
       filter,
     });
 
+    if (validateLimits) {
+      validateLimitsForAllBundles(log, config);
+      return;
+    }
+
     let update$ = runOptimizer(config);
 
     if (reportStats) {
@@ -121,6 +137,10 @@ run(
     }
 
     await update$.pipe(logOptimizerState(log, config)).toPromise();
+
+    if (updateLimits) {
+      updateBundleLimits(log, config);
+    }
   },
   {
     flags: {
@@ -134,6 +154,8 @@ run(
         'profile',
         'inspect-workers',
         'report-stats',
+        'validate-limits',
+        'update-limits',
       ],
       string: ['workers', 'scan-dir', 'filter'],
       default: {
@@ -152,10 +174,12 @@ run(
         --no-cache         disable the cache
         --filter           comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported
         --no-examples      don't build the example plugins
-        --dist             create bundles that are suitable for inclusion in the Kibana distributable
+        --dist             create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)
         --no-inspect-workers  when inspecting the parent process, don't inspect the workers
         --report-stats     attempt to report stats about this execution of the build to the kibana-ci-stats service using this name
+        --validate-limits  validate the limits.yml config to ensure that there are limits defined for every bundle
+        --update-limits    run a build and rewrite the limits file to include the current bundle sizes +5kb
       `,
     },
   }

packages/kbn-optimizer/src/index.ts

@@ -22,3 +22,4 @@ export * from './run_optimizer';
 export * from './log_optimizer_state';
 export * from './report_optimizer_stats';
 export * from './node';
+export * from './limits';

packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap

@@ -57,6 +57,7 @@ OptimizerConfig {
   "cache": true,
   "dist": false,
   "inspectWorkers": false,
+  "limits": "<Limits>",
   "maxWorkerCount": 1,
   "plugins": Array [
     Object {

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -27,7 +27,13 @@ import del from 'del';
 import { toArray, tap, filter } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/utils';
 import { ToolingLog } from '@kbn/dev-utils';
-import { runOptimizer, OptimizerConfig, OptimizerUpdate, logOptimizerState } from '@kbn/optimizer';
+import {
+  runOptimizer,
+  OptimizerConfig,
+  OptimizerUpdate,
+  logOptimizerState,
+  readLimits,
+} from '@kbn/optimizer';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
@@ -72,6 +78,9 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     dist: false,
   });
 
+  expect(config.limits).toEqual(readLimits());
+  (config as any).limits = '<Limits>';
+
   expect(config).toMatchSnapshot('OptimizerConfig');
 
   const msgs = await runOptimizer(config)

packages/kbn-optimizer/src/limits.ts

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import dedent from 'dedent';
+import Yaml from 'js-yaml';
+import { createFailError, ToolingLog } from '@kbn/dev-utils';
+
+import { OptimizerConfig, getMetrics } from './optimizer';
+
+const LIMITS_PATH = require.resolve('../limits.yml');
+const DEFAULT_BUDGET = 15000;
+
+const diff = <T>(a: T[], b: T[]): T[] => a.filter((item) => !b.includes(item));
+
+export function readLimits() {
+  return Yaml.safeLoad(Fs.readFileSync(LIMITS_PATH, 'utf8'));
+}
+
+export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) {
+  const limitBundleIds = Object.keys(config.limits.pageLoadAssetSize);
+  const configBundleIds = config.bundles.map((b) => b.id);
+
+  const missingBundleIds = diff(configBundleIds, limitBundleIds);
+  const extraBundleIds = diff(limitBundleIds, configBundleIds);
+
+  const issues = [];
+  if (missingBundleIds.length) {
+    issues.push(`missing: ${missingBundleIds.join(', ')}`);
+  }
+  if (extraBundleIds.length) {
+    issues.push(`extra: ${extraBundleIds.join(', ')}`);
+  }
+  if (issues.length) {
+    throw createFailError(
+      dedent`
+        The limits defined in packages/kbn-optimizer/limits.yml are outdated. Please update
+        this file with a limit (in bytes) for every production bundle.
+
+          ${issues.join('\n          ')}
+
+        To validate your changes locally, run:
+
+          node scripts/build_kibana_platform_plugins.js --validate-limits
+      ` + '\n'
+    );
+  }
+
+  log.success('limits.yml file valid');
+}
+
+export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) {
+  const metrics = getMetrics(log, config);
+
+  const number = (input: number) => input.toLocaleString('en').split(',').join('_');
+
+  let yaml = `pageLoadAssetSize:\n`;
+  for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) {
+    if (metric.group === 'page load bundle size') {
+      yaml += `  ${metric.id}: ${number(metric.value + DEFAULT_BUDGET)}\n`;
+    }
+  }
+
+  Fs.writeFileSync(LIMITS_PATH, yaml);
+  log.success(`wrote updated limits to ${LIMITS_PATH}`);
+}

packages/kbn-optimizer/src/optimizer/get_output_stats.ts

@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+import Path from 'path';
+
+import { ToolingLog, CiStatsMetrics } from '@kbn/dev-utils';
+import { OptimizerConfig } from './optimizer_config';
+
+const flatten = <T>(arr: Array<T | T[]>): T[] =>
+  arr.reduce((acc: T[], item) => acc.concat(item), []);
+
+interface Entry {
+  relPath: string;
+  stats: Fs.Stats;
+}
+
+const IGNORED_EXTNAME = ['.map', '.br', '.gz'];
+
+const getFiles = (dir: string, parent?: string) =>
+  flatten(
+    Fs.readdirSync(dir).map((name): Entry | Entry[] => {
+      const absPath = Path.join(dir, name);
+      const relPath = parent ? Path.join(parent, name) : name;
+      const stats = Fs.statSync(absPath);
+
+      if (stats.isDirectory()) {
+        return getFiles(absPath, relPath);
+      }
+
+      return {
+        relPath,
+        stats,
+      };
+    })
+  ).filter((file) => {
+    const filename = Path.basename(file.relPath);
+    if (filename.startsWith('.')) {
+      return false;
+    }
+
+    const ext = Path.extname(filename);
+    if (IGNORED_EXTNAME.includes(ext)) {
+      return false;
+    }
+
+    return true;
+  });
+
+export function getMetrics(log: ToolingLog, config: OptimizerConfig) {
+  return flatten(
+    config.bundles.map((bundle) => {
+      // make the cache read from the cache file since it was likely updated by the worker
+      bundle.cache.refresh();
+
+      const outputFiles = getFiles(bundle.outputDir);
+      const entryName = `${bundle.id}.${bundle.type}.js`;
+      const entry = outputFiles.find((f) => f.relPath === entryName);
+      if (!entry) {
+        throw new Error(
+          `Unable to find bundle entry named [${entryName}] in [${bundle.outputDir}]`
+        );
+      }
+
+      const chunkPrefix = `${bundle.id}.chunk.`;
+      const asyncChunks = outputFiles.filter((f) => f.relPath.startsWith(chunkPrefix));
+      const miscFiles = outputFiles.filter((f) => f !== entry && !asyncChunks.includes(f));
+
+      if (asyncChunks.length) {
+        log.verbose(bundle.id, 'async chunks', asyncChunks);
+      }
+      if (miscFiles.length) {
+        log.verbose(bundle.id, 'misc files', asyncChunks);
+      }
+
+      const sumSize = (files: Entry[]) => files.reduce((acc: number, f) => acc + f.stats!.size, 0);
+
+      const bundleMetrics: CiStatsMetrics = [
+        {
+          group: `@kbn/optimizer bundle module count`,
+          id: bundle.id,
+          value: bundle.cache.getModuleCount() || 0,
+        },
+        {
+          group: `page load bundle size`,
+          id: bundle.id,
+          value: entry.stats!.size,
+          limit: config.limits.pageLoadAssetSize[bundle.id],
+          limitConfigPath: `packages/kbn-optimizer/limits.yml`,
+        },
+        {
+          group: `async chunks size`,
+          id: bundle.id,
+          value: sumSize(asyncChunks),
+        },
+        {
+          group: `miscellaneous assets size`,
+          id: bundle.id,
+          value: sumSize(miscFiles),
+        },
+      ];
+
+      log.debug(bundle.id, 'metrics', bundleMetrics);
+
+      return bundleMetrics;
+    })
+  );
+}

packages/kbn-optimizer/src/optimizer/index.ts

@@ -25,3 +25,4 @@ export * from './watch_bundles_for_changes';
 export * from './run_workers';
 export * from './bundle_cache';
 export * from './handle_optimizer_completion';
+export * from './get_output_stats';

packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts

@@ -22,6 +22,7 @@ jest.mock('./kibana_platform_plugins.ts');
 jest.mock('./get_plugin_bundles.ts');
 jest.mock('../common/theme_tags.ts');
 jest.mock('./filter_by_id.ts');
+jest.mock('../limits.ts');
 
 jest.mock('os', () => {
   const realOs = jest.requireActual('os');
@@ -385,6 +386,7 @@ describe('OptimizerConfig::create()', () => {
     .findKibanaPlatformPlugins;
   const getPluginBundles: jest.Mock = jest.requireMock('./get_plugin_bundles.ts').getPluginBundles;
   const filterById: jest.Mock = jest.requireMock('./filter_by_id.ts').filterById;
+  const readLimits: jest.Mock = jest.requireMock('../limits.ts').readLimits;
 
   beforeEach(() => {
     if ('mock' in OptimizerConfig.parseOptions) {
@@ -398,6 +400,7 @@ describe('OptimizerConfig::create()', () => {
     findKibanaPlatformPlugins.mockReturnValue(Symbol('new platform plugins'));
     getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]);
     filterById.mockReturnValue(Symbol('filtered bundles'));
+    readLimits.mockReturnValue(Symbol('limits'));
 
     jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): {
       [key in keyof ParsedOptions]: any;
@@ -429,6 +432,7 @@ describe('OptimizerConfig::create()', () => {
         "cache": Symbol(parsed cache),
         "dist": Symbol(parsed dist),
         "inspectWorkers": Symbol(parsed inspect workers),
+        "limits": Symbol(limits),
         "maxWorkerCount": Symbol(parsed max worker count),
         "plugins": Symbol(new platform plugins),
         "profileWebpack": Symbol(parsed profile webpack),

packages/kbn-optimizer/src/optimizer/optimizer_config.ts

@@ -32,6 +32,13 @@ import {
 import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins';
 import { getPluginBundles } from './get_plugin_bundles';
 import { filterById } from './filter_by_id';
+import { readLimits } from '../limits';
+
+export interface Limits {
+  pageLoadAssetSize: {
+    [id: string]: number | undefined;
+  };
+}
 
 function pickMaxWorkerCount(dist: boolean) {
   // don't break if cpus() returns nothing, or an empty array
@@ -238,7 +245,8 @@ export class OptimizerConfig {
       options.maxWorkerCount,
       options.dist,
       options.profileWebpack,
-      options.themeTags
+      options.themeTags,
+      readLimits()
     );
   }
 
@@ -252,7 +260,8 @@ export class OptimizerConfig {
     public readonly maxWorkerCount: number,
     public readonly dist: boolean,
     public readonly profileWebpack: boolean,
-    public readonly themeTags: ThemeTags
+    public readonly themeTags: ThemeTags,
+    public readonly limits: Limits
   ) {}
 
   getWorkerConfig(optimizerCacheKey: unknown): WorkerConfig {

packages/kbn-optimizer/src/report_optimizer_stats.ts

@@ -17,136 +17,41 @@
  * under the License.
  */
 
-import Fs from 'fs';
-import Path from 'path';
-
 import { materialize, mergeMap, dematerialize } from 'rxjs/operators';
-import { CiStatsReporter, CiStatsMetrics, ToolingLog } from '@kbn/dev-utils';
+import { CiStatsReporter, ToolingLog } from '@kbn/dev-utils';
 
 import { OptimizerUpdate$ } from './run_optimizer';
-import { OptimizerState, OptimizerConfig } from './optimizer';
+import { OptimizerConfig, getMetrics } from './optimizer';
 import { pipeClosure } from './common';
 
-const flatten = <T>(arr: Array<T | T[]>): T[] =>
-  arr.reduce((acc: T[], item) => acc.concat(item), []);
-
-interface Entry {
-  relPath: string;
-  stats: Fs.Stats;
-}
-
-const IGNORED_EXTNAME = ['.map', '.br', '.gz'];
-
-const getFiles = (dir: string, parent?: string) =>
-  flatten(
-    Fs.readdirSync(dir).map((name): Entry | Entry[] => {
-      const absPath = Path.join(dir, name);
-      const relPath = parent ? Path.join(parent, name) : name;
-      const stats = Fs.statSync(absPath);
-
-      if (stats.isDirectory()) {
-        return getFiles(absPath, relPath);
-      }
-
-      return {
-        relPath,
-        stats,
-      };
-    })
-  ).filter((file) => {
-    const filename = Path.basename(file.relPath);
-    if (filename.startsWith('.')) {
-      return false;
-    }
-
-    const ext = Path.extname(filename);
-    if (IGNORED_EXTNAME.includes(ext)) {
-      return false;
-    }
-
-    return true;
-  });
-
 export function reportOptimizerStats(
   reporter: CiStatsReporter,
   config: OptimizerConfig,
   log: ToolingLog
 ) {
-  return pipeClosure((update$: OptimizerUpdate$) => {
-    let lastState: OptimizerState | undefined;
-    return update$.pipe(
+  return pipeClosure((update$: OptimizerUpdate$) =>
+    update$.pipe(
       materialize(),
       mergeMap(async (n) => {
-        if (n.kind === 'N' && n.value?.state) {
-          lastState = n.value?.state;
-        }
+        if (n.kind === 'C') {
+          const metrics = getMetrics(log, config);
 
-        if (n.kind === 'C' && lastState) {
-          await reporter.metrics(
-            flatten(
-              config.bundles.map((bundle) => {
-                // make the cache read from the cache file since it was likely updated by the worker
-                bundle.cache.refresh();
+          await reporter.metrics(metrics);
 
-                const outputFiles = getFiles(bundle.outputDir);
-                const entryName = `${bundle.id}.${bundle.type}.js`;
-                const entry = outputFiles.find((f) => f.relPath === entryName);
-                if (!entry) {
-                  throw new Error(
-                    `Unable to find bundle entry named [${entryName}] in [${bundle.outputDir}]`
-                  );
-                }
-
-                const chunkPrefix = `${bundle.id}.chunk.`;
-                const asyncChunks = outputFiles.filter((f) => f.relPath.startsWith(chunkPrefix));
-                const miscFiles = outputFiles.filter(
-                  (f) => f !== entry && !asyncChunks.includes(f)
-                );
-
-                if (asyncChunks.length) {
-                  log.verbose(bundle.id, 'async chunks', asyncChunks);
-                }
-                if (miscFiles.length) {
-                  log.verbose(bundle.id, 'misc files', asyncChunks);
-                }
-
-                const sumSize = (files: Entry[]) =>
-                  files.reduce((acc: number, f) => acc + f.stats!.size, 0);
-
-                const metrics: CiStatsMetrics = [
-                  {
-                    group: `@kbn/optimizer bundle module count`,
-                    id: bundle.id,
-                    value: bundle.cache.getModuleCount() || 0,
-                  },
-                  {
-                    group: `page load bundle size`,
-                    id: bundle.id,
-                    value: entry.stats!.size,
-                  },
-                  {
-                    group: `async chunks size`,
-                    id: bundle.id,
-                    value: sumSize(asyncChunks),
-                  },
-                  {
-                    group: `miscellaneous assets size`,
-                    id: bundle.id,
-                    value: sumSize(miscFiles),
-                  },
-                ];
-
-                log.info(bundle.id, 'metrics', metrics);
-
-                return metrics;
-              })
-            )
-          );
+          for (const metric of metrics) {
+            if (metric.limit != null && metric.value > metric.limit) {
+              const value = metric.value.toLocaleString();
+              const limit = metric.limit.toLocaleString();
+              log.warning(
+                `Metric [${metric.group}] for [${metric.id}] of [${value}] over the limit of [${limit}]`
+              );
+            }
+          }
         }
 
         return n;
       }),
       dematerialize()
-    );
-  });
+    )
+  );
 }

test/scripts/checks/bundle_limits.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+node scripts/build_kibana_platform_plugins --validate-limits

vars/githubPr.groovy

@@ -87,15 +87,6 @@ def getLatestBuildInfo(comment) {
   return comment ? getBuildInfoFromComment(comment.body) : null
 }
 
-def createBuildInfo() {
-  return [
-    status: buildUtils.getBuildStatus(),
-    url: env.BUILD_URL,
-    number: env.BUILD_NUMBER,
-    commit: getCommitHash()
-  ]
-}
-
 def getHistoryText(builds) {
   if (!builds || builds.size() < 1) {
     return ""
@@ -155,6 +146,16 @@ def getTestFailuresMessage() {
   return messages.join("\n")
 }
 
+def getBuildStatusIncludingMetrics() {
+  def status = buildUtils.getBuildStatus()
+
+  if (status == 'SUCCESS' && !ciStats.getMetricsSuccess()) {
+    return 'FAILURE'
+  }
+
+  return status
+}
+
 def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) {
   def info = previousCommentInfo ?: [:]
   info.builds = previousCommentInfo.builds ?: []
@@ -163,7 +164,10 @@ def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) {
   info.builds = info.builds.findAll { it.number != env.BUILD_NUMBER }
 
   def messages = []
-  def status = buildUtils.getBuildStatus()
+
+  def status = isFinal
+    ? getBuildStatusIncludingMetrics()
+    : buildUtils.getBuildStatus()
 
   if (!isFinal) {
     def failuresPart = status != 'SUCCESS' ? ', with failures' : ''
@@ -228,7 +232,12 @@ def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) {
 
   messages << "To update your PR or re-run it, just comment with:\n`@elasticmachine merge upstream`"
 
-  info.builds << createBuildInfo()
+  info.builds << [
+    status: status,
+    url: env.BUILD_URL,
+    number: env.BUILD_NUMBER,
+    commit: getCommitHash()
+  ]
 
   messages << """
     <!--PIPELINE

vars/tasks.groovy

@@ -8,6 +8,7 @@ def check() {
     kibanaPipeline.scriptTask('Check TypeScript Projects', 'test/scripts/checks/ts_projects.sh'),
     kibanaPipeline.scriptTask('Check Doc API Changes', 'test/scripts/checks/doc_api_changes.sh'),
     kibanaPipeline.scriptTask('Check Types', 'test/scripts/checks/type_check.sh'),
+    kibanaPipeline.scriptTask('Check Bundle Limits', 'test/scripts/checks/bundle_limits.sh'),
     kibanaPipeline.scriptTask('Check i18n', 'test/scripts/checks/i18n.sh'),
     kibanaPipeline.scriptTask('Check File Casing', 'test/scripts/checks/file_casing.sh'),
     kibanaPipeline.scriptTask('Check Lockfile Symlinks', 'test/scripts/checks/lock_file_symlinks.sh'),