chore(NA): merge and solve conflicts with master (#54782)

package.json

@@ -251,6 +251,7 @@
     "rison-node": "1.0.2",
     "rxjs": "^6.5.3",
     "script-loader": "0.7.2",
+    "seedrandom": "^3.0.5",
     "semver": "^5.5.0",
     "style-loader": "0.23.1",
     "symbol-observable": "^1.2.0",

src/dev/build/README.md

@@ -44,7 +44,7 @@ The majority of this logic is extracted from the grunt build that has existed fo
 
 We have introduced in our bundle a webpack dll for the client vendor modules in order to improve
 the optimization time both in dev and in production. As for those modules we already have the 
-code into the vendors.bundle.dll.js we have decided to delete those bundled modules from the 
+code into the vendors_${chunk_number}.bundle.dll.js we have decided to delete those bundled modules from the 
 distributable node_modules folder. However, in order to accomplish this, we need to exclude 
 every node_module used in the server side code. This logic is performed 
 under `nodejs_modules/clean_client_modules_on_dll_task.js`. In case we need to add any new cli

src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js

@@ -98,12 +98,16 @@ export const CleanClientModulesOnDLLTask = {
     // Consider this as our whiteList for the modules we can't delete
     const whiteListedModules = [...serverDependencies, ...kbnWebpackLoaders, ...manualExceptions];
 
-    // Resolve the client vendors dll manifest path
-    const dllManifestPath = `${baseDir}/built_assets/dlls/vendors.manifest.dll.json`;
+    // Resolve the client vendors dll manifest paths
+    // excluding the runtime one
+    const dllManifestPaths = await globby([
+      `${baseDir}/built_assets/dlls/vendors_*.manifest.dll.json`,
+      `!${baseDir}/built_assets/dlls/vendors_runtime.manifest.dll.json`,
+    ]);
 
     // Get dll entries filtering out the ones
     // from any whitelisted module
-    const dllEntries = await getDllEntries(dllManifestPath, whiteListedModules, baseDir);
+    const dllEntries = await getDllEntries(dllManifestPaths, whiteListedModules, baseDir);
 
     for (const relativeEntryPath of dllEntries) {
       const entryPath = `${baseDir}/${relativeEntryPath}`;

src/dev/build/tasks/nodejs_modules/webpack_dll.js

@@ -28,27 +28,37 @@ function checkDllEntryAccess(entry, baseDir = '') {
   return isFileAccessible(resolvedPath);
 }
 
-export async function getDllEntries(manifestPath, whiteListedModules, baseDir = '') {
-  const manifest = JSON.parse(await read(manifestPath));
+export async function getDllEntries(manifestPaths, whiteListedModules, baseDir = '') {
+  // Read and parse all manifests
+  const manifests = await Promise.all(
+    manifestPaths.map(async manifestPath => JSON.parse(await read(manifestPath)))
+  );
 
-  if (!manifest || !manifest.content) {
-    // It should fails because if we don't have the manifest file
-    // or it is malformed something wrong is happening and we
-    // should stop
-    throw new Error(`The following dll manifest doesn't exists: ${manifestPath}`);
-  }
+  // Process and group modules from all manifests
+  const manifestsModules = manifests.flatMap((manifest, idx) => {
+    if (!manifest || !manifest.content) {
+      // It should fails because if we don't have the manifest file
+      // or it is malformed something wrong is happening and we
+      // should stop
+      throw new Error(`The following dll manifest doesn't exists: ${manifestPaths[idx]}`);
+    }
 
-  const modules = Object.keys(manifest.content);
-  if (!modules.length) {
-    // It should fails because if we don't have any
-    // module inside the client vendors dll something
-    // wrong is happening and we should stop too
-    throw new Error(`The following dll manifest is reporting an empty dll: ${manifestPath}`);
-  }
+    const modules = Object.keys(manifest.content);
+    if (!modules.length) {
+      // It should fails because if we don't have any
+      // module inside the client vendors dll something
+      // wrong is happening and we should stop too
+      throw new Error(
+        `The following dll manifest is reporting an empty dll: ${manifestPaths[idx]}`
+      );
+    }
+
+    return modules;
+  });
 
   // Only includes modules who are not in the white list of modules
   // and that are node_modules
-  return modules.filter(entry => {
+  return manifestsModules.filter(entry => {
     const isWhiteListed = whiteListedModules.some(nonEntry =>
       normalizePosixPath(entry).includes(`node_modules/${nonEntry}`)
     );

src/dev/build/tasks/nodejs_modules/webpack_dll.test.js

@@ -52,7 +52,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
 
     isFileAccessible.mockImplementation(() => true);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
     const mockModulesWhitelist = ['dep1'];
     const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
 
@@ -66,7 +66,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
 
     isFileAccessible.mockImplementation(() => false);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
     const mockModulesWhitelist = ['dep1'];
     const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
 
@@ -78,7 +78,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for no manifest file', async () => {
     read.mockImplementationOnce(async () => noManifestMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);
@@ -92,7 +92,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for no manifest content field', async () => {
     read.mockImplementation(async () => noContentFieldManifestMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);
@@ -106,7 +106,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for manifest file without any content', async () => {
     read.mockImplementation(async () => emptyManifestContentMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);

src/legacy/ui/ui_render/bootstrap/template.js.hbs

@@ -13,7 +13,10 @@ if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) {
 
   window.onload = function () {
     var files = [
-      '{{dllBundlePath}}/vendors.bundle.dll.js',
+      '{{dllBundlePath}}/vendors_runtime.bundle.dll.js',
+      {{#each dllJsChunks}}
+        '{{this}}',
+      {{/each}}
       '{{regularBundlePath}}/kbn-ui-shared-deps/{{sharedDepsFilename}}',
       '{{regularBundlePath}}/commons.bundle.js',
       '{{regularBundlePath}}/{{appId}}.bundle.js'

src/legacy/ui/ui_render/ui_render_mixin.js

@@ -25,6 +25,7 @@ import * as UiSharedDeps from '@kbn/ui-shared-deps';
 import { AppBootstrap } from './bootstrap';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 import { fromRoot } from '../../../core/server/utils';
+import { DllCompiler } from '../../../optimize/dynamic_dll_plugin';
 
 /**
  * @typedef {import('../../server/kbn_server').default} KbnServer
@@ -102,8 +103,14 @@ export function uiRenderMixin(kbnServer, server, config) {
         const basePath = config.get('server.basePath');
         const regularBundlePath = `${basePath}/bundles`;
         const dllBundlePath = `${basePath}/built_assets/dlls`;
+        const dllStyleChunks = DllCompiler.getRawDllConfig().chunks.map(
+          chunk => `${dllBundlePath}/vendors${chunk}.style.dll.css`
+        );
+        const dllJsChunks = DllCompiler.getRawDllConfig().chunks.map(
+          chunk => `${dllBundlePath}/vendors${chunk}.bundle.dll.js`
+        );
         const styleSheetPaths = [
-          `${dllBundlePath}/vendors.style.dll.css`,
+          ...dllStyleChunks,
           ...(darkMode
             ? [
                 `${basePath}/bundles/kbn-ui-shared-deps/${UiSharedDeps.darkCssDistFilename}`,
@@ -131,6 +138,7 @@ export function uiRenderMixin(kbnServer, server, config) {
             appId: isCore ? 'core' : app.getId(),
             regularBundlePath,
             dllBundlePath,
+            dllJsChunks,
             styleSheetPaths,
             sharedDepsFilename: UiSharedDeps.distFilename,
           },

src/optimize/dynamic_dll_plugin/dll_compiler.js

@@ -23,6 +23,11 @@ import {
   notInNodeModules,
   inDllPluginPublic,
 } from './dll_allowed_modules';
+import {
+  dllEntryFileContentArrayToString,
+  dllEntryFileContentStringToArray,
+  dllMergeAllEntryFilesContent,
+} from './dll_entry_template';
 import { fromRoot } from '../../core/server/utils';
 import { PUBLIC_PATH_PLACEHOLDER } from '../public_path_placeholder';
 import fs from 'fs';
@@ -30,6 +35,8 @@ import webpack from 'webpack';
 import { promisify } from 'util';
 import path from 'path';
 import del from 'del';
+import { chunk } from 'lodash';
+import seedrandom from 'seedrandom';
 
 const readFileAsync = promisify(fs.readFile);
 const mkdirAsync = promisify(fs.mkdir);
@@ -37,11 +44,17 @@ const accessAsync = promisify(fs.access);
 const writeFileAsync = promisify(fs.writeFile);
 
 export class DllCompiler {
-  static getRawDllConfig(uiBundles = {}, babelLoaderCacheDir = '', threadLoaderPoolConfig = {}) {
+  static getRawDllConfig(
+    uiBundles = {},
+    babelLoaderCacheDir = '',
+    threadLoaderPoolConfig = {},
+    chunks = Array.from(Array(4).keys()).map(chunkN => `_${chunkN}`)
+  ) {
     return {
       uiBundles,
       babelLoaderCacheDir,
       threadLoaderPoolConfig,
+      chunks,
       context: fromRoot('.'),
       entryName: 'vendors',
       dllName: '[name]',
@@ -66,13 +79,49 @@ export class DllCompiler {
   }
 
   async init() {
-    await this.ensureEntryFileExists();
-    await this.ensureManifestFileExists();
+    await this.ensureEntryFilesExists();
+    await this.ensureManifestFilesExists();
     await this.ensureOutputPathExists();
   }
 
-  async upsertEntryFile(content) {
-    await this.upsertFile(this.getEntryPath(), content);
+  seededShuffle(array) {
+    // Implementation based on https://github.com/TimothyGu/knuth-shuffle-seeded/blob/gh-pages/index.js#L46
+    let currentIndex;
+    let temporaryValue;
+    let randomIndex;
+    const rand = seedrandom('predictable', { global: false });
+
+    if (array.constructor !== Array) throw new Error('Input is not an array');
+    currentIndex = array.length;
+
+    // While there remain elements to shuffle...
+    while (0 !== currentIndex) {
+      // Pick a remaining element...
+      randomIndex = Math.floor(rand() * currentIndex--);
+
+      // And swap it with the current element.
+      temporaryValue = array[currentIndex];
+      array[currentIndex] = array[randomIndex];
+      array[randomIndex] = temporaryValue;
+    }
+
+    return array;
+  }
+
+  async upsertEntryFiles(content) {
+    const arrayContent = this.seededShuffle(dllEntryFileContentStringToArray(content));
+    const chunks = chunk(
+      arrayContent,
+      Math.ceil(arrayContent.length / this.rawDllConfig.chunks.length)
+    );
+    const entryPaths = this.getEntryPaths();
+
+    await Promise.all(
+      entryPaths.map(
+        async (entryPath, idx) =>
+          await this.upsertFile(entryPath, dllEntryFileContentArrayToString(chunks[idx]))
+      )
+    );
   }
 
   async upsertFile(filePath, content = '') {
@@ -80,38 +129,57 @@ export class DllCompiler {
     await writeFileAsync(filePath, content, 'utf8');
   }
 
-  getDllPath() {
-    return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.dllExt}`);
+  getDllPaths() {
+    return this.rawDllConfig.chunks.map(chunk =>
+      this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.dllExt}`)
+    );
   }
 
-  getEntryPath() {
-    return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.entryExt}`);
+  getEntryPaths() {
+    return this.rawDllConfig.chunks.map(chunk =>
+      this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.entryExt}`)
+    );
   }
 
-  getManifestPath() {
-    return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.manifestExt}`);
+  getManifestPaths() {
+    return this.rawDllConfig.chunks.map(chunk =>
+      this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.manifestExt}`)
+    );
   }
 
-  getStylePath() {
-    return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.styleExt}`);
+  getStylePaths() {
+    return this.rawDllConfig.chunks.map(chunk =>
+      this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.styleExt}`)
+    );
   }
 
-  async ensureEntryFileExists() {
-    await this.ensureFileExists(this.getEntryPath());
+  async ensureEntryFilesExists() {
+    const entryPaths = this.getEntryPaths();
+
+    await Promise.all(entryPaths.map(async entryPath => await this.ensureFileExists(entryPath)));
   }
 
-  async ensureManifestFileExists() {
-    await this.ensureFileExists(
-      this.getManifestPath(),
-      JSON.stringify({
-        name: this.rawDllConfig.entryName,
-        content: {},
-      })
+  async ensureManifestFilesExists() {
+    const manifestPaths = this.getManifestPaths();
+
+    await Promise.all(
+      manifestPaths.map(
+        async (manifestPath, idx) =>
+          await this.ensureFileExists(
+            manifestPath,
+            JSON.stringify({
+              name: `${this.rawDllConfig.entryName}${this.rawDllConfig.chunks[idx]}`,
+              content: {},
+            })
+          )
+      )
     );
   }
 
   async ensureStyleFileExists() {
-    await this.ensureFileExists(this.getStylePath());
+    const stylePaths = this.getStylePaths();
+
+    await Promise.all(stylePaths.map(async stylePath => await this.ensureFileExists(stylePath)));
   }
 
   async ensureFileExists(filePath, content) {
@@ -137,8 +205,10 @@ export class DllCompiler {
     await this.ensurePathExists(this.rawDllConfig.outputPath);
   }
 
-  dllExistsSync() {
-    return this.existsSync(this.getDllPath());
+  dllsExistsSync() {
+    const dllPaths = this.getDllPaths();
+
+    return dllPaths.every(dllPath => this.existsSync(dllPath));
   }
 
   existsSync(filePath) {
@@ -149,8 +219,16 @@ export class DllCompiler {
     return path.resolve(this.rawDllConfig.outputPath, ...arguments);
   }
 
-  async readEntryFile() {
-    return await this.readFile(this.getEntryPath());
+  async readEntryFiles() {
+    const entryPaths = this.getEntryPaths();
+
+    const entryFilesContent = await Promise.all(
+      entryPaths.map(async entryPath => await this.readFile(entryPath))
+    );
+
+    // merge all the module contents from entry files again into
+    // sorted single one
+    return dllMergeAllEntryFilesContent(entryFilesContent);
   }
 
   async readFile(filePath, content) {
@@ -160,7 +238,7 @@ export class DllCompiler {
 
   async run(dllEntries) {
     const dllConfig = this.dllConfigGenerator(this.rawDllConfig);
-    await this.upsertEntryFile(dllEntries);
+    await this.upsertEntryFiles(dllEntries);
 
     try {
       this.logWithMetadata(
@@ -234,7 +312,7 @@ export class DllCompiler {
 
           // ignore if this module represents the
           // dll entry file
-          if (module.resource === this.getEntryPath()) {
+          if (this.getEntryPaths().includes(module.resource)) {
             return;
           }
 
@@ -259,7 +337,6 @@ export class DllCompiler {
               // node_module or no?
               if (notInNodeModules(reason.module.resource)) {
                 notAllowedModules.push(module.resource);
-                return;
               }
             });
           }

src/optimize/dynamic_dll_plugin/dll_config_model.js

@@ -140,6 +140,13 @@ function generateDLL(config) {
         filename: dllStyleFilename,
       }),
     ],
+    // Single runtime for the dll bundles which assures that common transient dependencies won't be evaluated twice.
+    // The module cache will be shared, even when module code may be duplicated across chunks.
+    optimization: {
+      runtimeChunk: {
+        name: 'vendors_runtime',
+      },
+    },
     performance: {
       // NOTE: we are disabling this as those hints
       // are more tailored for the final bundles result
@@ -158,6 +165,7 @@ function extendRawConfig(rawConfig) {
   const dllNoParseRules = rawConfig.uiBundles.getWebpackNoParseRules();
   const dllDevMode = rawConfig.uiBundles.isDevMode();
   const dllContext = rawConfig.context;
+  const dllChunks = rawConfig.chunks;
   const dllEntry = {};
   const dllEntryName = rawConfig.entryName;
   const dllBundleName = rawConfig.dllName;
@@ -176,7 +184,12 @@ function extendRawConfig(rawConfig) {
   const threadLoaderPoolConfig = rawConfig.threadLoaderPoolConfig;
 
   // Create webpack entry object key with the provided dllEntryName
-  dllEntry[dllEntryName] = [`${dllOutputPath}/${dllEntryName}${dllEntryExt}`];
+  dllChunks.reduce((dllEntryObj, chunk) => {
+    dllEntryObj[`${dllEntryName}${chunk}`] = [
+      `${dllOutputPath}/${dllEntryName}${chunk}${dllEntryExt}`,
+    ];
+    return dllEntryObj;
+  }, dllEntry);
 
   // Export dll config map
   return {

src/optimize/dynamic_dll_plugin/dll_entry_template.js

@@ -23,3 +23,19 @@ export function dllEntryTemplate(requirePaths = []) {
     .sort()
     .join('\n');
 }
+
+export function dllEntryFileContentStringToArray(content = '') {
+  return content.split('\n');
+}
+
+export function dllEntryFileContentArrayToString(content = []) {
+  return content.join('\n');
+}
+
+export function dllMergeAllEntryFilesContent(content = []) {
+  return content
+    .join('\n')
+    .split('\n')
+    .sort()
+    .join('\n');
+}

src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js

@@ -44,7 +44,7 @@ export class DynamicDllPlugin {
 
   async init() {
     await this.dllCompiler.init();
-    this.entryPaths = await this.dllCompiler.readEntryFile();
+    this.entryPaths = await this.dllCompiler.readEntryFiles();
   }
 
   apply(compiler) {
@@ -70,12 +70,14 @@ export class DynamicDllPlugin {
   bindDllReferencePlugin(compiler) {
     const rawDllConfig = this.dllCompiler.rawDllConfig;
     const dllContext = rawDllConfig.context;
-    const dllManifestPath = this.dllCompiler.getManifestPath();
+    const dllManifestPaths = this.dllCompiler.getManifestPaths();
 
-    new webpack.DllReferencePlugin({
-      context: dllContext,
-      manifest: dllManifestPath,
-    }).apply(compiler);
+    dllManifestPaths.forEach(dllChunkManifestPath => {
+      new webpack.DllReferencePlugin({
+        context: dllContext,
+        manifest: dllChunkManifestPath,
+      }).apply(compiler);
+    });
   }
 
   registerInitBasicHooks(compiler) {
@@ -192,7 +194,7 @@ export class DynamicDllPlugin {
         // then will be set to false
         compilation.needsDLLCompilation =
           this.afterCompilationEntryPaths !== this.entryPaths ||
-          !this.dllCompiler.dllExistsSync() ||
+          !this.dllCompiler.dllsExistsSync() ||
           (this.isToForceDLLCreation() && this.performedCompilations === 0);
         this.entryPaths = this.afterCompilationEntryPaths;
 
@@ -337,7 +339,9 @@ export class DynamicDllPlugin {
     // We need to purge the cache into the inputFileSystem
     // for every single built in previous compilation
     // that we rely in next ones.
-    mainCompiler.inputFileSystem.purge(this.dllCompiler.getManifestPath());
+    this.dllCompiler
+      .getManifestPaths()
+      .forEach(chunkDllManifestPath => mainCompiler.inputFileSystem.purge(chunkDllManifestPath));
 
     this.performedCompilations++;
 

tasks/config/karma.js

@@ -21,6 +21,7 @@ import { dirname } from 'path';
 import { times } from 'lodash';
 import { makeJunitReportPath } from '@kbn/test';
 import * as UiSharedDeps from '@kbn/ui-shared-deps';
+import { DllCompiler } from '../../src/optimize/dynamic_dll_plugin';
 
 const TOTAL_CI_SHARDS = 4;
 const ROOT = dirname(require.resolve('../../package.json'));
@@ -54,7 +55,10 @@ module.exports = function(grunt) {
       'http://localhost:5610/test_bundle/built_css.css',
 
       `http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.distFilename}`,
-      'http://localhost:5610/built_assets/dlls/vendors.bundle.dll.js',
+      'http://localhost:5610/built_assets/dlls/vendors_runtime.bundle.dll.js',
+      ...DllCompiler.getRawDllConfig().chunks.map(
+        chunk => `http://localhost:5610/built_assets/dlls/vendors${chunk}.bundle.dll.js`
+      ),
 
       shardNum === undefined
         ? `http://localhost:5610/bundles/tests.bundle.js`
@@ -63,7 +67,9 @@ module.exports = function(grunt) {
       // this causes tilemap tests to fail, probably because the eui styles haven't been
       // included in the karma harness a long some time, if ever
       // `http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.lightCssDistFilename}`,
-      'http://localhost:5610/built_assets/dlls/vendors.style.dll.css',
+      ...DllCompiler.getRawDllConfig().chunks.map(
+        chunk => `http://localhost:5610/built_assets/dlls/vendors${chunk}.style.dll.css`
+      ),
       'http://localhost:5610/bundles/tests.style.css',
     ];
   }

yarn.lock

@@ -25895,6 +25895,11 @@ scss-tokenizer@^0.2.3:
     js-base64 "^2.1.8"
     source-map "^0.4.2"
 
+seedrandom@^3.0.5:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
+  integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
+
 seek-bzip@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.5.tgz#cfe917cb3d274bcffac792758af53173eb1fabdc"