Optimize with thread loader and terser (#27014) (#27426)

* multi thread expensive loaders

* revert styles

* feat(NA): added thread-loader and cache-loader to base optimizer and dll compiler.

* feat(NA): added cache-loader and thread-loader to the optimizer and dll compiler.

* feat(NA): use new terser plugin instead of old unmaintained uglifyjs webpack plugin.

* refact(NA): remove unused configs from base optimizer and dll compiler.

* fix(NA): available cpu calculated number.

* docs(NA): fix comment about what we are doing in prod on base_optimizer config.

* docs(NA): explain why we are setting memory into each thread loader worker.

* fix(NA): add dev only loaders to the thread-loader warmup config.

* refact(NA): change name from babelCacheDir to babelLoaderCacheDir.

* fix(NA): logic for calculating available cpus.

* feat(NA): pass NODE_OPTIONS along for the optimizer forked process and also for the optimizer workers.

* feat(NA): remove terser webpack plugin compression from base_optimizer and only run it on dll compiler.

* chore(NA): update function to calculate available cpus for works.

* fix(NA): apply upperbound to the number of workers we can have on thread-loader.

* fix(NA): decrease the max number of thread pool workers. refact(NA): use the same calculated number of cpus to use on parallel tasks on thread loader pool config for terser parallel. refact(NA): lower down the poolTimeout on non dev mode. refact(NA): change devOnlyModules to nonDistributableOnlyModules on warmupThreadLoader config.

* chore(NA): update yarn lock deps after merging with master.
This commit is contained in:
Tiago Costa 2018-12-18 20:06:43 +00:00 committed by GitHub
parent ce3d215d6a
commit d53a9a295f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 232 additions and 76 deletions

View file

@ -233,6 +233,8 @@
"stream-stream": "^1.2.6",
"style-loader": "0.23.1",
"tar": "2.2.0",
"terser-webpack-plugin": "^1.1.0",
"thread-loader": "^1.2.0",
"tinygradient": "0.3.0",
"tinymath": "1.1.1",
"topojson-client": "3.0.0",
@ -241,7 +243,6 @@
"ts-optchain": "^0.1.1",
"tslib": "^1.9.3",
"type-detect": "^4.0.8",
"uglifyjs-webpack-plugin": "^1.2.7",
"ui-select": "0.19.6",
"url-loader": "1.1.2",
"uuid": "3.0.1",

View file

@ -59,6 +59,7 @@ export default class Worker extends EventEmitter {
this.processBinder = new BinderFor(process);
this.env = {
NODE_OPTIONS: process.env.NODE_OPTIONS || '',
kbnWorkerType: this.type,
kbnWorkerArgv: JSON.stringify([
...(opts.baseArgv || baseArgv),

View file

@ -21,9 +21,10 @@ import { writeFile } from 'fs';
import os from 'os';
import Boom from 'boom';
import MiniCssExtractPlugin from 'mini-css-extract-plugin';
import UglifyJsPlugin from 'uglifyjs-webpack-plugin';
import TerserPlugin from 'terser-webpack-plugin';
import webpack from 'webpack';
import Stats from 'webpack/lib/Stats';
import * as threadLoader from 'thread-loader';
import webpackMerge from 'webpack-merge';
import { DynamicDllPlugin } from './dynamic_dll_plugin';
@ -63,6 +64,9 @@ export default class BaseOptimizer {
break;
}
// Run some pre loading in order to prevent
// high delay when booting thread loader workers
this.warmupThreadLoaderPool();
}
async init() {
@ -102,6 +106,63 @@ export default class BaseOptimizer {
});
}
warmupThreadLoaderPool() {
const baseModules = [
'babel-loader',
BABEL_PRESET_PATH
];
const nonDistributableOnlyModules = !IS_KIBANA_DISTRIBUTABLE
? ['ts-loader']
: [];
threadLoader.warmup(
// pool options, like passed to loader options
// must match loader options to boot the correct pool
this.getThreadLoaderPoolConfig(),
[
// modules to load on the pool
...baseModules,
...nonDistributableOnlyModules
]
);
}
getThreadPoolCpuCount() {
const cpus = os.cpus();
if (!cpus) {
// sometimes this call returns undefined so we fall back to 1: https://github.com/nodejs/node/issues/19022
return 1;
}
return Math.max(
1,
Math.min(
cpus.length - 1,
7
)
);
}
getThreadLoaderPoolConfig() {
// Calculate the node options from the NODE_OPTIONS env var
const parsedNodeOptions = process.env.NODE_OPTIONS ? process.env.NODE_OPTIONS.split(/\s/) : [];
return {
name: 'optimizer-thread-loader-main-pool',
workers: this.getThreadPoolCpuCount(),
workerParallelJobs: 20,
// This is a safe check in order to set
// the parent node options applied from
// the NODE_OPTIONS env var for every launched worker.
// Otherwise, if the user sets max_old_space_size, as they
// are used to, into NODE_OPTIONS, it won't affect the workers.
workerNodeArgs: parsedNodeOptions,
poolParallelJobs: this.getThreadPoolCpuCount() * 20,
poolTimeout: this.uiBundles.isDevMode() ? Infinity : 2000
};
}
getConfig() {
function getStyleLoaderExtractor() {
return [
@ -177,7 +238,6 @@ export default class BaseOptimizer {
mode: 'development',
node: { fs: 'empty' },
context: fromRoot('.'),
parallelism: os.cpus().length - 1,
cache: true,
entry: this.uiBundles.toWebpackEntries(),
@ -209,6 +269,7 @@ export default class BaseOptimizer {
plugins: [
new DynamicDllPlugin({
uiBundles: this.uiBundles,
threadLoaderPoolConfig: this.getThreadLoaderPoolConfig(),
log: this.log
}),
@ -275,6 +336,10 @@ export default class BaseOptimizer {
{
resource: createSourceFileResourceSelector(/\.js$/),
use: maybeAddCacheLoader('babel', [
{
loader: 'thread-loader',
options: this.getThreadLoaderPoolConfig()
},
{
loader: 'babel-loader',
options: {
@ -339,9 +404,14 @@ export default class BaseOptimizer {
{
resource: createSourceFileResourceSelector(/\.tsx?$/),
use: maybeAddCacheLoader('typescript', [
{
loader: 'thread-loader',
options: this.getThreadLoaderPoolConfig()
},
{
loader: 'ts-loader',
options: {
happyPackMode: true,
transpileOnly: true,
experimentalWatchApi: true,
onlyCompileBundledFiles: true,
@ -387,46 +457,17 @@ export default class BaseOptimizer {
]
};
// in production we set the process.env.NODE_ENV and uglify our bundles
// in production we set the process.env.NODE_ENV and run
// the terser minimizer over our bundles
const productionConfig = {
mode: 'production',
optimization: {
minimize: true,
minimizer: [
new UglifyJsPlugin({
new TerserPlugin({
parallel: true,
sourceMap: false,
uglifyOptions: {
compress: {
// The following is required for dead-code the removal
// check in React DevTools
//
// default
unused: true,
dead_code: true,
conditionals: true,
evaluate: true,
// changed
keep_fnames: true,
keep_infinity: true,
comparisons: false,
sequences: false,
properties: false,
drop_debugger: false,
booleans: false,
loops: false,
toplevel: false,
top_retain: false,
hoist_funs: false,
if_return: false,
join_vars: false,
collapse_vars: false,
reduce_vars: false,
warnings: false,
negate_iife: false,
side_effects: false
},
terserOptions: {
compress: false,
mangle: false
}
}),

View file

@ -32,9 +32,11 @@ const existsAsync = promisify(fs.exists);
const writeFileAsync = promisify(fs.writeFile);
export class DllCompiler {
static getRawDllConfig(uiBundles = {}) {
static getRawDllConfig(uiBundles = {}, babelLoaderCacheDir = '', threadLoaderPoolConfig = {}) {
return {
uiBundles,
babelLoaderCacheDir,
threadLoaderPoolConfig,
context: fromRoot('.'),
entryName: 'vendors',
dllName: '[name]',
@ -49,9 +51,11 @@ export class DllCompiler {
};
}
constructor(uiBundles, log) {
constructor(uiBundles, threadLoaderPoolConfig, log) {
this.rawDllConfig = DllCompiler.getRawDllConfig(
uiBundles
uiBundles,
uiBundles.getCacheDirectory('babel'),
threadLoaderPoolConfig
);
this.log = log || (() => null);
}

View file

@ -17,11 +17,11 @@
* under the License.
*/
import { fromRoot } from '../../utils';
import { fromRoot, IS_KIBANA_DISTRIBUTABLE } from '../../utils';
import webpack from 'webpack';
import webpackMerge from 'webpack-merge';
import MiniCssExtractPlugin from 'mini-css-extract-plugin';
import UglifyJsPlugin from 'uglifyjs-webpack-plugin';
import TerserPlugin from 'terser-webpack-plugin';
function generateDLL(config) {
const {
@ -34,7 +34,9 @@ function generateDLL(config) {
dllBundleName,
dllBundleFilename,
dllStyleFilename,
dllManifestPath
dllManifestPath,
babelLoaderCacheDir,
threadLoaderPoolConfig
} = config;
const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
@ -78,15 +80,41 @@ function generateDLL(config) {
exclude: /[\/\\]node_modules[\/\\]x-pack[\/\\](.+?[\/\\])*node_modules[\/\\]/,
}
],
use: {
loader: 'babel-loader',
options: {
babelrc: false,
presets: [
BABEL_PRESET_PATH,
],
// Self calling function with the equivalent logic
// from maybeAddCacheLoader one from base optimizer
use: ((babelLoaderCacheDirPath, loaders) => {
// Only deactivate cache-loader and thread-loader on
// distributable. It is valid when running from source
// both with dev or prod bundles or even when running
// kibana for dev only.
if (IS_KIBANA_DISTRIBUTABLE) {
return loaders;
}
return [
{
loader: 'cache-loader',
options: {
cacheDirectory: babelLoaderCacheDirPath
}
},
...loaders
];
})(babelLoaderCacheDir, [
{
loader: 'thread-loader',
options: threadLoaderPoolConfig
},
}
{
loader: 'babel-loader',
options: {
babelrc: false,
presets: [
BABEL_PRESET_PATH,
],
},
}
])
},
{
test: /\.(html|tmpl)$/,
@ -149,6 +177,8 @@ function extendRawConfig(rawConfig) {
const dllBundleFilename = `${dllBundleName}${dllBundleExt}`;
const dllManifestPath = `${dllOutputPath}/${dllManifestName}${dllManifestExt}`;
const dllStyleFilename = `${dllStyleName}${dllStyleExt}`;
const babelLoaderCacheDir = rawConfig.babelLoaderCacheDir;
const threadLoaderPoolConfig = rawConfig.threadLoaderPoolConfig;
// Create webpack entry object key with the provided dllEntryName
dllEntry[dllEntryName] = [
@ -167,7 +197,9 @@ function extendRawConfig(rawConfig) {
dllBundleName,
dllBundleFilename,
dllStyleFilename,
dllManifestPath
dllManifestPath,
babelLoaderCacheDir,
threadLoaderPoolConfig
};
}
@ -177,17 +209,19 @@ function common(config) {
);
}
function optimized() {
function optimized(config) {
return webpackMerge(
{
mode: 'production',
optimization: {
minimize: true,
minimizer: [
new UglifyJsPlugin({
parallel: true,
new TerserPlugin({
// Apply the same logic used to calculate the
// threadLoaderPool workers number to spawn
// the parallel processes on terser
parallel: config.threadLoaderPoolConfig.workers,
sourceMap: false,
uglifyOptions: {
terserOptions: {
compress: {
// The following is required for dead-code the removal
// check in React DevTools
@ -242,5 +276,5 @@ export function configModel(rawConfig = {}) {
return webpackMerge(common(config), unoptimized());
}
return webpackMerge(common(config), optimized());
return webpackMerge(common(config), optimized(config));
}

View file

@ -40,9 +40,9 @@ function inPluginNodeModules(checkPath) {
}
export class DynamicDllPlugin {
constructor({ uiBundles, log, maxCompilations = 1 }) {
constructor({ uiBundles, threadLoaderPoolConfig, log, maxCompilations = 1 }) {
this.log = log || (() => null);
this.dllCompiler = new DllCompiler(uiBundles, log);
this.dllCompiler = new DllCompiler(uiBundles, threadLoaderPoolConfig, log);
this.entryPaths = '';
this.afterCompilationEntryPaths = '';
this.maxCompilations = maxCompilations;

107
yarn.lock
View file

@ -2847,7 +2847,7 @@ async@^2.0.0, async@^2.1.4:
dependencies:
lodash "^4.14.0"
async@^2.5.0, async@^2.6.0, async@^2.6.1:
async@^2.3.0, async@^2.5.0, async@^2.6.0, async@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/async/-/async-2.6.1.tgz#b245a23ca71930044ec53fa46aa00a3e87c6a610"
integrity sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==
@ -4540,6 +4540,26 @@ cacache@^10.0.4:
unique-filename "^1.1.0"
y18n "^4.0.0"
cacache@^11.0.2:
version "11.3.1"
resolved "https://registry.yarnpkg.com/cacache/-/cacache-11.3.1.tgz#d09d25f6c4aca7a6d305d141ae332613aa1d515f"
integrity sha512-2PEw4cRRDu+iQvBTTuttQifacYjLPhET+SYO/gEFMy8uhi+jlJREDAjSF5FWSdV/Aw5h18caHA7vMTw2c+wDzA==
dependencies:
bluebird "^3.5.1"
chownr "^1.0.1"
figgy-pudding "^3.1.0"
glob "^7.1.2"
graceful-fs "^4.1.11"
lru-cache "^4.1.3"
mississippi "^3.0.0"
mkdirp "^0.5.1"
move-concurrently "^1.0.1"
promise-inflight "^1.0.1"
rimraf "^2.6.2"
ssri "^6.0.0"
unique-filename "^1.1.0"
y18n "^4.0.0"
cache-base@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2"
@ -8499,6 +8519,11 @@ fetch-mock@^5.13.1:
node-fetch "^1.3.3"
path-to-regexp "^1.7.0"
figgy-pudding@^3.1.0, figgy-pudding@^3.5.1:
version "3.5.1"
resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790"
integrity sha512-vNKxJHTEKNThjfrdJwHc7brvM6eVevuO5nTj6ez8ZQ1qbXTvGthucRF7S4vf2cr71QVnT70V34v0S1DyQsti0w==
figures@^1.3.5, figures@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
@ -8656,6 +8681,15 @@ find-cache-dir@^1.0.0:
make-dir "^1.0.0"
pkg-dir "^2.0.0"
find-cache-dir@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.0.0.tgz#4c1faed59f45184530fb9d7fa123a4d04a98472d"
integrity sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==
dependencies:
commondir "^1.0.1"
make-dir "^1.0.0"
pkg-dir "^3.0.0"
find-index@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/find-index/-/find-index-0.1.1.tgz#675d358b2ca3892d795a1ab47232f8b6e2e0dde4"
@ -14266,6 +14300,22 @@ mississippi@^2.0.0:
stream-each "^1.1.0"
through2 "^2.0.0"
mississippi@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
integrity sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==
dependencies:
concat-stream "^1.5.0"
duplexify "^3.4.2"
end-of-stream "^1.1.0"
flush-write-stream "^1.0.0"
from2 "^2.1.0"
parallel-transform "^1.1.0"
pump "^3.0.0"
pumpify "^1.3.3"
stream-each "^1.1.0"
through2 "^2.0.0"
mixin-deep@^1.2.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.0.tgz#47a8732ba97799457c8c1eca28f95132d7e8150a"
@ -18970,7 +19020,7 @@ source-map-support@^0.5.1:
buffer-from "^1.0.0"
source-map "^0.6.0"
source-map-support@^0.5.6:
source-map-support@^0.5.6, source-map-support@~0.5.6:
version "0.5.9"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.9.tgz#41bc953b2534267ea2d605bccfa7bfa3111ced5f"
integrity sha512-gR6Rw4MvUlYy83vP0vxoVNzM6t8MUXqNuRsuBmBHQDu1Fh6X015FrLdgoDKcNdkwGubozq0P4N0Q37UyFVr1EA==
@ -19173,6 +19223,13 @@ ssri@^5.2.4:
dependencies:
safe-buffer "^5.1.1"
ssri@^6.0.0:
version "6.0.1"
resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.1.tgz#2a3c41b28dd45b62b63676ecb74001265ae9edd8"
integrity sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==
dependencies:
figgy-pudding "^3.5.1"
stack-trace@0.0.x:
version "0.0.10"
resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
@ -19863,6 +19920,29 @@ term-size@^1.2.0:
dependencies:
execa "^0.7.0"
terser-webpack-plugin@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.1.0.tgz#cf7c25a1eee25bf121f4a587bb9e004e3f80e528"
integrity sha512-61lV0DSxMAZ8AyZG7/A4a3UPlrbOBo8NIQ4tJzLPAdGOQ+yoNC7l5ijEow27lBAL2humer01KLS6bGIMYQxKoA==
dependencies:
cacache "^11.0.2"
find-cache-dir "^2.0.0"
schema-utils "^1.0.0"
serialize-javascript "^1.4.0"
source-map "^0.6.1"
terser "^3.8.1"
webpack-sources "^1.1.0"
worker-farm "^1.5.2"
terser@^3.8.1:
version "3.11.0"
resolved "https://registry.yarnpkg.com/terser/-/terser-3.11.0.tgz#60782893e1f4d6788acc696351f40636d0e37af0"
integrity sha512-5iLMdhEPIq3zFWskpmbzmKwMQixKmTYwY3Ox9pjtSklBLnHiuQ0GKJLhL1HSYtyffHM3/lDIFBnb82m9D7ewwQ==
dependencies:
commander "~2.17.1"
source-map "~0.6.1"
source-map-support "~0.5.6"
test-exclude@^4.2.1:
version "4.2.3"
resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-4.2.3.tgz#a9a5e64474e4398339245a0a769ad7c2f4a97c20"
@ -19899,6 +19979,15 @@ textextensions@2:
resolved "https://registry.yarnpkg.com/textextensions/-/textextensions-2.2.0.tgz#38ac676151285b658654581987a0ce1a4490d286"
integrity sha512-j5EMxnryTvKxwH2Cq+Pb43tsf6sdEgw6Pdwxk83mPaq0ToeFJt6WE4J3s5BqY7vmjlLgkgXvhtXUxo80FyBhCA==
thread-loader@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/thread-loader/-/thread-loader-1.2.0.tgz#35dedb23cf294afbbce6c45c1339b950ed17e7a4"
integrity sha512-acJ0rvUk53+ly9cqYWNOpPqOgCkNpmHLPDGduNm4hDQWF7EDKEJXAopG9iEWsPPcml09wePkq3NF+ZUqnO6tbg==
dependencies:
async "^2.3.0"
loader-runner "^2.3.0"
loader-utils "^1.1.0"
throat@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/throat/-/throat-4.1.0.tgz#89037cbc92c56ab18926e6ba4cbb200e15672a6a"
@ -20551,20 +20640,6 @@ uglifyjs-webpack-plugin@^1.2.4:
webpack-sources "^1.1.0"
worker-farm "^1.5.2"
uglifyjs-webpack-plugin@^1.2.7:
version "1.2.7"
resolved "https://registry.yarnpkg.com/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.2.7.tgz#57638dd99c853a1ebfe9d97b42160a8a507f9d00"
integrity sha512-1VicfKhCYHLS8m1DCApqBhoulnASsEoJ/BvpUpP4zoNAPpKzdH+ghk0olGJMmwX2/jprK2j3hAHdUbczBSy2FA==
dependencies:
cacache "^10.0.4"
find-cache-dir "^1.0.0"
schema-utils "^0.4.5"
serialize-javascript "^1.4.0"
source-map "^0.6.1"
uglify-es "^3.3.4"
webpack-sources "^1.1.0"
worker-farm "^1.5.2"
ui-select@0.19.4:
version "0.19.4"
resolved "https://registry.yarnpkg.com/ui-select/-/ui-select-0.19.4.tgz#f5702c90cd91eca094202188a7fdbc9483e58797"