github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

chore(NA): teardown dynamic dll plugin (#72096) (#72248)

Browse source 
* chore(NA): teardown dynamic dll plugin

* chore(NA): remove missing ts-ignore

* chore(NA): remove last mentions to the DLL machinery

* chore(NA): update notice file

* prevent duplication and searching target/public

* remove changes to es-ui code to unblock pr

* add node internals override for legacy tests bundle

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
# Conflicts:
#	src/dev/notice/generate_notice_from_source.ts
#	src/optimize/base_optimizer.js
This commit is contained in:
Tiago Costa 2020-07-17 13:51:29 +01:00 • committed by GitHub
parent a8477b0d6c
commit ce9176093b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 15 additions and 2430 deletions
Download patch file Download diff file Expand all files Collapse all files

3
NOTICE.txt
View file

@ -26,9 +26,6 @@ This module was heavily inspired by the externals plugin that ships with webpack
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
---
This product has relied on ASTExplorer that is licensed under MIT.
---
This product includes code that is based on Ace editor, which was available
under a "BSD" license.

1
package.json
View file

@ -134,7 +134,6 @@
"@hapi/good-squeeze": "5.2.1",
"@hapi/wreck": "^15.0.2",
"@kbn/analytics": "1.0.0",
"@kbn/babel-code-parser": "1.0.0",
"@kbn/babel-preset": "1.0.0",
"@kbn/config-schema": "1.0.0",
"@kbn/i18n": "1.0.0",

3
packages/kbn-babel-code-parser/.babelrc
View file

@ -1,3 +0,0 @@
{
"presets": ["@kbn/babel-preset/node_preset"]
}

19
packages/kbn-babel-code-parser/README.md
View file

@ -1,19 +0,0 @@
# @kbn/babel-code-parser
Simple abstraction over the `@babel/parser` and the `@babel/traverse` in order
to build a code parser on top.
We have two main functions `parseSingleFile` (sync and sync version) and the
`parseEntries` (only async version). The first one just parse one entry file
and the second one parses recursively all the files from a list of
start entry points.
Then we have `visitors` and `strategies`. The first ones are basically the
`visitors` to use into the ast from the `@babel/traverse`. They are the only
way to collect info when using the `parseSingleFile`. The `strategies` are
meant to be used with the `parseEntries` and configures the info we want
to collect from our parsed code. After each loop, one per entry file, the
`parseEntries` method will call the given `strategy` expecting that
`strategy` would call the desired visitors, assemble the important
information to collect and adds them to the final results.

26
packages/kbn-babel-code-parser/package.json
View file

@ -1,26 +0,0 @@
{
"name": "@kbn/babel-code-parser",
"description": "babel code parser for Kibana",
"private": true,
"version": "1.0.0",
"main": "./target/index.js",
"license": "Apache-2.0",
"repository": {
"type": "git",
"url": "https://github.com/elastic/kibana/tree/master/packages/kbn-babel-code-parser"
},
"scripts": {
"build": "babel src --out-dir target",
"kbn:bootstrap": "yarn build --quiet",
"kbn:watch": "yarn build --watch"
},
"devDependencies": {
"@babel/cli": "^7.10.1"
},
"dependencies": {
"@kbn/babel-preset": "1.0.0",
"@babel/parser": "^7.10.2",
"@babel/traverse": "^7.10.1",
"lodash": "^4.17.15"
}
}

36
packages/kbn-babel-code-parser/src/can_require.js
View file

@ -1,36 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export function canRequire(entry, cwd = require.resolve.paths(entry) || []) {
try {
// We will try to test if we can resolve
// this entry through the require.resolve
// setting as the start looking path the
// given cwd. That cwd variable could be
// a path or an array of paths
// from where Require.resolve will keep
// looking recursively as normal starting
// from those locations.
return require.resolve(entry, {
paths: [].concat(cwd),
});
} catch (e) {
return false;
}
}

105
packages/kbn-babel-code-parser/src/code_parser.js
View file

@ -1,105 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { canRequire } from './can_require';
import { readFile, readFileSync } from 'fs';
import { extname } from 'path';
import { promisify } from 'util';
import * as parser from '@babel/parser';
import traverse from '@babel/traverse';
import * as babelParserOptions from '@kbn/babel-preset/common_babel_parser_options';
const read = promisify(readFile);
function _cannotParseFile(filePath) {
return extname(filePath) !== '.js';
}
function _parseAndTraverseFileContent(fileContent, visitorsGenerator) {
const results = [];
// Parse and get the code AST
// All the babel parser plugins
// were enabled
const ast = parser.parse(fileContent, babelParserOptions);
// Loop through the code AST with
// the defined visitors
traverse(ast, visitorsGenerator(results));
return results;
}
export async function parseSingleFile(filePath, visitorsGenerator) {
// Don't parse any other files than .js ones
if (_cannotParseFile(filePath)) {
return [];
}
// Read the file
const content = await read(filePath, { encoding: 'utf8' });
// return the results found on parse and traverse
// the file content with the given visitors
return _parseAndTraverseFileContent(content, visitorsGenerator);
}
export function parseSingleFileSync(filePath, visitorsGenerator) {
// Don't parse any other files than .js ones
if (_cannotParseFile(filePath)) {
return [];
}
// Read the file
const content = readFileSync(filePath, { encoding: 'utf8' });
// return the results found on parse and traverse
// the file content with the given visitors
return _parseAndTraverseFileContent(content, visitorsGenerator);
}
export async function parseEntries(cwd, entries, strategy, results, wasParsed = {}) {
// Assure that we always have a cwd
const sanitizedCwd = cwd || process.cwd();
// Test each entry against canRequire function
const entriesQueue = entries.map((entry) => canRequire(entry));
while (entriesQueue.length) {
// Get the first element in the queue as
// select it as our current entry to parse
const mainEntry = entriesQueue.shift();
// Avoid parse the current entry if it is not valid
// or it was already parsed
if (typeof mainEntry !== 'string' || wasParsed[mainEntry]) {
continue;
}
// Find new entries and adds them to the end of the queue
entriesQueue.push(
...(await strategy(sanitizedCwd, parseSingleFile, mainEntry, wasParsed, results))
);
// Mark the current main entry as already parsed
wasParsed[mainEntry] = true;
}
return results;
}

22
packages/kbn-babel-code-parser/src/index.js
View file

@ -1,22 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { dependenciesParseStrategy } from './strategies';
export { dependenciesVisitorsGenerator } from './visitors';
export { parseSingleFile, parseSingleFileSync, parseEntries } from './code_parser';

103
packages/kbn-babel-code-parser/src/strategies.js
View file

@ -1,103 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { canRequire } from './can_require';
import { dependenciesVisitorsGenerator } from './visitors';
import { dirname, isAbsolute, resolve } from 'path';
import { builtinModules } from 'module';
export function _calculateTopLevelDependency(inputDep, outputDep = '') {
// The path separator will be always the forward slash
// as at this point we only have the found entries into
// the provided source code entries where we just use it
const pathSeparator = '/';
const depSplitPaths = inputDep.split(pathSeparator);
const firstPart = depSplitPaths.shift();
const outputDepFirstArgAppend = outputDep ? pathSeparator : '';
outputDep += `${outputDepFirstArgAppend}${firstPart}`;
// In case our dependency isn't started by @
// we are already done and we can return the
// dependency value we already have
if (firstPart.charAt(0) !== '@') {
return outputDep;
}
// Otherwise we need to keep constructing the dependency
// value because dependencies starting with @ points to
// folders of dependencies. For example, in case we found
// dependencies values with '@the-deps/a' and '@the-deps/a/b'
// we don't want to map it to '@the-deps' but also to @'the-deps/a'
// because inside '@the-deps' we can also have '@the-dep/b'
return _calculateTopLevelDependency(depSplitPaths.join(pathSeparator), outputDep);
}
export async function dependenciesParseStrategy(
cwd,
parseSingleFile,
mainEntry,
wasParsed,
results
) {
// Get dependencies from a single file and filter
// out node native modules from the result
const dependencies = (await parseSingleFile(mainEntry, dependenciesVisitorsGenerator)).filter(
(dep) => !builtinModules.includes(dep)
);
// Return the list of all the new entries found into
// the current mainEntry that we could use to look for
// new dependencies
return dependencies.reduce((filteredEntries, entry) => {
const absEntryPath = resolve(cwd, dirname(mainEntry), entry);
// NOTE: cwd for following canRequires is absEntryPath
// because we should start looking from there
const requiredPath = canRequire(absEntryPath, absEntryPath);
const requiredRelativePath = canRequire(entry, absEntryPath);
const isRelativeFile = !isAbsolute(entry);
const isNodeModuleDep = isRelativeFile && !requiredPath && requiredRelativePath;
const isNewEntry = isRelativeFile && requiredPath;
// If it is a node_module add it to the results and also
// add the resolved path for the node_module main file
// as an entry point to look for dependencies it was
// not already parsed
if (isNodeModuleDep) {
// Save the result as the top level dependency
results[_calculateTopLevelDependency(entry)] = true;
if (!wasParsed[requiredRelativePath]) {
filteredEntries.push(requiredRelativePath);
}
}
// If a new, not yet parsed, relative entry were found
// add it to the list of entries to be parsed
if (isNewEntry && !wasParsed[requiredPath]) {
if (!wasParsed[requiredPath]) {
filteredEntries.push(requiredPath);
}
}
return filteredEntries;
}, []);
}

108
packages/kbn-babel-code-parser/src/strategies.test.js
View file

@ -1,108 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { readFile } from 'fs';
import { canRequire } from './can_require';
import { parseSingleFile } from './code_parser';
import { _calculateTopLevelDependency, dependenciesParseStrategy } from './strategies';
jest.mock('./can_require', () => ({
canRequire: jest.fn(),
}));
jest.mock('fs', () => ({
readFile: jest.fn(),
}));
const mockCwd = '/tmp/project/dir/';
describe('Code Parser Strategies', () => {
it('should calculate the top level dependencies correctly', () => {
const plainDep = 'dep1/file';
const foldedDep = '@kbn/es/file';
const otherFoldedDep = '@kbn/es';
expect(_calculateTopLevelDependency(plainDep)).toEqual('dep1');
expect(_calculateTopLevelDependency(foldedDep)).toEqual('@kbn/es');
expect(_calculateTopLevelDependency(otherFoldedDep)).toEqual('@kbn/es');
});
it('should exclude native modules', async () => {
readFile.mockImplementationOnce((path, options, cb) => {
cb(null, `require('fs')`);
});
const results = [];
await dependenciesParseStrategy(mockCwd, parseSingleFile, 'dep1/file.js', {}, results);
expect(results.length).toBe(0);
});
it('should return a dep from_modules', async () => {
readFile.mockImplementationOnce((path, options, cb) => {
cb(null, `require('dep_from_node_modules')`);
});
canRequire.mockImplementation((entry, cwd) => {
if (entry === `${cwd}dep1/dep_from_node_modules`) {
return false;
}
if (entry === 'dep_from_node_modules') {
return `${mockCwd}node_modules/dep_from_node_modules/index.js`;
}
});
const results = await dependenciesParseStrategy(
mockCwd,
parseSingleFile,
'dep1/file.js',
{},
{}
);
expect(results[0]).toBe(`${mockCwd}node_modules/dep_from_node_modules/index.js`);
});
it('should return a relative dep file', async () => {
readFile.mockImplementationOnce((path, options, cb) => {
cb(null, `require('./relative_dep')`);
});
canRequire.mockImplementation((entry) => {
if (entry === `${mockCwd}dep1/relative_dep`) {
return `${entry}/index.js`;
}
return false;
});
const results = await dependenciesParseStrategy(
mockCwd,
parseSingleFile,
'dep1/file.js',
{},
{}
);
expect(results[0]).toBe(`${mockCwd}dep1/relative_dep/index.js`);
});
afterAll(() => {
jest.clearAllMocks();
});
});

124
packages/kbn-babel-code-parser/src/visitors.js
View file

@ -1,124 +0,0 @@
/* eslint-disable @kbn/eslint/require-license-header */
import { matches } from 'lodash';
/* @notice
*
* This product has relied on ASTExplorer that is licensed under MIT.
*/
export function dependenciesVisitorsGenerator(dependenciesAcc) {
return (() => {
// This was built with help on an ast explorer and some ESTree docs
// like the babel parser ast spec and the main docs for the Esprima
// which is a complete and useful docs for the ESTree spec.
//
// https://astexplorer.net
// https://github.com/babel/babel/blob/master/packages/babel-parser/ast/spec.md
// https://esprima.readthedocs.io/en/latest/syntax-tree-format.html
// https://github.com/estree/estree
return {
// Visitors to traverse and found dependencies
// raw values on require + require.resolve
CallExpression: ({ node }) => {
// AST check for require expressions
const isRequire = (node) => {
return matches({
callee: {
type: 'Identifier',
name: 'require',
},
})(node);
};
// AST check for require.resolve expressions
const isRequireResolve = (node) => {
return matches({
callee: {
type: 'MemberExpression',
object: {
type: 'Identifier',
name: 'require',
},
property: {
type: 'Identifier',
name: 'resolve',
},
},
})(node);
};
// Get string values inside the expressions
// whether they are require or require.resolve
if (isRequire(node) || isRequireResolve(node)) {
const nodeArguments = node.arguments;
const reqArg = Array.isArray(nodeArguments) ? nodeArguments.shift() : null;
if (!reqArg) {
return;
}
if (reqArg.type === 'StringLiteral') {
dependenciesAcc.push(reqArg.value);
}
}
},
// Visitors to traverse and found dependencies
// raw values on import
ImportDeclaration: ({ node }) => {
// AST check for supported import expressions
const isImport = (node) => {
return matches({
type: 'ImportDeclaration',
source: {
type: 'StringLiteral',
},
})(node);
};
// Get string values from import expressions
if (isImport(node)) {
const importSource = node.source;
dependenciesAcc.push(importSource.value);
}
},
// Visitors to traverse and found dependencies
// raw values on export from
ExportNamedDeclaration: ({ node }) => {
// AST check for supported export from expressions
const isExportFrom = (node) => {
return matches({
type: 'ExportNamedDeclaration',
source: {
type: 'StringLiteral',
},
})(node);
};
// Get string values from export from expressions
if (isExportFrom(node)) {
const exportFromSource = node.source;
dependenciesAcc.push(exportFromSource.value);
}
},
// Visitors to traverse and found dependencies
// raw values on export * from
ExportAllDeclaration: ({ node }) => {
// AST check for supported export * from expressions
const isExportAllFrom = (node) => {
return matches({
type: 'ExportAllDeclaration',
source: {
type: 'StringLiteral',
},
})(node);
};
// Get string values from export * from expressions
if (isExportAllFrom(node)) {
const exportAllFromSource = node.source;
dependenciesAcc.push(exportAllFromSource.value);
}
},
};
})();
}

68
packages/kbn-babel-code-parser/src/visitors.test.js
View file

@ -1,68 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import * as parser from '@babel/parser';
import traverse from '@babel/traverse';
import { dependenciesVisitorsGenerator } from './visitors';
const visitorsApplier = (code) => {
const result = [];
traverse(
parser.parse(code, {
sourceType: 'unambiguous',
plugins: ['exportDefaultFrom'],
}),
dependenciesVisitorsGenerator(result)
);
return result;
};
describe('Code Parser Visitors', () => {
it('should get values from require', () => {
const rawCode = `/*foo*/require('dep1'); const bar = 1;`;
const foundDeps = visitorsApplier(rawCode);
expect(foundDeps[0] === 'dep1');
});
it('should get values from require.resolve', () => {
const rawCode = `/*foo*/require.resolve('dep2'); const bar = 1;`;
const foundDeps = visitorsApplier(rawCode);
expect(foundDeps[0] === 'dep2');
});
it('should get values from import', () => {
const rawCode = `/*foo*/import dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`;
const foundDeps = visitorsApplier(rawCode);
expect(foundDeps[0] === 'dep1');
expect(foundDeps[1] === 'dep2');
});
it('should get values from export from', () => {
const rawCode = `/*foo*/export dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`;
const foundDeps = visitorsApplier(rawCode);
expect(foundDeps[0] === 'dep1');
});
it('should get values from export * from', () => {
const rawCode = `/*foo*/export * from 'dep1'; export dep2 from 'dep2';const bar = 1;`;
const foundDeps = visitorsApplier(rawCode);
expect(foundDeps[0] === 'dep1');
expect(foundDeps[1] === 'dep2');
});
});

1
packages/kbn-babel-code-parser/yarn.lock
View file

@ -1 +0,0 @@
../../yarn.lock

11
src/dev/build/README.md
View file

@ -39,14 +39,3 @@ The majority of this logic is extracted from the grunt build that has existed fo
[lib/build.js]: ./lib/build.js
[build_distributables.js]: ./build_distributables.js
[../tooling_log/tooling_log.js]: ../tooling_log/tooling_log.js
# Client Node Modules Cleaning
We have introduced in our bundle a webpack dll for the client vendor modules in order to improve
the optimization time both in dev and in production. As for those modules we already have the
code into the vendors_${chunk_number}.bundle.dll.js we have decided to delete those bundled modules from the
distributable node_modules folder. However, in order to accomplish this, we need to exclude
every node_module used in the server side code. This logic is performed
under `nodejs_modules/clean_client_modules_on_dll_task.js`. In case we need to add any new cli
or any other piece of server code other than `x-pack` or `core_plugins` we'll need
to update the globs present on `clean_client_modules_on_dll_task.js` accordingly.

2
src/dev/build/build_distributables.js
View file

@ -22,7 +22,6 @@ import { getConfig, createRunner } from './lib';
import {
BuildKibanaPlatformPluginsTask,
BuildPackagesTask,
CleanClientModulesOnDLLTask,
CleanEmptyFoldersTask,
CleanExtraBinScriptsTask,
CleanExtraFilesFromModulesTask,
@ -127,7 +126,6 @@ export async function buildDistributables(options) {
await run(TranspileScssTask);
await run(BuildKibanaPlatformPluginsTask);
await run(OptimizeBuildTask);
await run(CleanClientModulesOnDLLTask);
await run(CleanTypescriptTask);
await run(CleanExtraFilesFromModulesTask);
await run(CleanEmptyFoldersTask);

1
src/dev/build/tasks/index.js
View file

@ -30,7 +30,6 @@ export * from './create_readme_task';
export * from './install_chromium';
export * from './install_dependencies_task';
export * from './license_file_task';
export * from './nodejs_modules';
export * from './nodejs';
export * from './notice_file_task';
export * from './optimize_task';

126
src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js
View file

@ -1,126 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import {
getDllEntries,
cleanDllModuleFromEntryPath,
writeEmptyFileForDllEntry,
} from './webpack_dll';
import { getDependencies } from './get_dependencies';
import globby from 'globby';
import normalizePosixPath from 'normalize-path';
export const CleanClientModulesOnDLLTask = {
description: 'Cleaning client node_modules bundled into the DLL',
async run(config, log, build) {
const baseDir = normalizePosixPath(build.resolvePath('.'));
const kbnPkg = config.getKibanaPkg();
const kbnPkgDependencies = (kbnPkg && kbnPkg.dependencies) || {};
const kbnWebpackLoaders = Object.keys(kbnPkgDependencies).filter(
(dep) => !!dep.includes('-loader')
);
// Define the entry points for the server code in order to
// start here later looking for the server side dependencies
const mainCodeEntries = [
`${baseDir}/src/cli`,
`${baseDir}/src/cli_keystore`,
`${baseDir}/src/cli_plugin`,
`${baseDir}/x-pack`,
...kbnWebpackLoaders.map((loader) => `${baseDir}/node_modules/${loader}`),
];
const discoveredLegacyCorePluginEntries = await globby([
`${baseDir}/src/legacy/core_plugins/*/index.js`,
`!${baseDir}/src/legacy/core_plugins/**/public`,
]);
const discoveredPluginEntries = await globby([
`${baseDir}/src/plugins/*/server/index.js`,
// Small exception to load dynamically discovered functions for timelion plugin
`${baseDir}/src/plugins/vis_type_timelion/server/*_functions/**/*.js`,
`!${baseDir}/src/plugins/**/public`,
]);
const discoveredNewPlatformXpackPlugins = await globby([
`${baseDir}/x-pack/plugins/*/server/index.js`,
`!${baseDir}/x-pack/plugins/**/public`,
]);
// Compose all the needed entries
const serverEntries = [
...mainCodeEntries,
...discoveredLegacyCorePluginEntries,
...discoveredPluginEntries,
...discoveredNewPlatformXpackPlugins,
];
// Get the dependencies found searching through the server
// side code entries that were provided
const serverDependencies = await getDependencies(baseDir, serverEntries);
// This fulfill a particular exceptional case where
// we need to keep loading a file from a node_module
// only used in the front-end like we do when using the file-loader
// in https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js
//
// manual list of exception modules
const manualExceptionModules = ['mapbox-gl'];
// consider the top modules as exceptions as the entry points
// to look for other exceptions dependent on that one
const manualExceptionEntries = [
...manualExceptionModules.map((module) => `${baseDir}/node_modules/${module}`),
];
// dependencies for declared exception modules
const manualExceptionModulesDependencies = await getDependencies(baseDir, [
...manualExceptionEntries,
]);
// final list of manual exceptions to add
const manualExceptions = [...manualExceptionModules, ...manualExceptionModulesDependencies];
// Consider this as our whiteList for the modules we can't delete
const whiteListedModules = [...serverDependencies, ...kbnWebpackLoaders, ...manualExceptions];
// Resolve the client vendors dll manifest paths
// excluding the runtime one
const dllManifestPaths = await globby([
`${baseDir}/built_assets/dlls/vendors_*.manifest.dll.json`,
`!${baseDir}/built_assets/dlls/vendors_runtime.manifest.dll.json`,
]);
// Get dll entries filtering out the ones
// from any whitelisted module
const dllEntries = await getDllEntries(dllManifestPaths, whiteListedModules, baseDir);
for (const relativeEntryPath of dllEntries) {
const entryPath = `${baseDir}/${relativeEntryPath}`;
if (entryPath.endsWith('package.json')) {
continue;
}
// Clean a module included into the dll
// and then write a blank file for each
// entry file present into the dll
await cleanDllModuleFromEntryPath(log, entryPath);
await writeEmptyFileForDllEntry(entryPath);
}
},
};

27
src/dev/build/tasks/nodejs_modules/get_dependencies.js
View file

@ -1,27 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { parseEntries, dependenciesParseStrategy } from '@kbn/babel-code-parser';
export async function getDependencies(cwd, entries) {
// Return the dependencies retrieve from the
// provided code entries (sanitized) and
// parseStrategy (dependencies one)
return Object.keys(await parseEntries(cwd, entries, dependenciesParseStrategy, {}));
}

20
src/dev/build/tasks/nodejs_modules/index.js
View file

@ -1,20 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { CleanClientModulesOnDLLTask } from './clean_client_modules_on_dll_task';

131
src/dev/build/tasks/nodejs_modules/webpack_dll.js
View file

@ -1,131 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { deleteAll, isFileAccessible, read, write } from '../../lib';
import { dirname, relative, resolve } from 'path';
import pkgUp from 'pkg-up';
import globby from 'globby';
import normalizePosixPath from 'normalize-path';
function checkDllEntryAccess(entry, baseDir = '') {
const resolvedPath = baseDir ? resolve(baseDir, entry) : entry;
return isFileAccessible(resolvedPath);
}
export async function getDllEntries(manifestPaths, whiteListedModules, baseDir = '') {
// Read and parse all manifests
const manifests = await Promise.all(
manifestPaths.map(async (manifestPath) => JSON.parse(await read(manifestPath)))
);
// Process and group modules from all manifests
const manifestsModules = manifests.flatMap((manifest, idx) => {
if (!manifest || !manifest.content) {
// It should fails because if we don't have the manifest file
// or it is malformed something wrong is happening and we
// should stop
throw new Error(`The following dll manifest doesn't exists: ${manifestPaths[idx]}`);
}
const modules = Object.keys(manifest.content);
if (!modules.length) {
// It should fails because if we don't have any
// module inside the client vendors dll something
// wrong is happening and we should stop too
throw new Error(
`The following dll manifest is reporting an empty dll: ${manifestPaths[idx]}`
);
}
return modules;
});
// Only includes modules who are not in the white list of modules
// and that are node_modules
return manifestsModules.filter((entry) => {
const isWhiteListed = whiteListedModules.some((nonEntry) =>
normalizePosixPath(entry).includes(`node_modules/${nonEntry}`)
);
const isNodeModule = entry.includes('node_modules');
// NOTE: when using dynamic imports on webpack the entry paths could be created
// with special context module (ex: lazy recursive) values over directories that are not real files
// and only exists in runtime, so we need to check if the entry is a real file.
// We found that problem through the issue https://github.com/elastic/kibana/issues/38481
//
// More info:
// https://github.com/webpack/webpack/blob/master/examples/code-splitting-harmony/README.md
// https://webpack.js.org/guides/dependency-management/#require-with-expression
const isAccessible = checkDllEntryAccess(entry, baseDir);
return !isWhiteListed && isNodeModule && isAccessible;
});
}
export async function cleanDllModuleFromEntryPath(logger, entryPath) {
const modulePkgPath = await pkgUp(entryPath);
const modulePkg = JSON.parse(await read(modulePkgPath));
const moduleDir = dirname(modulePkgPath);
const normalizedModuleDir = normalizePosixPath(moduleDir);
// Cancel the cleanup for this module as it
// was already done.
if (modulePkg.cleaned) {
return;
}
// Clear dependencies from dll module package.json
if (modulePkg.dependencies) {
modulePkg.dependencies = {};
}
// Clear devDependencies from dll module package.json
if (modulePkg.devDependencies) {
modulePkg.devDependencies = {};
}
// Delete module contents. It will delete everything
// excepts package.json, images and css
//
// NOTE: We can't use cwd option with globby
// until the following issue gets closed
// https://github.com/sindresorhus/globby/issues/87
const filesToDelete = await globby([
`${normalizedModuleDir}/**`,
`!${normalizedModuleDir}/**/*.+(css)`,
`!${normalizedModuleDir}/**/*.+(gif|ico|jpeg|jpg|tiff|tif|svg|png|webp)`,
]);
await deleteAll(
filesToDelete.filter((path) => {
const relativePath = relative(moduleDir, path);
return !relativePath.endsWith('package.json') || relativePath.includes('node_modules');
})
);
// Mark this module as cleaned
modulePkg.cleaned = true;
// Rewrite modified package.json
await write(modulePkgPath, JSON.stringify(modulePkg, null, 2));
}
export async function writeEmptyFileForDllEntry(entryPath) {
await write(entryPath, '');
}

123
src/dev/build/tasks/nodejs_modules/webpack_dll.test.js
View file

@ -1,123 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { isFileAccessible, read } from '../../lib';
import { getDllEntries } from './webpack_dll';
jest.mock('../../lib', () => ({
read: jest.fn(),
isFileAccessible: jest.fn(),
}));
const manifestContentMock = JSON.stringify({
name: 'vendors',
content: {
'/mock/node_modules/dep1': {},
'/mock/node_modules/dep2': {},
'/mock/node_modules/dep3': {},
'/mock/tmp/dep2': {},
},
});
const emptyManifestContentMock = JSON.stringify({
name: 'vendors',
content: {},
});
const noManifestMock = JSON.stringify(null);
const noContentFieldManifestMock = JSON.stringify({
name: 'vendors',
});
describe('Webpack DLL Build Tasks Utils', () => {
it('should get dll entries correctly', async () => {
read.mockImplementationOnce(async () => manifestContentMock);
isFileAccessible.mockImplementation(() => true);
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
const mockModulesWhitelist = ['dep1'];
const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
expect(dllEntries).toEqual(
expect.arrayContaining(['/mock/node_modules/dep2', '/mock/node_modules/dep3'])
);
});
it('should only include accessible files', async () => {
read.mockImplementationOnce(async () => manifestContentMock);
isFileAccessible.mockImplementation(() => false);
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
const mockModulesWhitelist = ['dep1'];
const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
isFileAccessible.mockRestore();
expect(dllEntries.length).toEqual(0);
});
it('should throw an error for no manifest file', async () => {
read.mockImplementationOnce(async () => noManifestMock);
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);
} catch (error) {
expect(error.message).toEqual(
`The following dll manifest doesn't exists: /mock/mock_dll_manifest.json`
);
}
});
it('should throw an error for no manifest content field', async () => {
read.mockImplementation(async () => noContentFieldManifestMock);
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);
} catch (error) {
expect(error.message).toEqual(
`The following dll manifest doesn't exists: /mock/mock_dll_manifest.json`
);
}
});
it('should throw an error for manifest file without any content', async () => {
read.mockImplementation(async () => emptyManifestContentMock);
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);
} catch (error) {
expect(error.message).toEqual(
`The following dll manifest is reporting an empty dll: /mock/mock_dll_manifest.json`
);
}
});
afterAll(() => {
jest.clearAllMocks();
});
});

1
src/dev/build/tasks/optimize_task.js
View file

@ -41,7 +41,6 @@ export const OptimizeBuildTask = {
await exec(log, kibanaScript, kibanaArgs, {
cwd: build.resolvePath('.'),
env: {
FORCE_DLL_CREATION: 'true',
KBN_CACHE_LOADER_WRITABLE: 'true',
NODE_OPTIONS: '--max-old-space-size=4096',
},

9
src/dev/notice/generate_notice_from_source.ts
View file

@ -47,10 +47,11 @@ export async function generateNoticeFromSource({ productName, directory, log }:
cwd: directory,
nodir: true,
ignore: [
'{node_modules,build,target,dist,optimize,built_assets}/**',
'packages/*/{node_modules,build,target,dist}/**',
'x-pack/{node_modules,build,target,dist,optimize}/**',
'x-pack/packages/*/{node_modules,build,target,dist}/**',
'{node_modules,build,dist,data,built_assets}/**',
'packages/*/{node_modules,build,dist}/**',
'x-pack/{node_modules,build,dist,data}/**',
'x-pack/packages/*/{node_modules,build,dist}/**',
'**/target/**',
],
};

7
src/legacy/core_plugins/tests_bundle/index.js
View file

@ -108,6 +108,13 @@ export default (kibana) => {
resolve: {
extensions: ['.karma_mock.js', '.karma_mock.tsx', '.karma_mock.ts'],
},
node: {
fs: 'empty',
child_process: 'empty',
dns: 'empty',
net: 'empty',
tls: 'empty',
},
},
webpackConfig
);

2
src/legacy/server/logging/log_format_string.js
View file

@ -41,8 +41,6 @@ const typeColors = {
optmzr: 'white',
manager: 'green',
optimize: 'magentaBright',
'optimize:dynamic_dll_plugin': 'magentaBright',
'optimize:watch_cache': 'magentaBright',
listening: 'magentaBright',
scss: 'magentaBright',
};

11
src/legacy/ui/ui_render/ui_render_mixin.js
View file

@ -24,7 +24,6 @@ import { i18n } from '@kbn/i18n';
import * as UiSharedDeps from '@kbn/ui-shared-deps';
import { AppBootstrap } from './bootstrap';
import { getApmConfig } from '../apm';
import { DllCompiler } from '../../../optimize/dynamic_dll_plugin';
/**
* @typedef {import('../../server/kbn_server').default} KbnServer
@ -106,17 +105,8 @@ export function uiRenderMixin(kbnServer, server, config) {
const basePath = config.get('server.basePath');
const regularBundlePath = `${basePath}/${buildHash}/bundles`;
const dllBundlePath = `${basePath}/${buildHash}/built_assets/dlls`;
const dllStyleChunks = DllCompiler.getRawDllConfig().chunks.map(
(chunk) => `${dllBundlePath}/vendors${chunk}.style.dll.css`
);
const dllJsChunks = DllCompiler.getRawDllConfig().chunks.map(
(chunk) => `${dllBundlePath}/vendors${chunk}.bundle.dll.js`
);
const styleSheetPaths = [
...(isCore ? [] : dllStyleChunks),
`${regularBundlePath}/kbn-ui-shared-deps/${UiSharedDeps.baseCssDistFilename}`,
...(darkMode
? [
@ -173,7 +163,6 @@ export function uiRenderMixin(kbnServer, server, config) {
(filename) => `${regularBundlePath}/kbn-ui-shared-deps/${filename}`
),
`${regularBundlePath}/kbn-ui-shared-deps/${UiSharedDeps.jsFilename}`,
...(isCore ? [] : [`${dllBundlePath}/vendors_runtime.bundle.dll.js`, ...dllJsChunks]),
`${regularBundlePath}/core/core.entry.js`,
...kpPluginBundlePaths,

7
src/optimize/base_optimizer.js
View file

@ -29,7 +29,6 @@ import * as threadLoader from 'thread-loader';
import webpackMerge from 'webpack-merge';
import * as UiSharedDeps from '@kbn/ui-shared-deps';
import { DynamicDllPlugin } from './dynamic_dll_plugin';
import { IS_KIBANA_DISTRIBUTABLE } from '../legacy/utils';
import { fromRoot } from '../core/server/utils';
import { PUBLIC_PATH_PLACEHOLDER } from './public_path_placeholder';
@ -282,12 +281,6 @@ export default class BaseOptimizer {
},
plugins: [
new DynamicDllPlugin({
uiBundles: this.uiBundles,
threadLoaderPoolConfig: this.getThreadLoaderPoolConfig(),
logWithMetadata: this.logWithMetadata,
}),
new MiniCssExtractPlugin({
filename: '[name].style.css',
}),

12
src/optimize/bundles_route/__tests__/bundles_route.js
View file

@ -56,7 +56,6 @@ describe('optimizer/bundle route', () => {
function createServer(options = {}) {
const {
regularBundlesPath = outputFixture,
dllBundlesPath = outputFixture,
basePublicPath = '',
builtCssPath = outputFixture,
npUiPluginPublicDirs = [],
@ -70,7 +69,6 @@ describe('optimizer/bundle route', () => {
server.route(
createBundlesRoute({
regularBundlesPath,
dllBundlesPath,
basePublicPath,
builtCssPath,
npUiPluginPublicDirs,
@ -89,28 +87,24 @@ describe('optimizer/bundle route', () => {
expect(() => {
createBundlesRoute({
regularBundlesPath: null,
dllBundlesPath: '/absolute/path',
basePublicPath: '',
});
}).to.throwError(/absolute path/);
expect(() => {
createBundlesRoute({
regularBundlesPath: './relative',
dllBundlesPath: '/absolute/path',
basePublicPath: '',
});
}).to.throwError(/absolute path/);
expect(() => {
createBundlesRoute({
regularBundlesPath: 1234,
dllBundlesPath: '/absolute/path',
basePublicPath: '',
});
}).to.throwError(/absolute path/);
expect(() => {
createBundlesRoute({
regularBundlesPath: '/absolute/path',
dllBundlesPath: '/absolute/path',
basePublicPath: '',
});
}).to.not.throwError();
@ -119,42 +113,36 @@ describe('optimizer/bundle route', () => {
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: 123,
});
}).to.throwError(/string/);
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: {},
});
}).to.throwError(/string/);
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: '/a/',
});
}).to.throwError(/start and not end with a \//);
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: 'a/',
});
}).to.throwError(/start and not end with a \//);
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: '/a',
});
}).to.not.throwError();
expect(() => {
createBundlesRoute({
regularBundlesPath: '/bundles',
dllBundlesPath: '/absolute/path',
basePublicPath: '',
});
}).to.not.throwError();

19
src/optimize/bundles_route/bundles_route.ts
View file

@ -28,22 +28,19 @@ import { assertIsNpUiPluginPublicDirs, NpUiPluginPublicDirs } from '../np_ui_plu
import { fromRoot } from '../../core/server/utils';
/**
* Creates the routes that serves files from `bundlesPath` or from
* `dllBundlesPath` (if they are dll bundle's related files). If the
* Creates the routes that serves files from `bundlesPath`. If the
* file is js or css then it is searched for instances of
* PUBLIC_PATH_PLACEHOLDER and replaces them with `publicPath`.
*
* @param {Object} options
* @property {Array<{id,path}>} options.npUiPluginPublicDirs array of ids and paths that should be served for new platform plugins
* @property {string} options.regularBundlesPath
* @property {string} options.dllBundlesPath
* @property {string} options.basePublicPath
*
* @return Array.of({Hapi.Route})
*/
export function createBundlesRoute({
regularBundlesPath,
dllBundlesPath,
basePublicPath,
builtCssPath,
npUiPluginPublicDirs = [],
@ -51,7 +48,6 @@ export function createBundlesRoute({
isDist = false,
}: {
regularBundlesPath: string;
dllBundlesPath: string;
basePublicPath: string;
builtCssPath: string;
npUiPluginPublicDirs?: NpUiPluginPublicDirs;
@ -70,12 +66,6 @@ export function createBundlesRoute({
);
}
if (typeof dllBundlesPath !== 'string' || !isAbsolute(dllBundlesPath)) {
throw new TypeError(
'dllBundlesPath must be an absolute path to the directory containing the dll bundles'
);
}
if (typeof basePublicPath !== 'string') {
throw new TypeError('basePublicPath must be a string');
}
@ -118,13 +108,6 @@ export function createBundlesRoute({
fileHashCache,
isDist,
}),
buildRouteForBundles({
publicPath: `${basePublicPath}/${buildHash}/built_assets/dlls/`,
routePath: `/${buildHash}/built_assets/dlls/`,
bundlesPath: dllBundlesPath,
fileHashCache,
isDist,
}),
buildRouteForBundles({
publicPath: `${basePublicPath}/`,
routePath: `/${buildHash}/built_assets/css/`,

1
src/optimize/bundles_route/proxy_bundles_route.ts
View file

@ -28,7 +28,6 @@ export function createProxyBundlesRoute({
}) {
return [
buildProxyRouteForBundles(`/${buildHash}/bundles/`, host, port),
buildProxyRouteForBundles(`/${buildHash}/built_assets/dlls/`, host, port),
buildProxyRouteForBundles(`/${buildHash}/built_assets/css/`, host, port),
];
}

36
src/optimize/dynamic_dll_plugin/dll_allowed_modules.js
View file

@ -1,36 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import path from 'path';
export function notInNodeModules(checkPath) {
return !checkPath.includes(`${path.sep}node_modules${path.sep}`);
}
export function notInNodeModulesOrWebpackShims(checkPath) {
return notInNodeModules(checkPath) && !checkPath.includes(`${path.sep}webpackShims${path.sep}`);
}
export function inPluginNodeModules(checkPath) {
return checkPath.match(/[\/\\]plugins.*[\/\\]node_modules/);
}
export function inDllPluginPublic(checkPath) {
return checkPath.includes(`${path.sep}dynamic_dll_plugin${path.sep}public${path.sep}`);
}

366
src/optimize/dynamic_dll_plugin/dll_compiler.js
View file

@ -1,366 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { configModel } from './dll_config_model';
import {
notInNodeModulesOrWebpackShims,
notInNodeModules,
inDllPluginPublic,
} from './dll_allowed_modules';
import {
dllEntryFileContentArrayToString,
dllEntryFileContentStringToArray,
dllMergeAllEntryFilesContent,
} from './dll_entry_template';
import { fromRoot } from '../../core/server/utils';
import { PUBLIC_PATH_PLACEHOLDER } from '../public_path_placeholder';
import fs from 'fs';
import webpack from 'webpack';
import { promisify } from 'util';
import path from 'path';
import del from 'del';
import { chunk } from 'lodash';
import seedrandom from 'seedrandom';
const readFileAsync = promisify(fs.readFile);
const mkdirAsync = promisify(fs.mkdir);
const accessAsync = promisify(fs.access);
const writeFileAsync = promisify(fs.writeFile);
export class DllCompiler {
static getRawDllConfig(
uiBundles = {},
babelLoaderCacheDir = '',
threadLoaderPoolConfig = {},
chunks = Array.from(Array(4).keys()).map((chunkN) => `_${chunkN}`)
) {
return {
uiBundles,
babelLoaderCacheDir,
threadLoaderPoolConfig,
chunks,
context: fromRoot('.'),
entryName: 'vendors',
dllName: '[name]',
manifestName: '[name]',
styleName: '[name]',
entryExt: '.entry.dll.js',
dllExt: '.bundle.dll.js',
manifestExt: '.manifest.dll.json',
styleExt: '.style.dll.css',
outputPath: fromRoot('built_assets/dlls'),
publicPath: PUBLIC_PATH_PLACEHOLDER,
};
}
constructor(uiBundles, threadLoaderPoolConfig, logWithMetadata) {
this.rawDllConfig = DllCompiler.getRawDllConfig(
uiBundles,
uiBundles.getCacheDirectory('babel'),
threadLoaderPoolConfig
);
this.logWithMetadata = logWithMetadata || (() => null);
}
async init() {
await this.ensureEntryFilesExists();
await this.ensureManifestFilesExists();
await this.ensureOutputPathExists();
}
seededShuffle(array) {
// Implementation based on https://github.com/TimothyGu/knuth-shuffle-seeded/blob/gh-pages/index.js#L46
let currentIndex;
let temporaryValue;
let randomIndex;
const rand = seedrandom('predictable', { global: false });
if (array.constructor !== Array) throw new Error('Input is not an array');
currentIndex = array.length;
// While there remain elements to shuffle...
while (0 !== currentIndex) {
// Pick a remaining element...
randomIndex = Math.floor(rand() * currentIndex--);
// And swap it with the current element.
temporaryValue = array[currentIndex];
array[currentIndex] = array[randomIndex];
array[randomIndex] = temporaryValue;
}
return array;
}
async upsertEntryFiles(content) {
const arrayContent = this.seededShuffle(dllEntryFileContentStringToArray(content));
const chunks = chunk(
arrayContent,
Math.ceil(arrayContent.length / this.rawDllConfig.chunks.length)
);
const entryPaths = this.getEntryPaths();
await Promise.all(
entryPaths.map(
async (entryPath, idx) =>
await this.upsertFile(entryPath, dllEntryFileContentArrayToString(chunks[idx]))
)
);
}
async upsertFile(filePath, content = '') {
await this.ensurePathExists(filePath);
await writeFileAsync(filePath, content, 'utf8');
}
getDllPaths() {
return this.rawDllConfig.chunks.map((chunk) =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.dllExt}`)
);
}
getEntryPaths() {
return this.rawDllConfig.chunks.map((chunk) =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.entryExt}`)
);
}
getManifestPaths() {
return this.rawDllConfig.chunks.map((chunk) =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.manifestExt}`)
);
}
getStylePaths() {
return this.rawDllConfig.chunks.map((chunk) =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.styleExt}`)
);
}
async ensureEntryFilesExists() {
const entryPaths = this.getEntryPaths();
await Promise.all(entryPaths.map(async (entryPath) => await this.ensureFileExists(entryPath)));
}
async ensureManifestFilesExists() {
const manifestPaths = this.getManifestPaths();
await Promise.all(
manifestPaths.map(
async (manifestPath, idx) =>
await this.ensureFileExists(
manifestPath,
JSON.stringify({
name: `${this.rawDllConfig.entryName}${this.rawDllConfig.chunks[idx]}`,
content: {},
})
)
)
);
}
async ensureStyleFileExists() {
const stylePaths = this.getStylePaths();
await Promise.all(stylePaths.map(async (stylePath) => await this.ensureFileExists(stylePath)));
}
async ensureFileExists(filePath, content) {
const exists = await this.ensurePathExists(filePath);
if (!exists) {
await this.upsertFile(filePath, content);
}
}
async ensurePathExists(filePath) {
try {
await accessAsync(filePath);
} catch (e) {
await mkdirAsync(path.dirname(filePath), { recursive: true });
return false;
}
return true;
}
async ensureOutputPathExists() {
await this.ensurePathExists(this.rawDllConfig.outputPath);
}
dllsExistsSync() {
const dllPaths = this.getDllPaths();
return dllPaths.every((dllPath) => this.existsSync(dllPath));
}
existsSync(filePath) {
return fs.existsSync(filePath);
}
resolvePath() {
return path.resolve(this.rawDllConfig.outputPath, ...arguments);
}
async readEntryFiles() {
const entryPaths = this.getEntryPaths();
const entryFilesContent = await Promise.all(
entryPaths.map(async (entryPath) => await this.readFile(entryPath))
);
// merge all the module contents from entry files again into
// sorted single one
return dllMergeAllEntryFilesContent(entryFilesContent);
}
async readFile(filePath, content) {
await this.ensureFileExists(filePath, content);
return await readFileAsync(filePath, 'utf8');
}
async run(dllEntries) {
const dllConfig = this.dllConfigGenerator(this.rawDllConfig);
await this.upsertEntryFiles(dllEntries);
try {
this.logWithMetadata(
['info', 'optimize:dynamic_dll_plugin'],
'Client vendors dll compilation started'
);
await this.runWebpack(dllConfig());
this.logWithMetadata(
['info', 'optimize:dynamic_dll_plugin'],
`Client vendors dll compilation finished with success`
);
} catch (e) {
this.logWithMetadata(
['fatal', 'optimize:dynamic_dll_plugin'],
`Client vendors dll compilation failed`
);
// Still throw the original error has here we just want
// log the fail message
throw e;
}
// Style dll file isn't always created but we are
// expecting it to exist always as we are referencing
// it from the bootstrap template
//
// NOTE: We should review the way we deal with the css extraction
// in ours webpack builds. The industry standard is about to
// only extract css for production but we are extracting it
// in every single compilation.
await this.ensureStyleFileExists();
}
dllConfigGenerator(dllConfig) {
return configModel.bind(this, dllConfig);
}
async runWebpack(config) {
return new Promise((resolve, reject) => {
webpack(config, async (err, stats) => {
// If a critical error occurs or we have
// errors in the stats compilation,
// reject the promise and logs the errors
const webpackErrors =
err ||
(stats.hasErrors() &&
stats.toString({
all: false,
colors: true,
errors: true,
errorDetails: true,
moduleTrace: true,
}));
if (webpackErrors) {
// Reject with webpack fatal errors
return reject(webpackErrors);
}
// Identify if we have not allowed modules
// bundled inside the dll bundle
const notAllowedModules = [];
stats.compilation.modules.forEach((module) => {
// ignore if no module or userRequest are defined
if (!module || !module.resource) {
return;
}
// ignore if this module represents the
// dll entry file
if (this.getEntryPaths().includes(module.resource)) {
return;
}
// ignore if this module is part of the
// files inside dynamic dll plugin public folder
if (inDllPluginPublic(module.resource)) {
return;
}
// A module is not allowed if it's not a node_module, a webpackShim
// or the reasons from being bundled into the dll are not node_modules
if (notInNodeModulesOrWebpackShims(module.resource)) {
const reasons = module.reasons || [];
reasons.forEach((reason) => {
// Skip if we can't read the reason info
if (!reason || !reason.module || !reason.module.resource) {
return;
}
// Is the reason for this module being bundle a
// node_module or no?
if (notInNodeModules(reason.module.resource)) {
notAllowedModules.push(module.resource);
}
});
}
});
if (notAllowedModules.length) {
// Delete the built dll, as it contains invalid modules, and reject listing
// all the not allowed modules
try {
await del(this.rawDllConfig.outputPath);
} catch (e) {
return reject(e);
}
return reject(
`The following modules are not allowed to be bundled into the dll: \n${notAllowedModules.join(
'\n'
)}`
);
}
// Otherwise it has succeed
return resolve(stats);
});
});
}
}

278
src/optimize/dynamic_dll_plugin/dll_config_model.js
View file

@ -1,278 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { IS_KIBANA_DISTRIBUTABLE } from '../../legacy/utils';
import { fromRoot } from '../../core/server/utils';
import webpack from 'webpack';
import webpackMerge from 'webpack-merge';
import MiniCssExtractPlugin from 'mini-css-extract-plugin';
import TerserPlugin from 'terser-webpack-plugin';
import * as UiSharedDeps from '@kbn/ui-shared-deps';
function generateDLL(config) {
const {
dllAlias,
dllValidateSyntax,
dllNoParseRules,
dllContext,
dllEntry,
dllOutputPath,
dllPublicPath,
dllBundleName,
dllBundleFilename,
dllStyleFilename,
dllManifestPath,
babelLoaderCacheDir,
threadLoaderPoolConfig,
} = config;
const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
const BABEL_EXCLUDE_RE = [/[\/\\](webpackShims|node_modules|bower_components)[\/\\]/];
/**
* Wrap plugin loading in a function so that we can require
* `@kbn/optimizer` only when absolutely necessary since we
* don't ship this package in the distributable but this code
* is still shipped, though it's not used.
*/
const getValidateSyntaxPlugins = () => {
if (!dllValidateSyntax) {
return [];
}
// only require @kbn/optimizer
const { DisallowedSyntaxPlugin } = require('@kbn/optimizer');
return [new DisallowedSyntaxPlugin()];
};
return {
entry: dllEntry,
context: dllContext,
output: {
futureEmitAssets: true, // TODO: remove on webpack 5
filename: dllBundleFilename,
path: dllOutputPath,
publicPath: dllPublicPath,
library: dllBundleName,
},
node: { fs: 'empty', child_process: 'empty', dns: 'empty', net: 'empty', tls: 'empty' },
resolve: {
extensions: ['.js', '.json'],
mainFields: ['browser', 'browserify', 'main'],
alias: dllAlias,
modules: ['webpackShims', fromRoot('webpackShims'), 'node_modules', fromRoot('node_modules')],
},
module: {
rules: [
{
resource: [
{
test: /\.js$/,
exclude: BABEL_EXCLUDE_RE.concat(dllNoParseRules),
},
{
test: /\.js$/,
include: /[\/\\]node_modules[\/\\]x-pack[\/\\]/,
exclude: /[\/\\]node_modules[\/\\]x-pack[\/\\](.+?[\/\\])*node_modules[\/\\]/,
},
// TODO: remove when we drop support for IE11
// We need because normalize-url is distributed without
// any kind of transpilation
// More info: https://github.com/elastic/kibana/pull/35804
{
test: /\.js$/,
include: /[\/\\]node_modules[\/\\]normalize-url[\/\\]/,
exclude: /[\/\\]node_modules[\/\\]normalize-url[\/\\](.+?[\/\\])*node_modules[\/\\]/,
},
],
// Self calling function with the equivalent logic
// from maybeAddCacheLoader one from base optimizer
use: ((babelLoaderCacheDirPath, loaders) => {
return [
{
loader: 'cache-loader',
options: {
cacheContext: fromRoot('.'),
cacheDirectory: babelLoaderCacheDirPath,
readOnly: process.env.KBN_CACHE_LOADER_WRITABLE ? false : IS_KIBANA_DISTRIBUTABLE,
},
},
...loaders,
];
})(babelLoaderCacheDir, [
{
loader: 'thread-loader',
options: threadLoaderPoolConfig,
},
{
loader: 'babel-loader',
options: {
babelrc: false,
presets: [BABEL_PRESET_PATH],
},
},
]),
},
{
test: /\.(html|tmpl)$/,
loader: 'raw-loader',
},
{
test: /\.css$/,
use: [MiniCssExtractPlugin.loader, 'css-loader'],
},
{
test: /\.png$/,
loader: 'url-loader',
},
{
test: /\.(woff|woff2|ttf|eot|svg|ico)(\?|$)/,
loader: 'file-loader',
},
],
noParse: dllNoParseRules,
},
plugins: [
new webpack.DllPlugin({
context: dllContext,
name: dllBundleName,
path: dllManifestPath,
}),
new MiniCssExtractPlugin({
filename: dllStyleFilename,
}),
...getValidateSyntaxPlugins(),
],
// Single runtime for the dll bundles which assures that common transient dependencies won't be evaluated twice.
// The module cache will be shared, even when module code may be duplicated across chunks.
optimization: {
runtimeChunk: {
name: 'vendors_runtime',
},
},
performance: {
// NOTE: we are disabling this as those hints
// are more tailored for the final bundles result
// and not for the webpack compilations performance itself
hints: false,
},
externals: {
...UiSharedDeps.externals,
},
};
}
function extendRawConfig(rawConfig) {
// Build all extended configs from raw config
const dllAlias = rawConfig.uiBundles.getAliases();
const dllValidateSyntax = rawConfig.uiBundles.shouldValidateSyntaxOfNodeModules();
const dllNoParseRules = rawConfig.uiBundles.getWebpackNoParseRules();
const dllDevMode = rawConfig.uiBundles.isDevMode();
const dllContext = rawConfig.context;
const dllChunks = rawConfig.chunks;
const dllEntry = {};
const dllEntryName = rawConfig.entryName;
const dllBundleName = rawConfig.dllName;
const dllManifestName = rawConfig.dllName;
const dllStyleName = rawConfig.styleName;
const dllEntryExt = rawConfig.entryExt;
const dllBundleExt = rawConfig.dllExt;
const dllManifestExt = rawConfig.manifestExt;
const dllStyleExt = rawConfig.styleExt;
const dllOutputPath = rawConfig.outputPath;
const dllPublicPath = rawConfig.publicPath;
const dllBundleFilename = `${dllBundleName}${dllBundleExt}`;
const dllManifestPath = `${dllOutputPath}/${dllManifestName}${dllManifestExt}`;
const dllStyleFilename = `${dllStyleName}${dllStyleExt}`;
const babelLoaderCacheDir = rawConfig.babelLoaderCacheDir;
const threadLoaderPoolConfig = rawConfig.threadLoaderPoolConfig;
// Create webpack entry object key with the provided dllEntryName
dllChunks.reduce((dllEntryObj, chunk) => {
dllEntryObj[`${dllEntryName}${chunk}`] = [
`${dllOutputPath}/${dllEntryName}${chunk}${dllEntryExt}`,
];
return dllEntryObj;
}, dllEntry);
// Export dll config map
return {
dllAlias,
dllValidateSyntax,
dllNoParseRules,
dllDevMode,
dllContext,
dllEntry,
dllOutputPath,
dllPublicPath,
dllBundleName,
dllBundleFilename,
dllStyleFilename,
dllManifestPath,
babelLoaderCacheDir,
threadLoaderPoolConfig,
};
}
function common(config) {
return webpackMerge(generateDLL(config));
}
function optimized() {
return webpackMerge({
mode: 'production',
optimization: {
minimizer: [
new TerserPlugin({
// NOTE: we should not enable that option for now
// Since 2.0.0 terser-webpack-plugin is using jest-worker
// to run tasks in a pool of workers. Currently it looks like
// is requiring too much memory and break on large entry points
// compilations (like this) one. Also the gain we have enabling
// that option was barely noticed.
// https://github.com/webpack-contrib/terser-webpack-plugin/issues/143
parallel: false,
sourceMap: false,
cache: false,
extractComments: false,
terserOptions: {
compress: false,
mangle: false,
},
}),
],
},
});
}
function unoptimized() {
return webpackMerge({
mode: 'development',
});
}
export function configModel(rawConfig = {}) {
const config = extendRawConfig(rawConfig);
if (config.dllDevMode) {
return webpackMerge(common(config), unoptimized());
}
return webpackMerge(common(config), optimized());
}

37
src/optimize/dynamic_dll_plugin/dll_entry_template.js
View file

@ -1,37 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export function dllEntryTemplate(requirePaths = []) {
return requirePaths
.map((path) => `require('${path}');`)
.sort()
.join('\n');
}
export function dllEntryFileContentStringToArray(content = '') {
return content.split('\n');
}
export function dllEntryFileContentArrayToString(content = []) {
return content.join('\n');
}
export function dllMergeAllEntryFilesContent(content = []) {
return content.join('\n').split('\n').sort().join('\n');
}

354
src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js
View file

@ -1,354 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { DllCompiler } from './dll_compiler';
import { notInNodeModulesOrWebpackShims, inPluginNodeModules } from './dll_allowed_modules';
import { IS_KIBANA_DISTRIBUTABLE } from '../../legacy/utils';
import { dllEntryTemplate } from './dll_entry_template';
import RawModule from 'webpack/lib/RawModule';
import webpack from 'webpack';
import path from 'path';
import normalizePosixPath from 'normalize-path';
import fs from 'fs';
import { promisify } from 'util';
const realPathAsync = promisify(fs.realpath);
const DLL_ENTRY_STUB_MODULE_TYPE = 'javascript/dll-entry-stub';
export class DynamicDllPlugin {
constructor({ uiBundles, threadLoaderPoolConfig, logWithMetadata, maxCompilations = 1 }) {
this.logWithMetadata = logWithMetadata || (() => null);
this.dllCompiler = new DllCompiler(uiBundles, threadLoaderPoolConfig, logWithMetadata);
this.entryPaths = dllEntryTemplate();
this.afterCompilationEntryPaths = dllEntryTemplate();
this.maxCompilations = maxCompilations;
this.performedCompilations = 0;
this.forceDLLCreationFlag = !!(process && process.env && process.env.FORCE_DLL_CREATION);
}
async init() {
await this.dllCompiler.init();
this.entryPaths = await this.dllCompiler.readEntryFiles();
}
apply(compiler) {
// Just register the init basic hooks
// in order to run the init function
this.registerInitBasicHooks(compiler);
// The dll reference should always be bind to the
// main webpack config.
this.bindDllReferencePlugin(compiler);
// Verify if we must init and run the dynamic dll plugin tasks.
// We must run it every time we are not under a distributable env
if (!this.mustRunDynamicDllPluginTasks()) {
return;
}
// This call init all the DynamicDllPlugin tasks
// as it attaches the plugin to the main webpack
// lifecycle hooks needed to perform the logic
this.registerTasksHooks(compiler);
}
bindDllReferencePlugin(compiler) {
const rawDllConfig = this.dllCompiler.rawDllConfig;
const dllContext = rawDllConfig.context;
const dllManifestPaths = this.dllCompiler.getManifestPaths();
dllManifestPaths.forEach((dllChunkManifestPath) => {
new webpack.DllReferencePlugin({
context: dllContext,
manifest: dllChunkManifestPath,
}).apply(compiler);
});
}
registerInitBasicHooks(compiler) {
this.registerRunHook(compiler);
this.registerWatchRunHook(compiler);
}
registerTasksHooks(compiler) {
this.logWithMetadata(
['info', 'optimize:dynamic_dll_plugin'],
'Started dynamic dll plugin tasks'
);
this.registerBeforeCompileHook(compiler);
this.registerCompilationHook(compiler);
this.registerDoneHook(compiler);
}
registerRunHook(compiler) {
compiler.hooks.run.tapPromise('DynamicDllPlugin', async () => {
await this.init();
});
}
registerWatchRunHook(compiler) {
compiler.hooks.watchRun.tapPromise('DynamicDllPlugin', async () => {
await this.init();
});
}
registerBeforeCompileHook(compiler) {
compiler.hooks.beforeCompile.tapPromise('DynamicDllPlugin', async ({ normalModuleFactory }) => {
normalModuleFactory.hooks.factory.tap('DynamicDllPlugin', (actualFactory) => (params, cb) => {
// This is used in order to avoid the cache for DLL modules
// resolved from other dependencies
normalModuleFactory.cachePredicate = (module) =>
!(module.stubType === DLL_ENTRY_STUB_MODULE_TYPE);
// Overrides the normalModuleFactory module creation behaviour
// in order to understand the modules we need to add to the DLL
actualFactory(params, (error, module) => {
if (error || !module) {
cb(error, module);
} else {
this.mapNormalModule(module).then(
(m = module) => cb(undefined, m),
(error) => cb(error)
);
}
});
});
});
}
registerCompilationHook(compiler) {
compiler.hooks.compilation.tap('DynamicDllPlugin', (compilation) => {
compilation.hooks.needAdditionalPass.tap('DynamicDllPlugin', () => {
// Run the procedures in order to execute our dll compilation
// The process is very straightforward in it's conception:
//
// * 1 - loop through every compilation module in order to start building
// the dll entry paths arrays and assume it is the new entry paths
// * 1.1 - start from adding the modules already included into the dll, if any.
// * 1.2 - adding the new discovered stub modules
// * 1.3 - check if the module added to the entry path is from node_modules or
// webpackShims, otherwise throw an error.
// * 1.3.1 - for the entry path modules coming from webpackShims search for every
// require statements inside of them
// * 1.3.2 - discard the ones that are not js dependencies
// * 1.3.3 - add those new discovered dependencies inside the webpackShims to the
// entry paths array
// * 2 - compare the built entry paths and compares it to the old one (if any)
// * 3 - runs a new dll compilation in case there is none old entry paths or if the
// new built one differs from the old one.
//
const rawDllConfig = this.dllCompiler.rawDllConfig;
const dllContext = rawDllConfig.context;
const dllOutputPath = rawDllConfig.outputPath;
const requiresMap = {};
for (const module of compilation.modules) {
// re-include requires for modules already handled by the dll
if (module.delegateData) {
const absoluteResource = path.resolve(dllContext, module.userRequest);
if (
absoluteResource.includes('node_modules') ||
absoluteResource.includes('webpackShims')
) {
// NOTE: normalizePosixPath is been used as we only want to have posix
// paths inside our final dll entry file
requiresMap[
normalizePosixPath(path.relative(dllOutputPath, absoluteResource))
] = true;
}
}
// include requires for modules that need to be added to the dll
if (module.stubType === DLL_ENTRY_STUB_MODULE_TYPE) {
requiresMap[
normalizePosixPath(path.relative(dllOutputPath, module.stubResource))
] = true;
}
}
// Sort and join all the discovered require deps
// in order to create a consistent entry file
this.afterCompilationEntryPaths = dllEntryTemplate(Object.keys(requiresMap));
// The dll compilation will run if on of the following conditions return true:
// 1 - the new generated entry paths are different from the
// old ones
// 2 - if no dll bundle is yet created
// 3 - if this.forceDLLCreationFlag were set from the node env var FORCE_DLL_CREATION and
// we are not running over the distributable. If we are running under the watch optimizer,
// this.forceDLLCreationFlag will only be applied in the very first execution,
// then will be set to false
compilation.needsDLLCompilation =
this.afterCompilationEntryPaths !== this.entryPaths ||
!this.dllCompiler.dllsExistsSync() ||
(this.isToForceDLLCreation() && this.performedCompilations === 0);
this.entryPaths = this.afterCompilationEntryPaths;
// Only run this info log in the first performed dll compilation
// per each execution run
if (this.performedCompilations === 0) {
this.logWithMetadata(
['info', 'optimize:dynamic_dll_plugin'],
compilation.needsDLLCompilation
? 'Need to compile the client vendors dll'
: 'No need to compile client vendors dll'
);
}
return compilation.needsDLLCompilation;
});
});
}
registerDoneHook(compiler) {
compiler.hooks.done.tapPromise('DynamicDllPlugin', async (stats) => {
if (stats.compilation.needsDLLCompilation) {
// Run the dlls compiler and increment
// the performed compilations
//
// NOTE: check the need for this extra try/catch after upgrading
// past webpack v4.29.3. For now it is needed so we can log the error
// otherwise the error log we'll get will be something like: [fatal] [object Object]
try {
await this.runDLLCompiler(compiler);
} catch (error) {
this.logWithMetadata(['error', 'optimize:dynamic_dll_plugin'], error.message);
throw error;
}
return;
}
this.performedCompilations = 0;
// reset this flag var set from the node env FORCE_DLL_CREATION on init,
// has the force_dll_creation is only valid for the very first run
if (this.forceDLLCreationFlag) {
this.forceDLLCreationFlag = false;
}
this.logWithMetadata(
['info', 'optimize:dynamic_dll_plugin'],
'Finished all dynamic dll plugin tasks'
);
});
}
isToForceDLLCreation() {
return this.forceDLLCreationFlag;
}
mustRunDynamicDllPluginTasks() {
return !IS_KIBANA_DISTRIBUTABLE || this.isToForceDLLCreation();
}
async mapNormalModule(module) {
// ignore anything that doesn't have a resource (ignored) or is already delegating to the DLL
if (!module.resource || module.delegateData) {
return;
}
// ignore anything that needs special loaders or config
if (module.request.includes('!') || module.request.includes('?')) {
return;
}
// ignore files that are not in node_modules
if (notInNodeModulesOrWebpackShims(module.resource)) {
return;
}
// also ignore files that are symlinked into node_modules, but only
// do the `realpath` call after checking the plain resource path
if (notInNodeModulesOrWebpackShims(await realPathAsync(module.resource))) {
return;
}
const dirs = module.resource.split(path.sep);
const nodeModuleName = dirs[dirs.lastIndexOf('node_modules') + 1];
// ignore webpack loader modules
if (nodeModuleName.endsWith('-loader')) {
return;
}
// ignore modules from plugins
if (inPluginNodeModules(module.resource)) {
return;
}
// also ignore files that are symlinked into plugins node_modules, but only
// do the `realpath` call after checking the plain resource path
if (inPluginNodeModules(await realPathAsync(module.resource))) {
return;
}
// This is a StubModule (as a RawModule) in order
// to mimic the missing modules from the DLL and
// also hold useful metadata
const stubModule = new RawModule(
`/* pending dll entry */`,
`dll pending:${module.resource}`,
module.resource
);
stubModule.stubType = DLL_ENTRY_STUB_MODULE_TYPE;
stubModule.stubResource = module.resource;
stubModule.stubOriginalModule = module;
return stubModule;
}
async assertMaxCompilations() {
// Logic to run the max compilation requirements.
// Only enable this for CI builds in order to ensure
// we have an healthy dll ecosystem.
if (this.performedCompilations === this.maxCompilations) {
throw new Error(
'All the allowed dll compilations were already performed and one more is needed which is not possible'
);
}
}
async runDLLCompiler(mainCompiler) {
const runCompilerErrors = [];
try {
await this.dllCompiler.run(this.entryPaths);
} catch (e) {
runCompilerErrors.push(e);
}
try {
await this.assertMaxCompilations();
} catch (e) {
runCompilerErrors.push(e);
}
// We need to purge the cache into the inputFileSystem
// for every single built in previous compilation
// that we rely in next ones.
this.dllCompiler
.getManifestPaths()
.forEach((chunkDllManifestPath) => mainCompiler.inputFileSystem.purge(chunkDllManifestPath));
this.performedCompilations++;
if (!runCompilerErrors.length) {
return;
}
throw new Error(runCompilerErrors.join('\n-'));
}
}

21
src/optimize/dynamic_dll_plugin/index.js
View file

@ -1,21 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { DynamicDllPlugin } from './dynamic_dll_plugin';
export { DllCompiler } from './dll_compiler';

5
src/optimize/optimize_mixin.ts
View file

@ -22,8 +22,6 @@ import Hapi from 'hapi';
// @ts-ignore not TS yet
import FsOptimizer from './fs_optimizer';
import { createBundlesRoute } from './bundles_route';
// @ts-ignore not TS yet
import { DllCompiler } from './dynamic_dll_plugin';
import { fromRoot } from '../core/server/utils';
import { getNpUiPluginPublicDirs } from './np_ui_plugin_public_dirs';
import KbnServer, { KibanaConfig } from '../legacy/server/kbn_server';
@ -40,7 +38,7 @@ export const optimizeMixin = async (
// bundles in a "middleware" style.
//
// the server listening on 5601 may be restarted a number of times, depending
// on the watch setup managed by the cli. It proxies all bundles/* and built_assets/dlls/*
// on the watch setup managed by the cli. It proxies all bundles/*
// requests to the other server. The server on 5602 is long running, in order
// to prevent complete rebuilds of the optimize content.
const watch = config.get('optimize.watch');
@ -53,7 +51,6 @@ export const optimizeMixin = async (
server.route(
createBundlesRoute({
regularBundlesPath: uiBundles.getWorkingDir(),
dllBundlesPath: DllCompiler.getRawDllConfig().outputPath,
basePublicPath: config.get('server.basePath'),
builtCssPath: fromRoot('built_assets/css'),
npUiPluginPublicDirs: getNpUiPluginPublicDirs(kbnServer),

10
src/optimize/watch/optmzr_role.js
View file

@ -17,12 +17,8 @@
* under the License.
*/
import { resolve } from 'path';
import WatchServer from './watch_server';
import WatchOptimizer, { STATUS } from './watch_optimizer';
import { DllCompiler } from '../dynamic_dll_plugin';
import { WatchCache } from './watch_cache';
import { getNpUiPluginPublicDirs } from '../np_ui_plugin_public_dirs';
export default async (kbnServer, kibanaHapiServer, config) => {
@ -36,12 +32,6 @@ export default async (kbnServer, kibanaHapiServer, config) => {
sourceMaps: config.get('optimize.sourceMaps'),
workers: config.get('optimize.workers'),
prebuild: config.get('optimize.watchPrebuild'),
watchCache: new WatchCache({
logWithMetadata,
outputPath: config.get('path.data'),
dllsPath: DllCompiler.getRawDllConfig().outputPath,
cachePath: resolve(kbnServer.uiBundles.getCacheDirectory(), '../'),
}),
});
const server = new WatchServer(

2
src/optimize/watch/watch.js
View file

@ -32,7 +32,7 @@ export default async (kbnServer) => {
* while the optimizer is running
*
* server: this process runs the entire kibana server and proxies
* all requests for /bundles/* or /built_assets/dlls/* to the optmzr process
* all requests for /bundles/* to the optmzr process
*
* @param {string} process.env.kbnWorkerType
*/

189
src/optimize/watch/watch_cache.ts
View file

@ -1,189 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createHash } from 'crypto';
import { readFile, writeFile, readdir, unlink, rmdir } from 'fs';
import { resolve } from 'path';
import { promisify } from 'util';
import path from 'path';
import del from 'del';
import normalizePosixPath from 'normalize-path';
const readAsync = promisify(readFile);
const writeAsync = promisify(writeFile);
const readdirAsync = promisify(readdir);
const unlinkAsync = promisify(unlink);
const rmdirAsync = promisify(rmdir);
interface Params {
logWithMetadata: (tags: string[], message: string, metadata?: { [key: string]: any }) => void;
outputPath: string;
dllsPath: string;
cachePath: string;
}
interface WatchCacheStateContent {
optimizerConfigSha?: string;
yarnLockSha?: string;
}
export class WatchCache {
private readonly logWithMetadata: Params['logWithMetadata'];
private readonly outputPath: Params['outputPath'];
private readonly dllsPath: Params['dllsPath'];
private readonly cachePath: Params['cachePath'];
private readonly cacheState: WatchCacheStateContent;
private statePath: string;
private diskCacheState: WatchCacheStateContent;
private isInitialized: boolean;
constructor(params: Params) {
this.logWithMetadata = params.logWithMetadata;
this.outputPath = params.outputPath;
this.dllsPath = params.dllsPath;
this.cachePath = params.cachePath;
this.isInitialized = false;
this.statePath = '';
this.cacheState = {};
this.diskCacheState = {};
this.cacheState.yarnLockSha = '';
this.cacheState.optimizerConfigSha = '';
}
public async tryInit() {
if (!this.isInitialized) {
this.statePath = resolve(this.outputPath, 'watch_optimizer_cache_state.json');
this.diskCacheState = await this.read();
this.cacheState.yarnLockSha = await this.buildYarnLockSha();
this.cacheState.optimizerConfigSha = await this.buildOptimizerConfigSha();
this.isInitialized = true;
}
}
public async tryReset() {
await this.tryInit();
if (!this.isResetNeeded()) {
return;
}
await this.reset();
}
public async reset() {
this.logWithMetadata(['info', 'optimize:watch_cache'], 'The optimizer watch cache will reset');
// start by deleting the state file to lower the
// amount of time that another process might be able to
// successfully read it once we decide to delete it
await del(this.statePath, { force: true });
// delete everything in optimize/.cache directory
await recursiveDelete(normalizePosixPath(this.cachePath));
// delete dlls
await del(this.dllsPath);
// re-write new cache state file
await this.write();
this.logWithMetadata(['info', 'optimize:watch_cache'], 'The optimizer watch cache has reset');
}
private async buildShaWithMultipleFiles(filePaths: string[]) {
const shaHash = createHash('sha1');
for (const filePath of filePaths) {
try {
shaHash.update(await readAsync(filePath, 'utf8'), 'utf8');
} catch (e) {
/* no-op */
}
}
return shaHash.digest('hex');
}
private async buildYarnLockSha() {
const kibanaYarnLock = resolve(__dirname, '../../../yarn.lock');
return await this.buildShaWithMultipleFiles([kibanaYarnLock]);
}
private async buildOptimizerConfigSha() {
const baseOptimizer = resolve(__dirname, '../base_optimizer.js');
const dynamicDllConfigModel = resolve(__dirname, '../dynamic_dll_plugin/dll_config_model.js');
const dynamicDllPlugin = resolve(__dirname, '../dynamic_dll_plugin/dynamic_dll_plugin.js');
return await this.buildShaWithMultipleFiles([
baseOptimizer,
dynamicDllConfigModel,
dynamicDllPlugin,
]);
}
private isResetNeeded() {
return this.hasYarnLockChanged() || this.hasOptimizerConfigChanged();
}
private hasYarnLockChanged() {
return this.cacheState.yarnLockSha !== this.diskCacheState.yarnLockSha;
}
private hasOptimizerConfigChanged() {
return this.cacheState.optimizerConfigSha !== this.diskCacheState.optimizerConfigSha;
}
private async write() {
await writeAsync(this.statePath, JSON.stringify(this.cacheState, null, 2), 'utf8');
this.diskCacheState = this.cacheState;
}
private async read(): Promise<WatchCacheStateContent> {
try {
return JSON.parse(await readAsync(this.statePath, 'utf8'));
} catch (error) {
return {};
}
}
}
/**
* Recursively deletes a folder. This is a workaround for a bug in `del` where
* very large folders (with 84K+ files) cause a stack overflow.
*/
async function recursiveDelete(directory: string) {
try {
const entries = await readdirAsync(directory, { withFileTypes: true });
await Promise.all(
entries.map((entry) => {
const absolutePath = path.join(directory, entry.name);
return entry.isDirectory() ? recursiveDelete(absolutePath) : unlinkAsync(absolutePath);
})
);
return rmdirAsync(directory);
} catch (error) {
if (error.code !== 'ENOENT') {
throw error;
}
}
}

6
src/optimize/watch/watch_optimizer.js
View file

@ -19,7 +19,6 @@
import BaseOptimizer from '../base_optimizer';
import { createBundlesRoute } from '../bundles_route';
import { DllCompiler } from '../dynamic_dll_plugin';
import { fromRoot } from '../../core/server/utils';
import * as Rx from 'rxjs';
import { mergeMap, take } from 'rxjs/operators';
@ -35,7 +34,6 @@ export default class WatchOptimizer extends BaseOptimizer {
constructor(opts) {
super(opts);
this.prebuild = opts.prebuild || false;
this.watchCache = opts.watchCache;
this.status$ = new Rx.ReplaySubject(1);
}
@ -43,9 +41,6 @@ export default class WatchOptimizer extends BaseOptimizer {
this.initializing = true;
this.initialBuildComplete = false;
// try reset the watch optimizer cache
await this.watchCache.tryReset();
// log status changes
this.status$.subscribe(this.onStatusChangeHandler);
await this.uiBundles.resetBundleDir();
@ -120,7 +115,6 @@ export default class WatchOptimizer extends BaseOptimizer {
npUiPluginPublicDirs: npUiPluginPublicDirs,
buildHash,
regularBundlesPath: this.compiler.outputPath,
dllBundlesPath: DllCompiler.getRawDllConfig().outputPath,
basePublicPath: basePath,
builtCssPath: fromRoot('built_assets/css'),
})

12
tasks/config/karma.js
View file

@ -21,7 +21,6 @@ import { dirname } from 'path';
import { times } from 'lodash';
import { makeJunitReportPath } from '@kbn/test';
import * as UiSharedDeps from '@kbn/ui-shared-deps';
import { DllCompiler } from '../../src/optimize/dynamic_dll_plugin';
const TOTAL_CI_SHARDS = 4;
const ROOT = dirname(require.resolve('../../package.json'));
@ -63,12 +62,6 @@ module.exports = function (grunt) {
),
`http://localhost:5610/${buildHash}/bundles/kbn-ui-shared-deps/${UiSharedDeps.jsFilename}`,
`http://localhost:5610/${buildHash}/built_assets/dlls/vendors_runtime.bundle.dll.js`,
...DllCompiler.getRawDllConfig().chunks.map(
(chunk) =>
`http://localhost:5610/${buildHash}/built_assets/dlls/vendors${chunk}.bundle.dll.js`
),
shardNum === undefined
? `http://localhost:5610/${buildHash}/bundles/tests.bundle.js`
: `http://localhost:5610/${buildHash}/bundles/tests.bundle.js?shards=${TOTAL_CI_SHARDS}&shard_num=${shardNum}`,
@ -77,10 +70,6 @@ module.exports = function (grunt) {
// this causes tilemap tests to fail, probably because the eui styles haven't been
// included in the karma harness a long some time, if ever
// `http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.lightCssDistFilename}`,
...DllCompiler.getRawDllConfig().chunks.map(
(chunk) =>
`http://localhost:5610/${buildHash}/built_assets/dlls/vendors${chunk}.style.dll.css`
),
`http://localhost:5610/${buildHash}/bundles/tests.style.css`,
];
}
@ -133,7 +122,6 @@ module.exports = function (grunt) {
'/tests/': 'http://localhost:5610/tests/',
'/test_bundle/': 'http://localhost:5610/test_bundle/',
[`/${buildHash}/bundles/`]: `http://localhost:5610/${buildHash}/bundles/`,
[`/${buildHash}/built_assets/dlls/`]: `http://localhost:5610/${buildHash}/built_assets/dlls/`,
},
client: {