fixed the test bundling

This commit is contained in:
spalger 2015-08-07 11:49:56 -07:00
parent 671319a782
commit cd5b6656aa
27 changed files with 263 additions and 253 deletions

View file

@ -1,4 +1,4 @@
require('babel/register');
require('babel/register')(require('./src/optimize/babelOptions'));
module.exports = function (grunt) {
// set the config once before calling load-grunt-config

View file

@ -49,10 +49,8 @@ module.exports = function (program) {
)
.option('--plugins <path>', 'an alias for --plugin-dir', pluginDirCollector)
.option('--dev', 'Run the server with development mode defaults')
.option('--no-watch', 'Prevent watching, use with --dev to prevent server restarts')
.option('--no-lazy', 'Prevent lazy optimization of applications, only works with --dev')
.action(function (opts) {
if (opts.dev && !isWorker) {
if (opts.dev && opts.lazy && !isWorker) {
// stop processing the action and handoff to cluster manager
let ClusterManager = require('../cluster/ClusterManager');
new ClusterManager(opts);
@ -68,7 +66,7 @@ module.exports = function (program) {
if (opts.dev) {
set('env', 'development');
set('optimize.lazy', opts.lazy);
set('optimize.watch', true);
}
if (opts.elasticsearch) set('elasticsearch.url', opts.elasticsearch);

View file

@ -89,7 +89,8 @@ class BaseOptimizer {
modulesDirectories: ['node_modules'],
loaderPostfixes: ['-loader', ''],
root: fromRoot('.'),
alias: this.env.aliases
alias: this.env.aliases,
unsafeCache: [/\/node_modules\//]
}
};
}

View file

@ -2,13 +2,9 @@ let { fromNode } = require('bluebird');
let BaseOptimizer = require('./BaseOptimizer');
module.exports = class FsOptimizer extends BaseOptimizer {
async init() {
await this.bundles.writeEntryFiles();
await this.bundles.filterCachedBundles();
}
async run() {
await this.initCompiler();
await fromNode(cb => {
this.compiler.run((err, stats) => {
if (err || !stats) return cb(err);
@ -23,5 +19,13 @@ module.exports = class FsOptimizer extends BaseOptimizer {
}
});
});
this.compiler = null;
}
getConfig() {
let config = BaseOptimizer.prototype.getConfig.call(this);
config.cache = false;
return config;
}
};

View file

@ -73,11 +73,13 @@ module.exports = class LiveOptimizer extends BaseOptimizer {
return content || '';
}
catch (error) {
if (error && error.message.match(/Path doesn't exist/)) {
error = Boom.notFound();
if (error && error.message.match(/path doesn't exist/i)) {
error = Boom.notFound(error.message);
} else {
console.log(error.stack);
}
console.log(error.stack);
throw error;
}
}

View file

@ -1,67 +0,0 @@
let { once, template } = require('lodash');
let { resolve } = require('path');
let { readFileSync } = require('fs');
let src = require('requirefrom')('src');
let fromRoot = src('utils/fromRoot');
let LiveOptimizer = src('optimize/LiveOptimizer');
let id = 'tests';
let globAll = require('./globAll');
let testEntryFileTemplate = template(readFileSync(resolve(__dirname, './testBundleEntry.js.tmpl')));
class TestBundler {
constructor(kbnServer) {
this.hapi = kbnServer.server;
this.plugins = kbnServer.plugins;
this.bundleDir = kbnServer.config.get('optimize.bundleDir');
this.init = once(this.init);
}
async init() {
await this.findTestFiles();
await this.setupOptimizer();
}
async findTestFiles() {
await globAll(fromRoot('.'), [
'src/**/public/**/__tests__/**/*.js',
'installedPlugins/**/public/**/__tests__/**/*.js'
]);
}
async setupOptimizer(testFiles) {
let deps = [];
let modules = [];
if (testFiles) {
modules = modules.concat(testFiles);
}
this.plugins.forEach(function (plugin) {
if (!plugin.app) return;
modules = modules.concat(plugin.app.getModules());
deps = deps.concat(plugin.app.getRelatedPlugins());
});
this.optimizer = new LiveOptimizer({
sourceMaps: true,
bundleDir: this.bundleDir,
entries: [
{
id: id,
deps: deps,
modules: modules,
template: testEntryFileTemplate
}
],
plugins: this.plugins
});
await this.optimizer.init();
this.optimizer.bindToServer(this.hapi);
}
}
module.exports = TestBundler;

View file

@ -1,17 +0,0 @@
let _ = require('lodash');
let { resolve } = require('path');
let { promisify } = require('bluebird');
let { all } = require('bluebird');
let glob = promisify(require('glob'));
module.exports = function (path, patterns) {
return all([].concat(patterns || []))
.map(function (pattern) {
return glob(pattern, { cwd: path, ignore: '**/_*.js' });
})
.then(_.flatten)
.then(_.uniq)
.map(function (match) {
return resolve(path, match);
});
};

View file

@ -1,52 +0,0 @@
module.exports = function (kbnServer, server, config) {
if (!config.get('env.dev')) return;
let Boom = require('boom');
let src = require('requirefrom')('src');
let fromRoot = src('utils/fromRoot');
let TestBundler = require('./TestBundler');
let bundler = new TestBundler(kbnServer, fromRoot('src'));
let renderPromise = false;
let renderComplete = false;
function send(reply, part, mimeType) {
if (!renderPromise || (part === 'bundle' && renderComplete)) {
renderPromise = bundler.render();
renderComplete = false;
renderPromise.then(function () { renderComplete = true; });
}
renderPromise.then(function (output) {
if (!output || !output.bundle) {
return reply(Boom.create(500, 'failed to build test bundle'));
}
return reply(output[part]).type(mimeType);
}, reply);
}
server.route({
path: '/bundles/tests.bundle.js',
method: 'GET',
handler: function (req, reply) {
send(reply, 'bundle', 'application/javascript');
}
});
server.route({
path: '/bundles/tests.bundle.js.map',
method: 'GET',
handler: function (req, reply) {
send(reply, 'sourceMap', 'text/plain');
}
});
server.route({
path: '/bundles/tests.bundle.style.css',
method: 'GET',
handler: function (req, reply) {
send(reply, 'style', 'text/css');
}
});
};

View file

@ -1,20 +0,0 @@
/**
* Optimized application entry file
*
* This is programatically created and updated, do not modify
*
* built using: <%= optimizerTagline %>
* includes code from:
<%
entry.deps.sort().forEach(function (plugin) {
print(` * - ${plugin}\n`);
})
%> *
*/
require('ui/testHarness');
<%
entry.modules.slice(0).reverse().forEach(function (id) {
print(`require('${id}');\n`);
});
%>require('ui/testHarness').bootstrap();

View file

@ -1,39 +0,0 @@
/**
* Optimize source code in a way that is suitable for distribution
*/
module.exports = async (kbnServer, server, config) => {
let FsOptimizer = require('../FsOptimizer');
let optimizer = new FsOptimizer({
env: kbnServer.bundles.env,
bundles: kbnServer.bundles,
sourceMaps: config.get('optimize.sourceMaps')
});
server.exposeStaticDir('/bundles/{path*}', kbnServer.bundles.env.workingDir);
try {
await optimizer.init();
let bundleIds = optimizer.bundles.getIds();
if (bundleIds.length) {
server.log(
['warning', 'optimize'],
`Optimizing bundles for ${bundleIds.join(', ')}. This may take a few minutes.`
);
} else {
server.log(
['debug', 'optimize'],
`All bundles are cached and ready to go!`
);
}
await optimizer.run();
} catch (e) {
if (e.stats) {
server.log(['error'], e.stats.toString({ colors: true }));
}
server.log(['fatal'], e);
process.exit(1); // eslint-disable-line no-process-exit
}
};

View file

@ -1,7 +1,63 @@
module.exports = function (kbnServer, server, config) {
module.exports = async (kbnServer, server, config) => {
if (!config.get('optimize.enabled')) return;
// the lazy optimizer sets up two threads, one is the server listening
// on 5601 and the other is a server listening on 5602 that builds the
// bundles in a "middleware" style.
//
// the server listening on 5601 may be restarted a number of times, depending
// on the watch setup managed by the cli. It proxies all bundles/* requests to
// the other server. The server on 5602 is long running, in order to prevent
// complete rebuilds of the optimize content.
let lazy = config.get('optimize.lazy');
let strategy = lazy ? require('./lazy/lazy') : require('./dist/dist');
return kbnServer.mixin(strategy);
if (lazy) {
return await kbnServer.mixin(require('./lazy/lazy'));
}
let bundles = kbnServer.bundles;
await bundles.writeEntryFiles();
// in prod, only bundle what looks invalid or missing
if (config.get('env.prod')) bundles = await kbnServer.bundles.getInvalidBundles();
let bundleIds = bundles.getIds();
// we might not have any work to do
if (!bundleIds.length) {
server.log(
['debug', 'optimize'],
`All bundles are cached and ready to go!`
);
return;
}
// only require the FsOptimizer when we need to
let FsOptimizer = require('./FsOptimizer');
let optimizer = new FsOptimizer({
env: bundles.env,
bundles: bundles,
sourceMaps: config.get('optimize.sourceMaps')
});
server.exposeStaticDir('/bundles/{path*}', bundles.env.workingDir);
try {
server.log(
['warning', 'optimize'],
`Optimizing bundles for ${bundleIds.join(', ')}. This may take a few minutes.`
);
await optimizer.run();
server.log(
['info', 'optimize'],
`Optimization of ${bundleIds.join(', ')} complete.`
);
} catch (e) {
if (e.stats) {
server.log(['error'], e.stats.toString({ colors: true }));
}
server.log(['fatal'], e);
process.exit(1); // eslint-disable-line no-process-exit
}
};

View file

@ -1,8 +1,11 @@
module.exports = function (kibana) {
module.exports = (kibana) => {
if (!kibana.config.get('env.dev')) return;
let { union } = require('lodash');
let utils = require('requirefrom')('src/utils');
let fromRoot = utils('fromRoot');
let findSourceFiles = utils('findSourceFiles');
return new kibana.Plugin({
uiExports: {
@ -10,6 +13,30 @@ module.exports = function (kibana) {
'plugins/devMode/visDebugSpyPanel'
],
bundle: async (UiBundle, env, apps) => {
let modules = [];
// add the modules from all of the apps
for (let app of apps) {
modules = union(modules, app.getModules());
}
let testFiles = await findSourceFiles([
'src/**/public/**/__tests__/**/*.js',
'installedPlugins/*/public/**/__tests__/**/*.js'
]);
for (let f of testFiles) modules.push(f);
return new UiBundle({
id: 'tests',
modules: modules,
template: require('./testsEntryTemplate'),
env: env
});
},
modules: {
ngMock$: fromRoot('src/plugins/devMode/public/ngMock'),
fixtures: fromRoot('src/fixtures'),

View file

@ -0,0 +1,34 @@
module.exports = require('lodash').template(
`
/**
* Optimized application entry file
*
* This is programatically created and updated, do not modify
*
* context: <%= JSON.stringify(env.context) %>
* includes code from:
<%
env.pluginInfo.sort().forEach(function (plugin, i) {
if (i > 0) print('\\n');
print(' * - ' + plugin);
});
%>
*
*/
require('ui/testHarness');
<%
bundle.modules.forEach(function (id, i) {
if (i > 0) print('\\n');
print(\`require('\${id}');\`);
});
%>
require('ui/testHarness').bootstrap(/* go! */);
`
);

View file

@ -15,8 +15,7 @@ module.exports = Joi.object({
env: Joi.object({
name: Joi.string().default(Joi.ref('$env')),
dev: Joi.boolean().default(Joi.ref('$dev')),
prod: Joi.boolean().default(Joi.ref('$prod')),
test: Joi.boolean().default(Joi.ref('$test')),
prod: Joi.boolean().default(Joi.ref('$prod'))
}).default(),
pid: Joi.object({
@ -72,9 +71,10 @@ module.exports = Joi.object({
optimize: Joi.object({
enabled: Joi.boolean().default(true),
bundleFilter: Joi.string().default('*'),
bundleDir: Joi.string().default(fromRoot('optimize/bundles')),
viewCaching: Joi.boolean().default(Joi.ref('$prod')),
watch: Joi.boolean().default(Joi.ref('$dev')),
watch: Joi.boolean().default(false),
lazy: Joi.boolean().when('watch', {
is: true,
then: Joi.default(true),

View file

@ -4,6 +4,16 @@ let moment = require('moment');
let LogFormat = require('./LogFormat');
let statuses = [
'err',
'info',
'error',
'warning',
'fatal',
'status',
'debug'
];
let typeColors = {
log: 'blue',
req: 'green',
@ -39,7 +49,9 @@ module.exports = class KbnLoggerJsonFormat extends LogFormat {
let tags = _(data.tags)
.sortBy(function (tag) {
return color(tag) === _.identity ? `1${tag}` : `0${tag}`;
if (color(tag) === _.identity) return `2${tag}`;
if (_.includes(statuses, tag)) return `0${tag}`;
return `1${tag}`;
})
.reduce(function (s, t) {
return s + `[${ color(t)(t) }]`;

View file

@ -3,9 +3,7 @@ module.exports = async (kbnServer, server, config) => {
let { fromNode } = require('bluebird');
let { readdir, stat } = require('fs');
let { resolve } = require('path');
let src = require('requirefrom')('src');
let { each } = src('utils/async');
let { each } = require('bluebird');
var Plugins = require('./Plugins');
var plugins = kbnServer.plugins = new Plugins(kbnServer);

View file

@ -8,17 +8,17 @@ let stat = promisify(require('fs').stat);
module.exports = class UiBundle {
constructor(opts) {
opts = opts || {};
this.env = opts.env;
opts = opts || {};
this.id = opts.id;
this.plugins = opts.plugins;
this.modules = opts.modules;
this.template = opts.template;
this.env = opts.env;
let pathBase = join(this.env.workingDir, this.id);
this.entryPath = `${pathBase}.entry.js`;
this.outputPath = `${pathBase}.bundle.js`;
}
renderContent() {
@ -60,7 +60,6 @@ module.exports = class UiBundle {
toJSON() {
return {
id: this.id,
plugins: this.plugins,
modules: this.modules,
entryPath: this.entryPath,
outputPath: this.outputPath

View file

@ -1,6 +1,7 @@
let { pull, transform, pluck } = require('lodash');
let { join } = require('path');
let { resolve, promisify } = require('bluebird');
let { makeRe } = require('minimatch');
let rimraf = promisify(require('rimraf'));
let mkdirp = promisify(require('mkdirp'));
let unlink = promisify(require('fs').unlink);
@ -11,13 +12,28 @@ let UiBundle = require('./UiBundle');
let appEntryTemplate = require('./appEntryTemplate');
class UiBundleCollection {
constructor(bundlerEnv) {
constructor(bundlerEnv, filter) {
this.each = [];
this.env = bundlerEnv;
this.filter = makeRe(filter || '*', {
noglobstar: true,
noext: true,
matchBase: true
});
}
add(bundle) {
if (!(bundle instanceof UiBundle)) {
throw new TypeError('expected bundle to be an instance of UiBundle');
}
if (this.filter.test(bundle.id)) {
this.each.push(bundle);
}
}
addApp(app) {
this.each.push(new UiBundle({
this.add(new UiBundle({
id: app.id,
modules: app.getModules(),
template: appEntryTemplate,
@ -43,11 +59,17 @@ class UiBundleCollection {
}
}
async filterCachedBundles() {
for (let bundle of this.each.slice()) {
async getInvalidBundles() {
let invalids = new UiBundleCollection(this.env);
for (let bundle of this.each) {
let exists = await bundle.checkForExistingOutput();
if (exists) pull(this.each, bundle);
if (!exists) {
invalids.add(bundle);
}
}
return invalids;
}
toWebpackEntries() {

View file

@ -9,6 +9,7 @@ class UiExports {
this.aliases = {};
this.exportConsumer = _.memoize(this.exportConsumer);
this.consumers = [];
this.bundleProviders = [];
}
consumePlugin(plugin) {
@ -54,6 +55,11 @@ class UiExports {
this.aliases[type] = _.union(this.aliases[type] || [], spec);
};
case 'bundle':
return (plugin, spec) => {
this.bundleProviders.push(spec);
};
case 'aliases':
return (plugin, specs) => {
_.forOwn(specs, (spec, adhocType) => {
@ -94,6 +100,10 @@ class UiExports {
getHiddenApp(id) {
return this.apps.hidden.byId[id];
}
getBundleProviders() {
return this.bundleProviders;
}
}
module.exports = UiExports;

View file

@ -7,6 +7,7 @@ module.exports = require('lodash').template(
* This is programatically created and updated, do not modify
*
* context: <%= JSON.stringify(env.context) %>
* includes code from:
<%
env.pluginInfo.sort().forEach(function (plugin) {
@ -19,11 +20,14 @@ module.exports = require('lodash').template(
require('ui/chrome');
<%
bundle.modules.forEach(function (id) {
bundle.modules
.filter(function (id) {
return id !== 'ui/chrome';
})
.forEach(function (id, i) {
if (id !== 'ui/chrome') {
print(\`require('\${id}');\n\`);
}
if (i > 0) print('\\n');
print(\`require('\${id}');\`);
});

View file

@ -1,10 +1,11 @@
module.exports = function (kbnServer, server, config) {
module.exports = async (kbnServer, server, config) => {
let _ = require('lodash');
let Boom = require('boom');
let formatUrl = require('url').format;
let { join, resolve } = require('path');
let UiExports = require('./UiExports');
let UiBundle = require('./UiBundle');
let UiBundleCollection = require('./UiBundleCollection');
let UiBundlerEnv = require('./UiBundlerEnv');
@ -21,12 +22,17 @@ module.exports = function (kbnServer, server, config) {
uiExports.consumePlugin(plugin);
}
let bundles = kbnServer.bundles = new UiBundleCollection(bundlerEnv);
let bundles = kbnServer.bundles = new UiBundleCollection(bundlerEnv, config.get('optimize.bundleFilter'));
for (let app of uiExports.getAllApps()) {
bundles.addApp(app);
}
for (let gen of uiExports.getBundleProviders()) {
let bundle = await gen(UiBundle, bundlerEnv, uiExports.getAllApps());
if (bundle) bundles.add(bundle);
}
// render all views from the ui/views directory
server.setupViews(resolve(__dirname, 'views'));

View file

@ -1,9 +0,0 @@
let { all } = require('bluebird');
exports.each = async (arr, fn) => {
await all(arr.map(fn)).return(undefined);
};
exports.map = async (arr, fn) => {
await all(arr.map(fn));
};

View file

@ -0,0 +1,41 @@
let { chain, memoize } = require('lodash');
let { resolve } = require('path');
let { map, fromNode } = require('bluebird');
let fromRoot = require('./fromRoot');
let { Glob } = require('glob');
let findSourceFiles = async (patterns, cwd = fromRoot('.')) => {
patterns = [].concat(patterns || []);
let matcheses = await map(patterns, async pattern => {
return await fromNode(cb => {
let g = new Glob(pattern, {
cwd: cwd,
ignore: [
'node_modules/**/*',
'bower_components/**/*',
'**/_*.js'
],
symlinks: findSourceFiles.symlinks,
statCache: findSourceFiles.statCache,
realpathCache: findSourceFiles.realpathCache,
cache: findSourceFiles.cache
}, cb);
});
});
return chain(matcheses)
.flatten()
.uniq()
.map(match => resolve(cwd, match))
.value();
};
findSourceFiles.symlinks = {};
findSourceFiles.statCache = {};
findSourceFiles.realpathCache = {};
findSourceFiles.cache = {};
module.exports = findSourceFiles;

View file

@ -3,15 +3,15 @@ module.exports = function (grunt) {
let root = p => resolve(__dirname, '../../', p);
return {
devServer: {
testServer: {
options: {
wait: false,
ready: /\[optimize\]\[status\] Status changed from [a-zA-Z]+ to green/,
ready: /Server running/,
quiet: false,
failOnError: false
},
cmd: './bin/kibana',
args: ['--dev', '--no-watch', '--logging.json=false']
args: ['--env.name=development', '--logging.json=false', '--optimize.bundleFilter=tests']
}
};

View file

@ -14,7 +14,7 @@ module.exports = function (grunt) {
diff(['--name-only', '--cached'])
.then(function (files) {
// match these patterns
var patterns = grunt.config.get('lintThese');
var patterns = grunt.config.get('eslint.files.src');
files = files.split('\n').filter(Boolean).map(function (file) {
return resolve(root, file);
});

View file

@ -50,9 +50,9 @@ module.exports = function (grunt) {
};
};
grunt.registerTask('maybeStartKibana', maybeStartServer({
grunt.registerTask('maybeStartTestServer', maybeStartServer({
name: 'kibana-server',
port: grunt.option('port') || 5601,
tasks: ['run:devServer']
tasks: ['run:testServer']
}));
};

View file

@ -8,7 +8,7 @@ module.exports = function (grunt) {
grunt.task.run(_.compact([
'eslint:source',
'maybeStartKibana',
'maybeStartTestServer',
'simplemocha:all',
'karma:unit'
]));
@ -16,14 +16,14 @@ module.exports = function (grunt) {
grunt.registerTask('quick-test', function () {
grunt.task.run([
'maybeStartKibana',
'maybeStartTestServer',
'simplemocha:all',
'karma:unit'
]);
});
grunt.registerTask('test:watch', [
'maybeStartKibana',
'maybeStartTestServer',
'watch:test'
]);
};