Build multiple Kibana packages for production (#16313)

* Build packages before running ESLint on CI

* Add production task to kbn-build

* Ensure packages are bootstrapped before running the build

* Run ESLint on kbn-build

.eslintignore

@@ -10,6 +10,9 @@
 /ui_framework/doc_site/build
 /tasks/vendor
 /packages/*/node_modules
+/packages/*/target
 /packages/eslint-config-kibana
 /packages/eslint-plugin-kibana-custom
 /packages/kbn-build/dist
+/packages/kbn-build/vendor
+/packages/kbn-build/**/fixtures

package.json

@@ -225,6 +225,7 @@
     "@elastic/eslint-config-kibana": "link:packages/eslint-config-kibana",
     "@elastic/eslint-import-resolver-kibana": "1.0.0",
     "@elastic/eslint-plugin-kibana-custom": "link:packages/eslint-plugin-kibana-custom",
+    "@elastic/kbn-build": "link:packages/kbn-build",
     "angular-mocks": "1.4.7",
     "babel-eslint": "8.1.2",
     "backport": "2.2.0",

packages/eslint-config-kibana/package.json

@@ -1,5 +1,10 @@
 {
   "name": "@elastic/eslint-config-kibana",
+  "kibana": {
+    "build": {
+      "skip": true
+    }
+  },
   "version": "0.15.0",
   "description": "The eslint config used by the kibana team",
   "main": ".eslintrc.js",

packages/eslint-plugin-kibana-custom/package.json

@@ -1,5 +1,10 @@
 {
   "name": "@elastic/eslint-plugin-kibana-custom",
+  "kibana": {
+    "build": {
+      "skip": true
+    }
+  },
   "version": "1.1.0",
   "description": "Custom ESLint rules for Kibana",
   "repository": {

packages/kbn-build/README.md

@@ -140,11 +140,18 @@ And if needed, you can skip packages in the same way as for bootstrapping, e.g.
 yarn kbn run build --skip-kibana
 ```
 
+## Building packages for production
+
+The production build process relies on both the Grunt setup at the root of the
+Kibana project and code in `kbn-build`. The full process is described in
+`tasks/build/packages.js`.
+
 ## Development
 
 This package is run from Kibana root, using `yarn kbn`. This will run the
 "pre-built" (aka built and committed to git) version of this tool, which is
-located in the `dist/` folder.
+located in the `dist/` folder. This will also use the included version of Yarn
+instead of using your local install of Yarn.
 
 If you need to build a new version of this package, run `yarn build` in this
 folder.
@@ -214,6 +221,10 @@ not necessarily optimized for our use-cases. It's also not ideal for the setup
 we currently have, with one app that "owns everything" and the rest being
 packages for that app.
 
+### Why a local version of Yarn?
+
+See the [vendor readme](./vendor/README.md).
+
 [npm-link]: https://docs.npmjs.com/cli/link
 [npm5-file]: https://github.com/npm/npm/pull/15900
 [yarn-workspaces]: https://yarnpkg.com/lang/en/docs/workspaces/

packages/kbn-build/cli.js

@@ -1 +1 @@
-require('./dist/cli').run(process.argv.slice(2));
+require('./dist').run(process.argv.slice(2));

packages/kbn-build/package.json

@@ -1,5 +1,11 @@
 {
   "name": "@elastic/kbn-build",
+  "kibana": {
+    "build": {
+      "skip": true
+    }
+  },
+  "main": "./dist/index.js",
   "version": "1.0.0",
   "license": "Apache-2.0",
   "private": true,
@@ -13,11 +19,13 @@
     "babel-preset-env": "^1.6.1",
     "babel-preset-stage-3": "^6.24.1",
     "chalk": "^2.3.0",
+    "cpy": "^6.0.0",
     "dedent": "^0.7.0",
     "del": "^3.0.0",
     "execa": "^0.8.0",
     "getopts": "^2.0.0",
     "glob": "^7.1.2",
+    "globby": "^7.1.1",
     "indent-string": "^3.2.0",
     "log-symbols": "^2.1.0",
     "ora": "^1.3.0",
@@ -27,7 +35,9 @@
     "spawn-sync": "^1.0.15",
     "string-replace-loader": "^1.3.0",
     "strong-log-transformer": "^1.0.6",
+    "tempy": "^0.2.1",
     "webpack": "^3.10.0",
-    "wrap-ansi": "^3.0.1"
+    "wrap-ansi": "^3.0.1",
+    "write-pkg": "^3.1.0"
   }
 }

packages/kbn-build/src/cli.js

@@ -6,9 +6,11 @@ import { resolve } from 'path';
 import * as commands from './commands';
 import { runCommand } from './run';
 
+const getCommand = name => commands[name]; // eslint-disable-line import/namespace
+
 function help() {
   const availableCommands = Object.keys(commands)
-    .map(commandName => commands[commandName]) // eslint-disable-line import/namespace
+    .map(commandName => getCommand(commandName))
     .map(command => `${command.name} - ${command.description}`);
 
   console.log(dedent`
@@ -51,7 +53,7 @@ export async function run(argv) {
 
   const commandOptions = { options, extraArgs, rootPath };
 
-  const command = commands[commandName]; // eslint-disable-line import/namespace
+  const command = getCommand(commandName);
   if (command === undefined) {
     console.log(
       chalk.red(`[${commandName}] is not a valid command, see 'kbn --help'`)

packages/kbn-build/src/index.js

@@ -0,0 +1,3 @@
+export { run } from './cli';
+export { buildProductionProjects } from './production';
+export { transformDependencies } from './utils/package_json';

packages/kbn-build/src/paths.js

@@ -0,0 +1,6 @@
+// This is placed directly in `./src` to be consistent with the relative
+// location from `./dist` in the production build
+
+import { join } from 'path';
+
+export const yarnPath = join(__dirname, '../vendor/yarn-1.3.2.js');

packages/kbn-build/src/production/__tests__/fixtures/packages/bar/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@elastic/bar",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./target/index.js",
+  "dependencies": {
+    "lodash": "4.17.4"
+  },
+  "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-preset-env": "^1.6.1"
+  },
+  "scripts": {
+    "build": "babel --presets env --out-dir target src"
+  }
+}

packages/kbn-build/src/production/__tests__/fixtures/packages/bar/src/index.js

@@ -0,0 +1,5 @@
+import _ from 'lodash';
+
+export default function (val) {
+  return `test second: ${_.upperCase(val)}`;
+}

packages/kbn-build/src/production/__tests__/fixtures/packages/foo/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@elastic/foo",
+  "version": "1.0.0",
+  "private": true,
+  "main": "./target/index.js",
+  "dependencies": {
+    "@elastic/bar": "link:../bar",
+    "lodash": "4.17.4"
+  },
+  "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-preset-env": "^1.6.1",
+    "moment": "2.20.1"
+  },
+  "scripts": {
+    "build": "babel --presets env --out-dir target src"
+  }
+}

packages/kbn-build/src/production/__tests__/fixtures/packages/foo/src/index.js

@@ -0,0 +1,5 @@
+import bar from '@elastic/bar';
+
+export default function (val) {
+  return 'test [' + val + '] (' + bar(val) + ')';
+}

packages/kbn-build/src/production/__tests__/index.js

@@ -0,0 +1,58 @@
+import expect from 'expect.js';
+import tempy from 'tempy';
+import copy from 'cpy';
+import { resolve } from 'path';
+import globby from 'globby';
+
+import { buildProductionProjects } from '../';
+import { getProjects } from '../../utils/projects';
+
+// This is specifically a Mocha test instead of a Jest test because it's slow
+// and more integration-y, as we're trying to not add very slow tests to our
+// Jest unit tests.
+
+describe('kbn-build production', function () {
+  it('builds and copies projects for production', async function () {
+    this.timeout(60 * 1000);
+
+    const tmpDir = tempy.directory();
+    const buildRoot = tempy.directory();
+    const fixturesPath = resolve(__dirname, 'fixtures');
+
+    // console.log({ tmpDir, buildRoot, __dirname });
+
+    // Copy all the test fixtures into a tmp dir, as we will be mutating them
+    await copy(['**/*'], tmpDir, {
+      cwd: fixturesPath,
+      parents: true,
+      nodir: true,
+      dot: true
+    });
+
+    const projects = await getProjects(tmpDir, ['./packages/*']);
+
+    for (const project of projects.values()) {
+      // This will both install dependencies and generate `yarn.lock` files
+      await project.installDependencies({
+        extraArgs: ['--silent', '--no-progress']
+      });
+    }
+
+    await buildProductionProjects({ kibanaRoot: tmpDir, buildRoot });
+
+    const files = await globby(['**/*', '!**/node_modules/**'], {
+      cwd: buildRoot
+    });
+
+    expect(files).to.eql([
+      'packages/bar/package.json',
+      'packages/bar/src/index.js',
+      'packages/bar/target/index.js',
+      'packages/bar/yarn.lock',
+      'packages/foo/package.json',
+      'packages/foo/src/index.js',
+      'packages/foo/target/index.js',
+      'packages/foo/yarn.lock'
+    ]);
+  });
+});

packages/kbn-build/src/production/index.js

@@ -0,0 +1,90 @@
+import del from 'del';
+import { relative, resolve } from 'path';
+import copy from 'cpy';
+
+import { getProjectPaths } from '../config';
+import {
+  getProjects,
+  buildProjectGraph,
+  topologicallyBatchProjects
+} from '../utils/projects';
+import {
+  createProductionPackageJson,
+  writePackageJson
+} from '../utils/package_json';
+import { isDirectory } from '../utils/fs';
+
+export async function buildProductionProjects({ kibanaRoot, buildRoot }) {
+  const projectPaths = getProjectPaths(kibanaRoot, {
+    'skip-kibana': true,
+    'skip-kibana-extra': true
+  });
+
+  const projects = await getProductionProjects(kibanaRoot, projectPaths);
+  const projectGraph = buildProjectGraph(projects);
+  const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
+
+  const projectNames = [...projects.values()].map(project => project.name);
+  console.log(`Preparing production build for [${projectNames.join(', ')}]`);
+
+  for (const batch of batchedProjects) {
+    for (const project of batch) {
+      await deleteTarget(project);
+      await buildProject(project);
+      await copyToBuild(project, kibanaRoot, buildRoot);
+    }
+  }
+}
+
+/**
+ * Returns only the projects that should be built into the production bundle
+ */
+async function getProductionProjects(kibanaRoot, projectPaths) {
+  const projects = await getProjects(kibanaRoot, projectPaths);
+
+  const buildProjects = new Map();
+  for (const [name, project] of projects.entries()) {
+    if (!project.skipFromBuild()) {
+      buildProjects.set(name, project);
+    }
+  }
+
+  return buildProjects;
+}
+
+async function deleteTarget(project) {
+  const targetDir = project.targetLocation;
+
+  if (await isDirectory(targetDir)) {
+    await del(targetDir, { force: true });
+  }
+}
+
+async function buildProject(project) {
+  if (project.hasScript('build')) {
+    await project.runScript('build');
+  }
+}
+
+async function copyToBuild(project, kibanaRoot, buildRoot) {
+  // We want the package to have the same relative location within the build
+  const relativeProjectPath = relative(kibanaRoot, project.path);
+  const buildProjectPath = resolve(buildRoot, relativeProjectPath);
+
+  // When copying all the files into the build, we exclude `package.json` as we
+  // write a separate "production-ready" `package.json` below, and we exclude
+  // `node_modules` because we want the Kibana build to actually install all
+  // dependencies. The primary reason for allowing the Kibana build process to
+  // install the dependencies is that it will "dedupe" them, so we don't include
+  // unnecessary copies of dependencies.
+  await copy(['**/*', '!package.json', '!node_modules/**'], buildProjectPath, {
+    cwd: project.path,
+    parents: true,
+    nodir: true,
+    dot: true
+  });
+
+  const packageJson = project.json;
+  const preparedPackageJson = createProductionPackageJson(packageJson);
+  await writePackageJson(buildProjectPath, preparedPackageJson);
+}

packages/kbn-build/src/utils/package_json.js

@@ -1,6 +1,38 @@
 import readPkg from 'read-pkg';
+import writePkg from 'write-pkg';
 import path from 'path';
 
 export function readPackageJson(dir) {
   return readPkg(path.join(dir, 'package.json'), { normalize: false });
 }
+
+export function writePackageJson(path, json) {
+  return writePkg(path, json);
+}
+
+export const createProductionPackageJson = pkgJson => ({
+  ...pkgJson,
+  dependencies: transformDependencies(pkgJson.dependencies)
+});
+
+/**
+ * Replaces `link:` dependencies with `file:` dependencies. When installing
+ * dependencies, these `file:` dependencies will be copied into `node_modules`
+ * instead of being symlinked.
+ *
+ * This will allow us to copy packages into the build and run `yarn`, which
+ * will then _copy_ the `file:` dependencies into `node_modules` instead of
+ * symlinking like we do in development.
+ */
+export function transformDependencies(dependencies = {}) {
+  const newDeps = {};
+  for (const name of Object.keys(dependencies)) {
+    const depVersion = dependencies[name];
+    if (depVersion.startsWith('link:')) {
+      newDeps[name] = depVersion.replace('link:', 'file:');
+    } else {
+      newDeps[name] = depVersion;
+    }
+  }
+  return newDeps;
+}

packages/kbn-build/src/utils/project.js

@@ -1,7 +1,11 @@
 import path from 'path';
 import chalk from 'chalk';
 
-import { installInDir, runScriptInPackageStreaming } from './scripts';
+import {
+  installInDir,
+  runScriptInPackage,
+  runScriptInPackageStreaming
+} from './scripts';
 import { readPackageJson } from './package_json';
 import { CliError } from './errors';
 
@@ -14,7 +18,7 @@ export class Project {
   }
 
   constructor(packageJson, projectPath) {
-    this._json = packageJson;
+    this.json = Object.freeze(packageJson);
     this.path = projectPath;
 
     this.packageJsonLocation = path.resolve(this.path, 'package.json');
@@ -22,19 +26,21 @@ export class Project {
     this.targetLocation = path.resolve(this.path, 'target');
 
     this.allDependencies = {
-      ...(this._json.devDependencies || {}),
-      ...(this._json.dependencies || {})
+      ...(this.json.devDependencies || {}),
+      ...(this.json.dependencies || {})
     };
 
-    this.scripts = this._json.scripts || {};
+    this.scripts = this.json.scripts || {};
   }
 
   get name() {
-    return this._json.name;
+    return this.json.name;
   }
 
   ensureValidProjectDependency(project) {
-    const relativePathToProject = normalizePath(path.relative(this.path, project.path));
+    const relativePathToProject = normalizePath(
+      path.relative(this.path, project.path)
+    );
 
     const versionInPackageJson = this.allDependencies[project.name];
     const expectedVersionInPackageJson = `${PREFIX}${relativePathToProject}`;
@@ -67,11 +73,27 @@ export class Project {
     );
   }
 
+  skipFromBuild() {
+    const json = this.json;
+    return json.kibana && json.kibana.build && json.kibana.build.skip === true;
+  }
+
   hasScript(name) {
     return name in this.scripts;
   }
 
-  runScriptStreaming(scriptName, args = []) {
+  async runScript(scriptName, args = []) {
+    console.log(
+      chalk.bold(
+        `\n\nRunning script [${chalk.green(scriptName)}] in [${chalk.green(
+          this.name
+        )}]:\n`
+      )
+    );
+    return runScriptInPackage(scriptName, args, this);
+  }
+
+  async runScriptStreaming(scriptName, args = []) {
     return runScriptInPackageStreaming(scriptName, args, this);
   }
 
@@ -79,7 +101,7 @@ export class Project {
     return Object.keys(this.allDependencies).length > 0;
   }
 
-  installDependencies({ extraArgs }) {
+  async installDependencies({ extraArgs }) {
     console.log(
       chalk.bold(
         `\n\nInstalling dependencies in [${chalk.green(this.name)}]:\n`

packages/kbn-build/src/utils/scripts.js

@@ -1,4 +1,5 @@
 import { spawn, spawnStreaming } from './child_process';
+import { yarnPath } from '../paths';
 
 /**
  * Install all dependencies in the given directory
@@ -13,11 +14,22 @@ export function installInDir(directory, extraArgs = []) {
 
   // We pass the mutex flag to ensure only one instance of yarn runs at any
   // given time (e.g. to avoid conflicts).
-  return spawn('yarn', options, {
+  return spawn(yarnPath, options, {
     cwd: directory
   });
 }
 
+/**
+ * Run script in the given directory
+ */
+export function runScriptInPackage(script, args, pkg) {
+  const execOpts = {
+    cwd: pkg.path
+  };
+
+  return spawn(yarnPath, ['run', script, ...args], execOpts);
+}
+
 /**
  * Run script in the given directory
  */
@@ -26,7 +38,7 @@ export function runScriptInPackageStreaming(script, args, pkg) {
     cwd: pkg.path
   };
 
-  return spawnStreaming('yarn', ['run', script, ...args], execOpts, {
+  return spawnStreaming(yarnPath, ['run', script, ...args], execOpts, {
     prefix: pkg.name
   });
 }

packages/kbn-build/webpack.config.js

@@ -2,7 +2,7 @@ const path = require('path');
 
 module.exports = {
   entry: {
-    cli: './src/cli.js'
+    index: './src/index.js'
   },
   target: 'node',
 

packages/kbn-build/yarn.lock

@@ -108,6 +108,10 @@ array-unique@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.2.1.tgz#a1d97ccafcbc2625cc70fadceb36a50c58b01a53"
 
+arrify@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
+
 asn1.js@^4.0.0:
   version "4.9.2"
   resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-4.9.2.tgz#8117ef4f7ed87cd8f89044b5bff97ac243a16c9a"
@@ -950,6 +954,25 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
 
+cp-file@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/cp-file/-/cp-file-5.0.0.tgz#bc700fd30ca32d24d46c7fb02b992e435fc5a978"
+  dependencies:
+    graceful-fs "^4.1.2"
+    make-dir "^1.0.0"
+    nested-error-stacks "^2.0.0"
+    pify "^3.0.0"
+    safe-buffer "^5.0.1"
+
+cpy@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/cpy/-/cpy-6.0.0.tgz#0b6888e037bb5a7b02a62249551316208a523253"
+  dependencies:
+    arrify "^1.0.1"
+    cp-file "^5.0.0"
+    globby "^6.0.0"
+    nested-error-stacks "^2.0.0"
+
 create-ecdh@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.0.tgz#888c723596cdf7612f6498233eebd7a35301737d"
@@ -1007,6 +1030,10 @@ crypto-browserify@^3.11.0:
     randombytes "^2.0.0"
     randomfill "^1.0.3"
 
+crypto-random-string@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e"
+
 d@1:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f"
@@ -1073,6 +1100,10 @@ detect-indent@^4.0.0:
   dependencies:
     repeating "^2.0.0"
 
+detect-indent@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d"
+
 detect-libc@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
@@ -1085,6 +1116,13 @@ diffie-hellman@^5.0.0:
     miller-rabin "^4.0.0"
     randombytes "^2.0.0"
 
+dir-glob@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-2.0.0.tgz#0b205d2b6aef98238ca286598a8204d29d0a0034"
+  dependencies:
+    arrify "^1.0.1"
+    path-type "^3.0.0"
+
 domain-browser@^1.1.1:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc"
@@ -1437,7 +1475,7 @@ globals@^9.18.0:
   version "9.18.0"
   resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a"
 
-globby@^6.1.0:
+globby@^6.0.0, globby@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/globby/-/globby-6.1.0.tgz#f5a6d70e8395e21c858fb0489d64df02424d506c"
   dependencies:
@@ -1447,7 +1485,18 @@ globby@^6.1.0:
     pify "^2.0.0"
     pinkie-promise "^2.0.0"
 
-graceful-fs@^4.1.2:
+globby@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-7.1.1.tgz#fb2ccff9401f8600945dfada97440cca972b8680"
+  dependencies:
+    array-union "^1.0.1"
+    dir-glob "^2.0.0"
+    glob "^7.1.2"
+    ignore "^3.3.5"
+    pify "^3.0.0"
+    slash "^1.0.0"
+
+graceful-fs@^4.1.11, graceful-fs@^4.1.2:
   version "4.1.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
 
@@ -1544,6 +1593,14 @@ ieee754@^1.1.4:
   version "1.1.8"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4"
 
+ignore@^3.3.5:
+  version "3.3.7"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.7.tgz#612289bfb3c220e186a58118618d5be8c1bab021"
+
+imurmurhash@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
+
 indent-string@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-3.2.0.tgz#4a5fd6d27cc332f37e5419a504dbb837105c9289"
@@ -1673,6 +1730,10 @@ is-path-inside@^1.0.0:
   dependencies:
     path-is-inside "^1.0.1"
 
+is-plain-obj@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
+
 is-posix-bracket@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz#3334dc79774368e92f016e6fbc0a88f5cd6e6bc4"
@@ -1967,6 +2028,12 @@ nan@^2.3.0:
   version "2.8.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a"
 
+nested-error-stacks@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/nested-error-stacks/-/nested-error-stacks-2.0.0.tgz#98b2ffaefb4610fa3936f1e71435d30700de2840"
+  dependencies:
+    inherits "~2.0.1"
+
 node-libs-browser@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.1.0.tgz#5f94263d404f6e44767d726901fff05478d600df"
@@ -2550,6 +2617,12 @@ sntp@1.x.x:
   dependencies:
     hoek "2.x.x"
 
+sort-keys@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128"
+  dependencies:
+    is-plain-obj "^1.0.0"
+
 source-list-map@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.0.tgz#aaa47403f7b245a92fbc97ea08f250d6087ed085"
@@ -2721,6 +2794,17 @@ tar@^2.2.1:
     fstream "^1.0.2"
     inherits "2"
 
+temp-dir@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d"
+
+tempy@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/tempy/-/tempy-0.2.1.tgz#9038e4dbd1c201b74472214179bc2c6f7776e54c"
+  dependencies:
+    temp-dir "^1.0.0"
+    unique-string "^1.0.0"
+
 through@^2.3.4:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -2792,6 +2876,12 @@ uid-number@^0.0.6:
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81"
 
+unique-string@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-1.0.0.tgz#9e1057cca851abb93398f8b33ae187b99caec11a"
+  dependencies:
+    crypto-random-string "^1.0.0"
+
 url@^0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
@@ -2918,6 +3008,32 @@ wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
 
+write-file-atomic@^2.0.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-2.3.0.tgz#1ff61575c2e2a4e8e510d6fa4e243cce183999ab"
+  dependencies:
+    graceful-fs "^4.1.11"
+    imurmurhash "^0.1.4"
+    signal-exit "^3.0.2"
+
+write-json-file@^2.2.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-2.3.0.tgz#2b64c8a33004d54b8698c76d585a77ceb61da32f"
+  dependencies:
+    detect-indent "^5.0.0"
+    graceful-fs "^4.1.2"
+    make-dir "^1.0.0"
+    pify "^3.0.0"
+    sort-keys "^2.0.0"
+    write-file-atomic "^2.0.0"
+
+write-pkg@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/write-pkg/-/write-pkg-3.1.0.tgz#030a9994cc9993d25b4e75a9f1a1923607291ce9"
+  dependencies:
+    sort-keys "^2.0.0"
+    write-json-file "^2.2.0"
+
 xtend@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"

src/babel-register/register.js

@@ -9,6 +9,7 @@ if (!process.env.BABEL_CACHE_PATH) {
 // paths that babel-register should ignore
 const ignore = [
   /[\\\/](node_modules|bower_components)[\\\/]/,
+  /[\\\/](kbn-build\/dist)[\\\/]/
 ];
 
 if (global.__BUILT_WITH_BABEL__) {

src/dev/jest/config.json

@@ -35,14 +35,16 @@
     "js",
     "json"
   ],
+  "modulePathIgnorePatterns": [
+    "__fixtures__/"
+  ],
   "testMatch": [
     "**/*.test.js"
   ],
   "testPathIgnorePatterns": [
     "<rootDir>/ui_framework/dist/",
     "<rootDir>/ui_framework/doc_site/",
-    "<rootDir>/ui_framework/generator-kui/",
-    ".*/__fixtures__/.*"
+    "<rootDir>/ui_framework/generator-kui/"
   ],
   "transform": {
     "^.+\\.js$": "<rootDir>/src/dev/jest/babel_transform.js"

tasks/bootstrap_kibana.js

@@ -0,0 +1,43 @@
+module.exports = grunt => {
+  grunt.registerTask(
+    'bootstrapKibana',
+    'Bootstrap Kibana and all Kibana packages',
+    async function () {
+      const done = this.async();
+
+      try {
+        await bootstrapKibana();
+        done();
+      } catch (e) {
+        grunt.fail.fatal(e);
+        done(e);
+      }
+    }
+  );
+
+  function bootstrapKibana() {
+    const serverCmd = {
+      cmd: 'yarn',
+      args: [
+        'kbn',
+        'bootstrap',
+        '--skip-kibana-extra'
+      ],
+      opts: {
+        stdio: 'inherit'
+      }
+    };
+
+    return new Promise((resolve, reject) => {
+      grunt.util.spawn(serverCmd, (error, result, code) => {
+        if (error || code !== 0) {
+          const error = new Error(`'yarn kbn bootstrap' exited with code ${code}`);
+          reject(error);
+          return;
+        }
+
+        resolve();
+      });
+    });
+  }
+};

tasks/build/index.js

@@ -2,6 +2,9 @@ import { flatten } from 'lodash';
 module.exports = function (grunt) {
   grunt.registerTask('build', 'Build packages', function () {
     grunt.task.run(flatten([
+      // We specifically bootstrap Kibana to make sure all dependencies are
+      // up-to-date before kicking of the rest of the build process
+      'bootstrapKibana',
       'clean:build',
       'clean:target',
       '_build:downloadNodeBuilds',
@@ -15,7 +18,9 @@ module.exports = function (grunt) {
       '_build:packageJson',
       '_build:readme',
       '_build:babelCache',
+      '_build:packages',
       '_build:installDependencies',
+      'clean:packages',
       '_build:notice',
       '_build:removePkgJsonDeps',
       'clean:testsFromModules',

tasks/build/install_dependencies.js

@@ -4,9 +4,13 @@ module.exports = function (grunt) {
     // We rely on a local version of Yarn that contains the bugfix from
     // https://github.com/yarnpkg/yarn/pull/5059. Once this fix is merged
     // and released we can use Yarn directly in the build.
-    const yarn = require.resolve('../vendor/yarn-1.3.2-with-ignore-fix.js');
+    const yarn = require.resolve('../../packages/kbn-build/vendor/yarn-1.3.2.js');
 
-    exec(`${yarn} --production --ignore-optional --frozen-lockfile`, {
+    // We're using `pure-lockfile` instead of `frozen-lockfile` because we
+    // rewrite `link:` dependencies to `file:` dependencies earlier in the
+    // build. This means the lockfile won't be consistent, so instead of
+    // verifying it, we just skip writing a new lockfile.
+    exec(`${yarn} --production --ignore-optional --pure-lockfile`, {
       cwd: grunt.config.process('<%= root %>/build/kibana')
     }, this.async());
   });

tasks/build/package_json.js

@@ -1,3 +1,5 @@
+import { transformDependencies } from '@elastic/kbn-build';
+
 module.exports = function (grunt) {
   const pkg = grunt.config.get('pkg');
 
@@ -20,7 +22,7 @@ module.exports = function (grunt) {
         engines: {
           node: pkg.engines.node
         },
-        dependencies: pkg.dependencies
+        dependencies: transformDependencies(pkg.dependencies)
       }, null, '  ')
     );
   });

tasks/build/packages.js

@@ -0,0 +1,56 @@
+import { buildProductionProjects } from '@elastic/kbn-build';
+
+/**
+ * High-level overview of how we enable shared packages in production:
+ *
+ * tl;dr We copy the packages directly into Kibana's `node_modules` folder,
+ * which means they will be available when `require(...)`d.
+ *
+ * During development we rely on `@elastic/kbn-build` to find all the packages
+ * in the Kibana repo and run Yarn in all the right places to create symlinks
+ * between these packages. This development setup is described in-depth in the
+ * readme in `@elastic/kbn-build`.
+ *
+ * However, for production we can't use `@elastic/kbn-build` as part of the
+ * installation as we don't have an install "tool/step" that can kick it off.
+ * We also can't include symlinks in the archives for the different platform, so
+ * we can't run `@elastic/kbn-build` in the same way we do for development and
+ * just package the result. That means we have two options: either we prepare
+ * everything in the built package or we perform the necessary actions when
+ * Kibana is starting up in production. We decided on the former: all the Kibana
+ * packages are prepared as part of the build process.
+ *
+ * (All of this is a bit different for Kibana plugins as they _do_ have an
+ * install step — the plugin CLI tool. However, Kibana plugins are not allowed
+ * to have separate packages yet.)
+ *
+ * How Kibana packages are prepared:
+ *
+ * 1. Run the build for each package
+ * 2. Copy all the packages into the `build/kibana` folder
+ * 3. Replace `link:` dependencies with `file:` dependencies in both Kibana's
+ *    `package.json` and in all the dependencies. Yarn will then copy the
+ *    sources of these dependencies into `node_modules` instead of setting up
+ *    symlinks.
+ *
+ * In the end after the `install dependencies` build step all Kibana packages
+ * will be located within the top-level `node_modules` folder, which means
+ * normal module resolution will apply and you can `require(...)` any of these
+ * packages when running Kibana in production.
+ */
+module.exports = function (grunt) {
+  grunt.registerTask('_build:packages', async function () {
+    const done = this.async();
+
+    const kibanaRoot = grunt.config.get('root');
+    const buildRoot = `${kibanaRoot}/build/kibana`;
+
+    try {
+      await buildProductionProjects({ kibanaRoot, buildRoot });
+      done();
+    } catch (err) {
+      grunt.fail.fatal(err);
+      done(err);
+    }
+  });
+};

tasks/build_packages.js

@@ -0,0 +1,45 @@
+module.exports = grunt => {
+  grunt.registerTask(
+    'buildPackages',
+    'Build all the Kibana specific packages',
+    async function () {
+      const done = this.async();
+
+      try {
+        await buildPackages();
+        done();
+      } catch (e) {
+        grunt.fail.fatal(e);
+        done(e);
+      }
+    }
+  );
+
+  function buildPackages() {
+    const serverCmd = {
+      cmd: 'yarn',
+      args: [
+        'kbn',
+        'run',
+        'build',
+        '--skip-kibana',
+        '--skip-kibana-extra'
+      ],
+      opts: {
+        stdio: 'inherit'
+      }
+    };
+
+    return new Promise((resolve, reject) => {
+      grunt.util.spawn(serverCmd, (error, result, code) => {
+        if (error || code !== 0) {
+          const error = new Error(`'yarn kbn run' exited with code ${code}`);
+          reject(error);
+          return;
+        }
+
+        resolve();
+      });
+    });
+  }
+};

tasks/config/clean.js

@@ -5,6 +5,7 @@ module.exports = function () {
     testsFromModules: 'build/kibana/node_modules/**/{test,tests}/**',
     examplesFromModules: 'build/kibana/node_modules/**/{example,examples}/**',
     devSourceForTestbed: 'build/kibana/src/core_plugins/testbed/',
-    nodeForOptimize: 'build/kibana/node'
+    nodeForOptimize: 'build/kibana/node',
+    packages: 'build/kibana/packages'
   };
 };

tasks/config/simplemocha.js

@@ -14,6 +14,7 @@ export default {
     src: [
       'test/**/__tests__/**/*.js',
       'src/**/__tests__/**/*.js',
+      'packages/kbn-build/**/__tests__/**/*.js',
       'tasks/**/__tests__/**/*.js',
       'test/fixtures/__tests__/*.js',
       '!**/__tests__/fixtures/**/*',

tasks/jenkins.js

@@ -29,6 +29,12 @@ module.exports = function (grunt) {
     'jenkins:env',
     'rejectRejFiles',
 
+    // We need to build all the Kibana packages so ESLint is able to verify that
+    // e.g. imports resolve properly. If we don't build the packages here, the
+    // `main` field in their `package.json` would link to a location that
+    // doesn't exist yet.
+    'buildPackages',
+
     'run:eslint',
     'licenses',
     'test:server',

yarn.lock

@@ -110,6 +110,10 @@
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@elastic/filesaver/-/filesaver-1.1.2.tgz#1998ffb3cd89c9da4ec12a7793bfcae10e30c77a"
 
+"@elastic/kbn-build@link:packages/kbn-build":
+  version "0.0.0"
+  uid ""
+
 "@elastic/numeral@2.3.1":
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/@elastic/numeral/-/numeral-2.3.1.tgz#743801971d8f0c975f9a122867d0e8939d31b3eb"