Run kbn:bootstrap script after installing deps (#16585) (#16590)

* Run 'kbn:bootstrap' scripts at the end of bootstrapping

* Add bootstrap command tests

* resetAllMocks

* Use 'absolutePathSnaphotSerializer'

* Mock console.log calls

* Strip ansi snapshot serializer

* reset in afterEach

* Log before running 'kbn:bootstrap' scripts

* mock linkProjectExecutables in bootstrap tests

packages/kbn-build/README.md

@@ -117,6 +117,9 @@ For more details, run:
 yarn kbn
 ```
 
+Bootstrapping also calls the `kbn:boostrap` script for every included project.
+This is intended for packages that need to be built/transpiled to be usable.
+
 ### Running scripts
 
 Some times you want to run the same script across multiple packages and plugins,

packages/kbn-build/dist/index.js

@@ -23339,7 +23339,7 @@ let run = exports.run = (() => {
     const frozenLockfile = options['frozen-lockfile'] === true;
     const extraArgs = frozenLockfile ? ['--frozen-lockfile'] : [];
 
-    console.log(_chalk2.default.bold('\nRunning installs in topological order'));
+    console.log(_chalk2.default.bold('\nRunning installs in topological order:'));
 
     for (const batch of batchedProjects) {
       for (const project of batch) {
@@ -23351,6 +23351,21 @@ let run = exports.run = (() => {
 
     console.log(_chalk2.default.bold('\nInstalls completed, linking package executables:\n'));
     yield (0, _link_project_executables.linkProjectExecutables)(projects, projectGraph);
+
+    /**
+     * At the end of the bootstrapping process we call all `kbn:bootstrap` scripts
+     * in the list of projects. We do this because some projects need to be
+     * transpiled before they can be used. Ideally we shouldn't do this unless we
+     * have to, as it will slow down the bootstrapping process.
+     */
+    console.log(_chalk2.default.bold('\nLinking executables completed, running `kbn:bootstrap` scripts\n'));
+    yield parallelizeBatches(batchedProjects, function (pkg) {
+      if (pkg.hasScript('kbn:bootstrap')) {
+        return pkg.runScriptStreaming('kbn:bootstrap');
+      }
+    });
+
+    console.log(_chalk2.default.green.bold('\nBootstrapping completed!\n'));
   });
 
   return function run(_x, _x2, _x3) {
@@ -23358,6 +23373,24 @@ let run = exports.run = (() => {
   };
 })();
 
+let parallelizeBatches = (() => {
+  var _ref2 = _asyncToGenerator(function* (batchedProjects, fn) {
+    for (const batch of batchedProjects) {
+      const running = batch.map(function (pkg) {
+        return fn(pkg);
+      });
+
+      // We need to make sure the entire batch has completed before we can move on
+      // to the next batch
+      yield Promise.all(running);
+    }
+  });
+
+  return function parallelizeBatches(_x4, _x5) {
+    return _ref2.apply(this, arguments);
+  };
+})();
+
 var _chalk = __webpack_require__(4);
 
 var _chalk2 = _interopRequireDefault(_chalk);

packages/kbn-build/package.json

@@ -27,6 +27,7 @@
     "getopts": "^2.0.0",
     "glob": "^7.1.2",
     "globby": "^7.1.1",
+    "has-ansi": "^3.0.0",
     "indent-string": "^3.2.0",
     "lodash.clonedeepwith": "^4.5.0",
     "log-symbols": "^2.1.0",
@@ -37,6 +38,7 @@
     "read-pkg": "^3.0.0",
     "spawn-sync": "^1.0.15",
     "string-replace-loader": "^1.3.0",
+    "strip-ansi": "^4.0.0",
     "strong-log-transformer": "^1.0.6",
     "tempy": "^0.2.1",
     "webpack": "^3.10.0",

packages/kbn-build/src/commands/__snapshots__/bootstrap.test.js.snap

@@ -0,0 +1,201 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`calls "kbn:bootstrap" scripts and links executables after installing deps: link bins 1`] = `
+Array [
+  Array [
+    Map {
+      "kibana" => Project {
+        "allDependencies": Object {
+          "bar": "link:packages/bar",
+        },
+        "json": Object {
+          "dependencies": Object {
+            "bar": "link:packages/bar",
+          },
+          "name": "kibana",
+          "version": "1.0.0",
+        },
+        "nodeModulesLocation": "<repoRoot>/packages/kbn-build/src/commands/node_modules",
+        "packageJsonLocation": "<repoRoot>/packages/kbn-build/src/commands/package.json",
+        "path": "<repoRoot>/packages/kbn-build/src/commands",
+        "scripts": Object {},
+        "targetLocation": "<repoRoot>/packages/kbn-build/src/commands/target",
+      },
+      "bar" => Project {
+        "allDependencies": Object {},
+        "json": Object {
+          "name": "bar",
+          "scripts": Object {
+            "kbn:bootstrap": "node ./bar.js",
+          },
+          "version": "1.0.0",
+        },
+        "nodeModulesLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/node_modules",
+        "packageJsonLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/package.json",
+        "path": "<repoRoot>/packages/kbn-build/src/commands/packages/bar",
+        "scripts": Object {
+          "kbn:bootstrap": "node ./bar.js",
+        },
+        "targetLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/target",
+      },
+    },
+    Map {
+      "kibana" => Array [
+        Project {
+          "allDependencies": Object {},
+          "json": Object {
+            "name": "bar",
+            "scripts": Object {
+              "kbn:bootstrap": "node ./bar.js",
+            },
+            "version": "1.0.0",
+          },
+          "nodeModulesLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/node_modules",
+          "packageJsonLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/package.json",
+          "path": "<repoRoot>/packages/kbn-build/src/commands/packages/bar",
+          "scripts": Object {
+            "kbn:bootstrap": "node ./bar.js",
+          },
+          "targetLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/target",
+        },
+      ],
+      "bar" => Array [],
+    },
+  ],
+]
+`;
+
+exports[`calls "kbn:bootstrap" scripts and links executables after installing deps: script 1`] = `
+Array [
+  Array [
+    "kbn:bootstrap",
+    Array [],
+    Project {
+      "allDependencies": Object {},
+      "json": Object {
+        "name": "bar",
+        "scripts": Object {
+          "kbn:bootstrap": "node ./bar.js",
+        },
+        "version": "1.0.0",
+      },
+      "nodeModulesLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/node_modules",
+      "packageJsonLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/package.json",
+      "path": "<repoRoot>/packages/kbn-build/src/commands/packages/bar",
+      "scripts": Object {
+        "kbn:bootstrap": "node ./bar.js",
+      },
+      "targetLocation": "<repoRoot>/packages/kbn-build/src/commands/packages/bar/target",
+    },
+  ],
+]
+`;
+
+exports[`does not run installer if no deps in package: install in dir 1`] = `
+Array [
+  Array [
+    "<repoRoot>/packages/kbn-build/src/commands",
+    Array [],
+  ],
+]
+`;
+
+exports[`does not run installer if no deps in package: logs 1`] = `
+Array [
+  Array [
+    "
+Running installs in topological order:",
+  ],
+  Array [
+    "
+
+Installing dependencies in [kibana]:
+",
+  ],
+  Array [
+    "
+Installs completed, linking package executables:
+",
+  ],
+  Array [
+    "
+Linking executables completed, running \`kbn:bootstrap\` scripts
+",
+  ],
+  Array [
+    "
+Bootstrapping completed!
+",
+  ],
+]
+`;
+
+exports[`handles "frozen-lockfile": install in dir 1`] = `
+Array [
+  Array [
+    "<repoRoot>/packages/kbn-build/src/commands",
+    Array [
+      "--frozen-lockfile",
+    ],
+  ],
+]
+`;
+
+exports[`handles dependencies of dependencies: install in dir 1`] = `
+Array [
+  Array [
+    "<repoRoot>/packages/kbn-build/src/commands/packages/bar",
+    Array [],
+  ],
+  Array [
+    "<repoRoot>/packages/kbn-build/src/commands",
+    Array [],
+  ],
+  Array [
+    "<repoRoot>/packages/kbn-build/src/commands/packages/foo",
+    Array [],
+  ],
+]
+`;
+
+exports[`handles dependencies of dependencies: logs 1`] = `
+Array [
+  Array [
+    "
+Running installs in topological order:",
+  ],
+  Array [
+    "
+
+Installing dependencies in [bar]:
+",
+  ],
+  Array [
+    "
+
+Installing dependencies in [kibana]:
+",
+  ],
+  Array [
+    "
+
+Installing dependencies in [foo]:
+",
+  ],
+  Array [
+    "
+Installs completed, linking package executables:
+",
+  ],
+  Array [
+    "
+Linking executables completed, running \`kbn:bootstrap\` scripts
+",
+  ],
+  Array [
+    "
+Bootstrapping completed!
+",
+  ],
+]
+`;

packages/kbn-build/src/commands/bootstrap.js

@@ -12,7 +12,7 @@ export async function run(projects, projectGraph, { options }) {
   const frozenLockfile = options['frozen-lockfile'] === true;
   const extraArgs = frozenLockfile ? ['--frozen-lockfile'] : [];
 
-  console.log(chalk.bold('\nRunning installs in topological order'));
+  console.log(chalk.bold('\nRunning installs in topological order:'));
 
   for (const batch of batchedProjects) {
     for (const project of batch) {
@@ -26,4 +26,33 @@ export async function run(projects, projectGraph, { options }) {
     chalk.bold('\nInstalls completed, linking package executables:\n')
   );
   await linkProjectExecutables(projects, projectGraph);
+
+  /**
+   * At the end of the bootstrapping process we call all `kbn:bootstrap` scripts
+   * in the list of projects. We do this because some projects need to be
+   * transpiled before they can be used. Ideally we shouldn't do this unless we
+   * have to, as it will slow down the bootstrapping process.
+   */
+  console.log(
+    chalk.bold(
+      '\nLinking executables completed, running `kbn:bootstrap` scripts\n'
+    )
+  );
+  await parallelizeBatches(batchedProjects, pkg => {
+    if (pkg.hasScript('kbn:bootstrap')) {
+      return pkg.runScriptStreaming('kbn:bootstrap');
+    }
+  });
+
+  console.log(chalk.green.bold('\nBootstrapping completed!\n'));
+}
+
+async function parallelizeBatches(batchedProjects, fn) {
+  for (const batch of batchedProjects) {
+    const running = batch.map(pkg => fn(pkg));
+
+    // We need to make sure the entire batch has completed before we can move on
+    // to the next batch
+    await Promise.all(running);
+  }
 }

packages/kbn-build/src/commands/bootstrap.test.js

@@ -0,0 +1,169 @@
+jest.mock('../utils/scripts', () => ({
+  installInDir: jest.fn(),
+  runScriptInPackageStreaming: jest.fn(),
+}));
+jest.mock('../utils/link_project_executables', () => ({
+  linkProjectExecutables: jest.fn(),
+}));
+
+import { resolve } from 'path';
+
+import {
+  absolutePathSnaphotSerializer,
+  stripAnsiSnapshotSerializer,
+} from '../test_helpers';
+import { run } from './bootstrap';
+import { Project } from '../utils/project';
+import { buildProjectGraph } from '../utils/projects';
+import { installInDir, runScriptInPackageStreaming } from '../utils/scripts';
+import { linkProjectExecutables } from '../utils/link_project_executables';
+
+const createProject = (fields, path = '.') =>
+  new Project(
+    {
+      name: 'kibana',
+      version: '1.0.0',
+      ...fields,
+    },
+    resolve(__dirname, path)
+  );
+
+expect.addSnapshotSerializer(absolutePathSnaphotSerializer);
+expect.addSnapshotSerializer(stripAnsiSnapshotSerializer);
+
+afterEach(() => {
+  jest.resetAllMocks();
+});
+
+test('handles dependencies of dependencies', async () => {
+  const kibana = createProject({
+    dependencies: {
+      bar: 'link:packages/bar',
+    },
+  });
+  const foo = createProject(
+    {
+      name: 'foo',
+      dependencies: {
+        bar: 'link:../bar',
+      },
+    },
+    'packages/foo'
+  );
+  const bar = createProject(
+    {
+      name: 'bar',
+      dependencies: {
+        baz: 'link:../baz',
+      },
+    },
+    'packages/bar'
+  );
+  const baz = createProject(
+    {
+      name: 'baz',
+    },
+    'packages/baz'
+  );
+  const projects = new Map([
+    ['kibana', kibana],
+    ['foo', foo],
+    ['bar', bar],
+    ['baz', baz],
+  ]);
+  const projectGraph = buildProjectGraph(projects);
+
+  const spy = jest.spyOn(console, 'log').mockImplementation(() => {});
+
+  await run(projects, projectGraph, {
+    options: {},
+  });
+
+  expect(installInDir.mock.calls).toMatchSnapshot('install in dir');
+  expect(spy.mock.calls).toMatchSnapshot('logs');
+
+  spy.mockRestore();
+});
+
+test('does not run installer if no deps in package', async () => {
+  const kibana = createProject({
+    dependencies: {
+      bar: 'link:packages/bar',
+    },
+  });
+  // bar has no dependencies
+  const bar = createProject(
+    {
+      name: 'bar',
+    },
+    'packages/bar'
+  );
+
+  const projects = new Map([['kibana', kibana], ['bar', bar]]);
+  const projectGraph = buildProjectGraph(projects);
+
+  const spy = jest.spyOn(console, 'log').mockImplementation(() => {});
+
+  await run(projects, projectGraph, {
+    options: {},
+  });
+
+  expect(installInDir.mock.calls).toMatchSnapshot('install in dir');
+  expect(spy.mock.calls).toMatchSnapshot('logs');
+
+  spy.mockRestore();
+});
+
+test('handles "frozen-lockfile"', async () => {
+  const kibana = createProject({
+    dependencies: {
+      foo: '2.2.0',
+    },
+  });
+
+  const projects = new Map([['kibana', kibana]]);
+  const projectGraph = buildProjectGraph(projects);
+
+  const spy = jest.spyOn(console, 'log').mockImplementation(() => {});
+
+  await run(projects, projectGraph, {
+    options: {
+      'frozen-lockfile': true,
+    },
+  });
+
+  expect(installInDir.mock.calls).toMatchSnapshot('install in dir');
+
+  spy.mockRestore();
+});
+
+test('calls "kbn:bootstrap" scripts and links executables after installing deps', async () => {
+  const kibana = createProject({
+    dependencies: {
+      bar: 'link:packages/bar',
+    },
+  });
+  const bar = createProject(
+    {
+      name: 'bar',
+      scripts: {
+        'kbn:bootstrap': 'node ./bar.js',
+      },
+    },
+    'packages/bar'
+  );
+
+  const projects = new Map([['kibana', kibana], ['bar', bar]]);
+  const projectGraph = buildProjectGraph(projects);
+
+  const spy = jest.spyOn(console, 'log').mockImplementation(() => {});
+
+  await run(projects, projectGraph, {
+    options: {},
+  });
+
+  expect(linkProjectExecutables.mock.calls).toMatchSnapshot('link bins');
+  expect(runScriptInPackageStreaming.mock.calls).toMatchSnapshot('script');
+
+  spy.mockRestore();
+});

packages/kbn-build/src/test_helpers/index.js

@@ -1,3 +1,5 @@
 export {
   absolutePathSnaphotSerializer,
 } from './absolute_path_snaphot_serializer';
+
+export { stripAnsiSnapshotSerializer } from './strip_ansi_snapshot_serializer';

packages/kbn-build/src/test_helpers/strip_ansi_snapshot_serializer.js

@@ -0,0 +1,12 @@
+import hasAnsi from 'has-ansi';
+import stripAnsi from 'strip-ansi';
+
+export const stripAnsiSnapshotSerializer = {
+  print: (value, serialize) => {
+    return serialize(stripAnsi(value));
+  },
+
+  test: value => {
+    return typeof value === 'string' && hasAnsi(value);
+  },
+};

packages/kbn-build/src/utils/__snapshots__/link_project_executables.test.js.snap

@@ -42,3 +42,11 @@ Object {
   ],
 }
 `;
+
+exports[`bin script points to a file creates a symlink in the project node_modules/.bin directory: logs 1`] = `
+Array [
+  Array [
+    "[foo] bar -> ../bar/bin/bar.js",
+  ],
+]
+`;

packages/kbn-build/src/utils/link_project_executables.test.js

@@ -1,6 +1,9 @@
 import { resolve } from 'path';
 
-import { absolutePathSnaphotSerializer } from '../test_helpers';
+import {
+  absolutePathSnaphotSerializer,
+  stripAnsiSnapshotSerializer,
+} from '../test_helpers';
 import { linkProjectExecutables } from './link_project_executables';
 import { Project } from './project';
 
@@ -46,6 +49,8 @@ function getFsMockCalls() {
 }
 
 expect.addSnapshotSerializer(absolutePathSnaphotSerializer);
+expect.addSnapshotSerializer(stripAnsiSnapshotSerializer);
+
 jest.mock('./fs');
 afterEach(() => {
   jest.resetAllMocks();
@@ -66,7 +71,11 @@ describe('bin script points to a file', () => {
     const fs = require('./fs');
     fs.isFile.mockReturnValue(true);
 
+    const logMock = jest.spyOn(console, 'log').mockImplementation(() => {});
     await linkProjectExecutables(projectsByName, projectGraph);
+    logMock.mockRestore();
+
     expect(getFsMockCalls()).toMatchSnapshot('fs module calls');
+    expect(logMock.mock.calls).toMatchSnapshot('logs');
   });
 });

packages/kbn-build/yarn.lock

@@ -1524,6 +1524,12 @@ has-ansi@^2.0.0:
   dependencies:
     ansi-regex "^2.0.0"
 
+has-ansi@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-3.0.0.tgz#36077ef1d15f333484aa7fa77a28606f1c655b37"
+  dependencies:
+    ansi-regex "^3.0.0"
+
 has-flag@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51"