[kbn-build/bootstrap] symlink executables from linked packages (#16584)

* [kbn-build/bootstrap] symlink executables from linked packages

* [kbn-build/bootstrap] use snapshots for tests

* [kbn-build/jest] add absolute path serializer

packages/kbn-build/package.json

@@ -19,6 +19,7 @@
     "babel-preset-env": "^1.6.1",
     "babel-preset-stage-3": "^6.24.1",
     "chalk": "^2.3.0",
+    "cmd-shim": "^2.0.2",
     "cpy": "^6.0.0",
     "dedent": "^0.7.0",
     "del": "^3.0.0",
@@ -27,7 +28,9 @@
     "glob": "^7.1.2",
     "globby": "^7.1.1",
     "indent-string": "^3.2.0",
+    "lodash.clonedeepwith": "^4.5.0",
     "log-symbols": "^2.1.0",
+    "mkdirp": "^0.5.1",
     "ora": "^1.3.0",
     "pify": "^3.0.0",
     "prettier": "^1.9.1",

packages/kbn-build/src/commands/bootstrap.js

@@ -1,6 +1,7 @@
 import chalk from 'chalk';
 
 import { topologicallyBatchProjects } from '../utils/projects';
+import { linkProjectExecutables } from '../utils/link_project_executables';
 
 export const name = 'bootstrap';
 export const description = 'Install dependencies and crosslink projects';
@@ -20,4 +21,9 @@ export async function run(projects, projectGraph, { options }) {
       }
     }
   }
+
+  console.log(
+    chalk.bold('\nInstalls completed, linking package executables:\n')
+  );
+  await linkProjectExecutables(projects, projectGraph);
 }

packages/kbn-build/src/test_helpers/absolute_path_snaphot_serializer.js

@@ -0,0 +1,32 @@
+import { resolve, sep as pathSep } from 'path';
+import cloneDeepWith from 'lodash.clonedeepwith';
+
+const repoRoot = resolve(__dirname, '../../../../');
+
+const normalizePaths = value => {
+  let didReplacement = false;
+  const clone = cloneDeepWith(value, v => {
+    if (typeof v === 'string' && v.startsWith(repoRoot)) {
+      didReplacement = true;
+      return v
+        .replace(repoRoot, '<repoRoot>')
+        .split(pathSep) // normalize path separators
+        .join('/');
+    }
+  });
+
+  return {
+    didReplacement,
+    clone,
+  };
+};
+
+export const absolutePathSnaphotSerializer = {
+  print: (value, serialize) => {
+    return serialize(normalizePaths(value).clone);
+  },
+
+  test: value => {
+    return normalizePaths(value).didReplacement;
+  },
+};

packages/kbn-build/src/test_helpers/index.js

@@ -0,0 +1,3 @@
+export {
+  absolutePathSnaphotSerializer,
+} from './absolute_path_snaphot_serializer';

packages/kbn-build/src/utils/__snapshots__/link_project_executables.test.js.snap

@@ -0,0 +1,44 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`bin script points nowhere does not try to create symlink or node_modules/.bin directory: fs module calls 1`] = `
+Object {
+  "chmod": Array [],
+  "createSymlink": Array [],
+  "isDirectory": Array [],
+  "isFile": Array [
+    Array [
+      "<repoRoot>/packages/kbn-build/src/utils/bar/bin/bar.js",
+    ],
+  ],
+  "mkdirp": Array [],
+}
+`;
+
+exports[`bin script points to a file creates a symlink in the project node_modules/.bin directory: fs module calls 1`] = `
+Object {
+  "chmod": Array [
+    Array [
+      "<repoRoot>/packages/kbn-build/src/utils/foo/node_modules/.bin/bar",
+      "755",
+    ],
+  ],
+  "createSymlink": Array [
+    Array [
+      "<repoRoot>/packages/kbn-build/src/utils/bar/bin/bar.js",
+      "<repoRoot>/packages/kbn-build/src/utils/foo/node_modules/.bin/bar",
+      "exec",
+    ],
+  ],
+  "isDirectory": Array [],
+  "isFile": Array [
+    Array [
+      "<repoRoot>/packages/kbn-build/src/utils/bar/bin/bar.js",
+    ],
+  ],
+  "mkdirp": Array [
+    Array [
+      "<repoRoot>/packages/kbn-build/src/utils/foo/node_modules/.bin",
+    ],
+  ],
+}
+`;

packages/kbn-build/src/utils/__snapshots__/project.test.js.snap

@@ -3,3 +3,5 @@
 exports[`#ensureValidProjectDependency using link:, but with wrong path 1`] = `"[kibana] depends on [foo] using 'link:', but the path is wrong. Update its package.json to the expected value below."`;
 
 exports[`#ensureValidProjectDependency using version instead of link: 1`] = `"[kibana] depends on [foo], but it's not using the local package. Update its package.json to the expected value below."`;
+
+exports[`#getExecutables() throws CliError when bin is something strange 1`] = `"[kibana] has an invalid \\"bin\\" field in its package.json, expected an object or a string"`;

packages/kbn-build/src/utils/fs.js

@@ -1,12 +1,21 @@
 import fs from 'fs';
+import { relative, dirname } from 'path';
 import promisify from 'pify';
+import cmdShimCb from 'cmd-shim';
+import mkdirpCb from 'mkdirp';
 
 const stat = promisify(fs.stat);
+const unlink = promisify(fs.unlink);
+const symlink = promisify(fs.symlink);
+const chmod = promisify(fs.chmod);
+const cmdShim = promisify(cmdShimCb);
+const mkdirp = promisify(mkdirpCb);
 
-export async function isDirectory(path) {
+export { chmod, mkdirp };
+
+async function statTest(path, block) {
   try {
-    const targetFolder = await stat(path);
-    return targetFolder.isDirectory();
+    return block(await stat(path));
   } catch (e) {
     if (e.code === 'ENOENT') {
       return false;
@@ -14,3 +23,59 @@ export async function isDirectory(path) {
     throw e;
   }
 }
+
+/**
+ * Test if a path points to a directory
+ * @param  {String} path
+ * @return {Promise<Boolean>}
+ */
+export async function isDirectory(path) {
+  return await statTest(path, stats => stats.isDirectory());
+}
+
+/**
+ * Test if a path points to a regular file
+ * @param  {String} path
+ * @return {Promise<Boolean>}
+ */
+export async function isFile(path) {
+  return await statTest(path, stats => stats.isFile());
+}
+
+/**
+ * Create a symlink at dest that points to src. Adapted from
+ * https://github.com/lerna/lerna/blob/2f1b87d9e2295f587e4ac74269f714271d8ed428/src/FileSystemUtilities.js#L103
+ *
+ * @param  {String} src
+ * @param  {String} dest
+ * @param  {String} type 'dir', 'file', 'junction', or 'exec'. 'exec' on
+ *  windows will use the `cmd-shim` module since symlinks can't be used
+ *  for executable files on windows.
+ * @return {Promise<undefined>}
+ */
+export async function createSymlink(src, dest, type) {
+  async function forceCreate(src, dest, type) {
+    try {
+      // If something exists at `dest` we need to remove it first.
+      await unlink(dest);
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error;
+      }
+    }
+
+    await symlink(src, dest, type);
+  }
+
+  if (process.platform === 'win32') {
+    if (type === 'exec') {
+      await cmdShim(src, dest);
+    } else {
+      await forceCreate(src, dest, type);
+    }
+  } else {
+    const posixType = type === 'exec' ? 'file' : type;
+    const relativeSource = relative(dirname(dest), src);
+    await forceCreate(relativeSource, dest, posixType);
+  }
+}

packages/kbn-build/src/utils/link_project_executables.js

@@ -0,0 +1,46 @@
+import { resolve, relative, dirname } from 'path';
+
+import chalk from 'chalk';
+
+import { createSymlink, isFile, chmod, mkdirp } from './fs';
+
+/**
+ * Yarn does not link the executables from dependencies that are installed
+ * using `link:` https://github.com/yarnpkg/yarn/pull/5046
+ *
+ * We simulate this functionality by walking through each project's project
+ * dependencies, and manually linking their executables if defined. The logic
+ * for linking was mostly adapted from lerna: https://github.com/lerna/lerna/blob/1d7eb9eeff65d5a7de64dea73613b1bf6bfa8d57/src/PackageUtilities.js#L348
+ */
+export async function linkProjectExecutables(projectsByName, projectGraph) {
+  for (const [projectName, projectDeps] of projectGraph) {
+    const project = projectsByName.get(projectName);
+    const binsDir = resolve(project.nodeModulesLocation, '.bin');
+
+    for (const projectDep of projectDeps) {
+      const executables = projectDep.getExecutables();
+      for (const name of Object.keys(executables)) {
+        const srcPath = executables[name];
+
+        // existing logic from lerna -- ensure that the bin we are going to
+        // point to exists or ignore it
+        if (!await isFile(srcPath)) {
+          continue;
+        }
+
+        const dest = resolve(binsDir, name);
+
+        console.log(
+          chalk`{dim [${project.name}]} ${name} -> {dim ${relative(
+            project.path,
+            srcPath
+          )}}`
+        );
+
+        await mkdirp(dirname(dest));
+        await createSymlink(srcPath, dest, 'exec');
+        await chmod(dest, '755');
+      }
+    }
+  }
+}

packages/kbn-build/src/utils/link_project_executables.test.js

@@ -0,0 +1,72 @@
+import { resolve } from 'path';
+
+import { absolutePathSnaphotSerializer } from '../test_helpers';
+import { linkProjectExecutables } from './link_project_executables';
+import { Project } from './project';
+
+const projectsByName = new Map([
+  [
+    'foo',
+    new Project(
+      {
+        name: 'foo',
+        dependencies: {
+          bar: 'link:../bar',
+        },
+      },
+      resolve(__dirname, 'foo')
+    ),
+  ],
+  [
+    'bar',
+    new Project(
+      {
+        name: 'bar',
+        bin: 'bin/bar.js',
+      },
+      resolve(__dirname, 'bar')
+    ),
+  ],
+]);
+
+const projectGraph = new Map([
+  ['foo', [projectsByName.get('bar')]],
+  ['bar', []],
+]);
+
+function getFsMockCalls() {
+  const fs = require('./fs');
+  const fsMockCalls = {};
+  Object.keys(fs).map(key => {
+    if (jest.isMockFunction(fs[key])) {
+      fsMockCalls[key] = fs[key].mock.calls;
+    }
+  });
+  return fsMockCalls;
+}
+
+expect.addSnapshotSerializer(absolutePathSnaphotSerializer);
+jest.mock('./fs');
+afterEach(() => {
+  jest.resetAllMocks();
+});
+
+describe('bin script points nowhere', () => {
+  test('does not try to create symlink or node_modules/.bin directory', async () => {
+    const fs = require('./fs');
+    fs.isFile.mockReturnValue(false);
+
+    await linkProjectExecutables(projectsByName, projectGraph);
+    expect(getFsMockCalls()).toMatchSnapshot('fs module calls');
+  });
+});
+
+describe('bin script points to a file', () => {
+  test('creates a symlink in the project node_modules/.bin directory', async () => {
+    const fs = require('./fs');
+    fs.isFile.mockReturnValue(true);
+
+    await linkProjectExecutables(projectsByName, projectGraph);
+    expect(getFsMockCalls()).toMatchSnapshot('fs module calls');
+  });
+});

packages/kbn-build/src/utils/project.js

@@ -1,4 +1,5 @@
 import path from 'path';
+import { inspect } from 'util';
 import chalk from 'chalk';
 
 import {
@@ -82,6 +83,37 @@ export class Project {
     return name in this.scripts;
   }
 
+  getExecutables() {
+    const raw = this.json.bin;
+
+    if (!raw) {
+      return {};
+    }
+
+    if (typeof raw === 'string') {
+      return {
+        [this.name]: path.resolve(this.path, raw),
+      };
+    }
+
+    if (typeof raw === 'object') {
+      const binsConfig = {};
+      for (const binName of Object.keys(raw)) {
+        binsConfig[binName] = path.resolve(this.path, raw[binName]);
+      }
+      return binsConfig;
+    }
+
+    throw new CliError(
+      `[${this.name}] has an invalid "bin" field in its package.json, ` +
+        `expected an object or a string`,
+      {
+        package: `${this.name} (${this.packageJsonLocation})`,
+        binConfig: inspect(raw),
+      }
+    );
+  }
+
   async runScript(scriptName, args = []) {
     console.log(
       chalk.bold(

packages/kbn-build/src/utils/project.test.js

@@ -103,3 +103,42 @@ describe('#ensureValidProjectDependency', () => {
     ).toThrowErrorMatchingSnapshot();
   });
 });
+
+describe('#getExecutables()', () => {
+  test('converts bin:string to an object with absolute paths', () => {
+    const project = createProjectWith({
+      bin: './bin/script.js',
+    });
+
+    expect(project.getExecutables()).toEqual({
+      kibana: resolve(rootPath, 'bin/script.js'),
+    });
+  });
+
+  test('resolves absolute paths when bin is an object', () => {
+    const project = createProjectWith({
+      bin: {
+        script1: 'bin/script1.js',
+        script2: './bin/script2.js',
+      },
+    });
+
+    expect(project.getExecutables()).toEqual({
+      script1: resolve(rootPath, 'bin/script1.js'),
+      script2: resolve(rootPath, 'bin/script2.js'),
+    });
+  });
+
+  test('returns empty object when bin is missing, or falsy', () => {
+    expect(createProjectWith({}).getExecutables()).toEqual({});
+    expect(createProjectWith({ bin: null }).getExecutables()).toEqual({});
+    expect(createProjectWith({ bin: false }).getExecutables()).toEqual({});
+    expect(createProjectWith({ bin: 0 }).getExecutables()).toEqual({});
+  });
+
+  test('throws CliError when bin is something strange', () => {
+    expect(() =>
+      createProjectWith({ bin: 1 }).getExecutables()
+    ).toThrowErrorMatchingSnapshot();
+  });
+});

packages/kbn-build/yarn.lock

@@ -888,6 +888,13 @@ cliui@^3.2.0:
     strip-ansi "^3.0.1"
     wrap-ansi "^2.0.0"
 
+cmd-shim@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-2.0.2.tgz#6fcbda99483a8fd15d7d30a196ca69d688a2efdb"
+  dependencies:
+    graceful-fs "^4.1.2"
+    mkdirp "~0.5.0"
+
 co@^4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
@@ -1886,6 +1893,10 @@ locate-path@^2.0.0:
     p-locate "^2.0.0"
     path-exists "^3.0.0"
 
+lodash.clonedeepwith@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.clonedeepwith/-/lodash.clonedeepwith-4.5.0.tgz#6ee30573a03a1a60d670a62ef33c10cf1afdbdd4"
+
 lodash@^4, lodash@^4.14.0, lodash@^4.17.4:
   version "4.17.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"