[plugin] Handle Kibana package dependencies (#16509) (#16680)

* [plugin] Handle Kibana package dependencies

* Clean up 'link:' dep check in plugin installer

* Tests for 'prepareProjectDependencies'

* Remove unnecessary fn from 'prepareProjectDependencies'

* Move prepareProjectDependencies into @kbn/build

* update snapshot

* Move test to Jest

* clarification

package.json

@@ -82,6 +82,7 @@
     "@elastic/test-subj-selector": "0.2.1",
     "@elastic/ui-ace": "0.2.3",
     "@kbn/babel-preset": "link:packages/kbn-babel-preset",
+    "@kbn/build": "link:packages/kbn-build",
     "JSONStream": "1.1.1",
     "accept-language-parser": "1.2.0",
     "angular": "1.6.5",
@@ -219,7 +220,6 @@
     "@elastic/eslint-config-kibana": "link:packages/eslint-config-kibana",
     "@elastic/eslint-import-resolver-kibana": "1.0.0",
     "@elastic/eslint-plugin-kibana-custom": "link:packages/eslint-plugin-kibana-custom",
-    "@kbn/build": "link:packages/kbn-build",
     "angular-mocks": "1.4.7",
     "babel-eslint": "8.1.2",
     "backport": "2.2.0",

packages/kbn-build/package.json

@@ -1,10 +1,5 @@
 {
   "name": "@kbn/build",
-  "kibana": {
-    "build": {
-      "skip": true
-    }
-  },
   "main": "./dist/index.js",
   "version": "1.0.0",
   "license": "Apache-2.0",

packages/kbn-build/src/index.js

@@ -1,3 +1,6 @@
 export { run } from './cli';
-export { buildProductionProjects } from './production';
+export {
+  buildProductionProjects,
+  prepareExternalProjectDependencies,
+} from './production';
 export { transformDependencies } from './utils/package_json';

packages/kbn-build/src/production/__fixtures__/external_packages/with_kibana_link_deps/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "quux",
+  "version": "1.0.0",
+  "dependencies": {
+    "@kbn/foo": "link:../../kibana/packages/foo"
+  }
+}

packages/kbn-build/src/production/__fixtures__/external_packages/with_other_link_deps/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "quux",
+  "version": "1.0.0",
+  "dependencies": {
+    "bar": "link:../foo/packages/bar"
+  }
+}

packages/kbn-build/src/production/__tests__/build_production_projects.js

@@ -4,7 +4,7 @@ import copy from 'cpy';
 import { resolve } from 'path';
 import globby from 'globby';
 
-import { buildProductionProjects } from '../';
+import { buildProductionProjects } from '../build_production_projects';
 import { getProjects } from '../../utils/projects';
 
 // This is specifically a Mocha test instead of a Jest test because it's slow

packages/kbn-build/src/production/build_production_projects.js

@@ -0,0 +1,90 @@
+import del from 'del';
+import { relative, resolve } from 'path';
+import copy from 'cpy';
+
+import { getProjectPaths } from '../config';
+import {
+  getProjects,
+  buildProjectGraph,
+  topologicallyBatchProjects,
+} from '../utils/projects';
+import {
+  createProductionPackageJson,
+  writePackageJson,
+} from '../utils/package_json';
+import { isDirectory } from '../utils/fs';
+
+export async function buildProductionProjects({ kibanaRoot, buildRoot }) {
+  const projectPaths = getProjectPaths(kibanaRoot, {
+    'skip-kibana': true,
+    'skip-kibana-extra': true,
+  });
+
+  const projects = await getProductionProjects(kibanaRoot, projectPaths);
+  const projectGraph = buildProjectGraph(projects);
+  const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
+
+  const projectNames = [...projects.values()].map(project => project.name);
+  console.log(`Preparing production build for [${projectNames.join(', ')}]`);
+
+  for (const batch of batchedProjects) {
+    for (const project of batch) {
+      await deleteTarget(project);
+      await buildProject(project);
+      await copyToBuild(project, kibanaRoot, buildRoot);
+    }
+  }
+}
+
+/**
+ * Returns only the projects that should be built into the production bundle
+ */
+async function getProductionProjects(kibanaRoot, projectPaths) {
+  const projects = await getProjects(kibanaRoot, projectPaths);
+
+  const buildProjects = new Map();
+  for (const [name, project] of projects.entries()) {
+    if (!project.skipFromBuild()) {
+      buildProjects.set(name, project);
+    }
+  }
+
+  return buildProjects;
+}
+
+async function deleteTarget(project) {
+  const targetDir = project.targetLocation;
+
+  if (await isDirectory(targetDir)) {
+    await del(targetDir, { force: true });
+  }
+}
+
+async function buildProject(project) {
+  if (project.hasScript('build')) {
+    await project.runScript('build');
+  }
+}
+
+async function copyToBuild(project, kibanaRoot, buildRoot) {
+  // We want the package to have the same relative location within the build
+  const relativeProjectPath = relative(kibanaRoot, project.path);
+  const buildProjectPath = resolve(buildRoot, relativeProjectPath);
+
+  // When copying all the files into the build, we exclude `package.json` as we
+  // write a separate "production-ready" `package.json` below, and we exclude
+  // `node_modules` because we want the Kibana build to actually install all
+  // dependencies. The primary reason for allowing the Kibana build process to
+  // install the dependencies is that it will "dedupe" them, so we don't include
+  // unnecessary copies of dependencies.
+  await copy(['**/*', '!package.json', '!node_modules/**'], buildProjectPath, {
+    cwd: project.path,
+    parents: true,
+    nodir: true,
+    dot: true,
+  });
+
+  const packageJson = project.json;
+  const preparedPackageJson = createProductionPackageJson(packageJson);
+  await writePackageJson(buildProjectPath, preparedPackageJson);
+}

packages/kbn-build/src/production/index.js

@@ -1,90 +1,4 @@
-import del from 'del';
-import { relative, resolve } from 'path';
-import copy from 'cpy';
-
-import { getProjectPaths } from '../config';
-import {
-  getProjects,
-  buildProjectGraph,
-  topologicallyBatchProjects,
-} from '../utils/projects';
-import {
-  createProductionPackageJson,
-  writePackageJson,
-} from '../utils/package_json';
-import { isDirectory } from '../utils/fs';
-
-export async function buildProductionProjects({ kibanaRoot, buildRoot }) {
-  const projectPaths = getProjectPaths(kibanaRoot, {
-    'skip-kibana': true,
-    'skip-kibana-extra': true,
-  });
-
-  const projects = await getProductionProjects(kibanaRoot, projectPaths);
-  const projectGraph = buildProjectGraph(projects);
-  const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
-
-  const projectNames = [...projects.values()].map(project => project.name);
-  console.log(`Preparing production build for [${projectNames.join(', ')}]`);
-
-  for (const batch of batchedProjects) {
-    for (const project of batch) {
-      await deleteTarget(project);
-      await buildProject(project);
-      await copyToBuild(project, kibanaRoot, buildRoot);
-    }
-  }
-}
-
-/**
- * Returns only the projects that should be built into the production bundle
- */
-async function getProductionProjects(kibanaRoot, projectPaths) {
-  const projects = await getProjects(kibanaRoot, projectPaths);
-
-  const buildProjects = new Map();
-  for (const [name, project] of projects.entries()) {
-    if (!project.skipFromBuild()) {
-      buildProjects.set(name, project);
-    }
-  }
-
-  return buildProjects;
-}
-
-async function deleteTarget(project) {
-  const targetDir = project.targetLocation;
-
-  if (await isDirectory(targetDir)) {
-    await del(targetDir, { force: true });
-  }
-}
-
-async function buildProject(project) {
-  if (project.hasScript('build')) {
-    await project.runScript('build');
-  }
-}
-
-async function copyToBuild(project, kibanaRoot, buildRoot) {
-  // We want the package to have the same relative location within the build
-  const relativeProjectPath = relative(kibanaRoot, project.path);
-  const buildProjectPath = resolve(buildRoot, relativeProjectPath);
-
-  // When copying all the files into the build, we exclude `package.json` as we
-  // write a separate "production-ready" `package.json` below, and we exclude
-  // `node_modules` because we want the Kibana build to actually install all
-  // dependencies. The primary reason for allowing the Kibana build process to
-  // install the dependencies is that it will "dedupe" them, so we don't include
-  // unnecessary copies of dependencies.
-  await copy(['**/*', '!package.json', '!node_modules/**'], buildProjectPath, {
-    cwd: project.path,
-    parents: true,
-    nodir: true,
-    dot: true,
-  });
-
-  const packageJson = project.json;
-  const preparedPackageJson = createProductionPackageJson(packageJson);
-  await writePackageJson(buildProjectPath, preparedPackageJson);
-}
+export { buildProductionProjects } from './build_production_projects';
+export {
+  prepareExternalProjectDependencies,
+} from './prepare_project_dependencies';

packages/kbn-build/src/production/prepare_project_dependencies.js

@@ -0,0 +1,37 @@
+import { Project } from '../utils/project';
+import { isLinkDependency } from '../utils/package_json';
+
+/**
+ * All external projects are located within `../kibana-extra/{plugin}` relative
+ * to Kibana itself.
+ */
+const isKibanaDep = depVersion => depVersion.includes('../../kibana/');
+
+/**
+ * This prepares the dependencies for an _external_ project.
+ */
+export async function prepareExternalProjectDependencies(projectPath) {
+  const project = await Project.fromPath(projectPath);
+
+  if (!project.hasDependencies()) {
+    return;
+  }
+
+  const deps = project.allDependencies;
+
+  for (const depName of Object.keys(deps)) {
+    const depVersion = deps[depName];
+
+    // Kibana currently only supports `link:` dependencies on Kibana's own
+    // packages, as these are packaged into the `node_modules` folder when
+    // Kibana is built, so we don't need to take any action to enable
+    // `require(...)` to resolve for these packages.
+    if (isLinkDependency(depVersion) && !isKibanaDep(depVersion)) {
+      // For non-Kibana packages we need to set up symlinks during the
+      // installation process, but this is not something we support yet.
+      throw new Error(
+        'This plugin is using `link:` dependencies for non-Kibana packages'
+      );
+    }
+  }
+}

packages/kbn-build/src/production/prepare_project_dependencies.test.js

@@ -0,0 +1,23 @@
+import { resolve, join } from 'path';
+
+import { prepareExternalProjectDependencies } from './prepare_project_dependencies';
+
+const packagesFixtures = resolve(__dirname, '__fixtures__/external_packages');
+
+test('does nothing when Kibana `link:` dependencies', async () => {
+  const projectPath = join(packagesFixtures, 'with_kibana_link_deps');
+
+  // We're checking for undefined, but we don't really care about what's
+  // returned, we only care about it resolving.
+  await expect(
+    prepareExternalProjectDependencies(projectPath)
+  ).resolves.toBeUndefined();
+});
+
+test('throws if non-Kibana `link` dependencies', async () => {
+  const projectPath = join(packagesFixtures, 'with_other_link_deps');
+
+  await expect(prepareExternalProjectDependencies(projectPath)).rejects.toThrow(
+    'This plugin is using `link:` dependencies for non-Kibana packages'
+  );
+});

packages/kbn-build/src/utils/__snapshots__/link_project_executables.test.js.snap

@@ -14,6 +14,7 @@ Object {
     ],
   ],
   "mkdirp": Array [],
+  "readFile": Array [],
 }
 `;
 
@@ -58,6 +59,7 @@ Object {
       "<repoRoot>/packages/kbn-build/src/utils/baz/node_modules/.bin",
     ],
   ],
+  "readFile": Array [],
 }
 `;
 

packages/kbn-build/src/utils/fs.js

@@ -5,13 +5,14 @@ import cmdShimCb from 'cmd-shim';
 import mkdirpCb from 'mkdirp';
 
 const stat = promisify(fs.stat);
+const readFile = promisify(fs.readFile);
 const unlink = promisify(fs.unlink);
 const symlink = promisify(fs.symlink);
 const chmod = promisify(fs.chmod);
 const cmdShim = promisify(cmdShimCb);
 const mkdirp = promisify(mkdirpCb);
 
-export { chmod, mkdirp };
+export { chmod, mkdirp, readFile };
 
 async function statTest(path, block) {
   try {

packages/kbn-build/src/utils/package_json.js

@@ -15,6 +15,8 @@ export const createProductionPackageJson = pkgJson => ({
   dependencies: transformDependencies(pkgJson.dependencies),
 });
 
+export const isLinkDependency = depVersion => depVersion.startsWith('link:');
+
 /**
  * Replaces `link:` dependencies with `file:` dependencies. When installing
  * dependencies, these `file:` dependencies will be copied into `node_modules`
@@ -28,7 +30,7 @@ export function transformDependencies(dependencies = {}) {
   const newDeps = {};
   for (const name of Object.keys(dependencies)) {
     const depVersion = dependencies[name];
-    if (depVersion.startsWith('link:')) {
+    if (isLinkDependency(depVersion)) {
       newDeps[name] = depVersion.replace('link:', 'file:');
     } else {
       newDeps[name] = depVersion;

packages/kbn-build/src/utils/project.js

@@ -7,11 +7,9 @@ import {
   runScriptInPackage,
   runScriptInPackageStreaming,
 } from './scripts';
-import { readPackageJson } from './package_json';
+import { readPackageJson, isLinkDependency } from './package_json';
 import { CliError } from './errors';
 
-const PREFIX = 'link:';
-
 export class Project {
   static async fromPath(path) {
     const pkgJson = await readPackageJson(path);
@@ -44,7 +42,7 @@ export class Project {
     );
 
     const versionInPackageJson = this.allDependencies[project.name];
-    const expectedVersionInPackageJson = `${PREFIX}${relativePathToProject}`;
+    const expectedVersionInPackageJson = `link:${relativePathToProject}`;
 
     if (versionInPackageJson === expectedVersionInPackageJson) {
       return;
@@ -57,11 +55,11 @@ export class Project {
       actual: `"${project.name}": "${versionInPackageJson}"`,
     };
 
-    if (versionInPackageJson.startsWith(PREFIX)) {
+    if (isLinkDependency(versionInPackageJson)) {
       throw new CliError(
         `[${this.name}] depends on [${
           project.name
-        }] using '${PREFIX}', but the path is wrong. ${updateMsg}`,
+        }] using 'link:', but the path is wrong. ${updateMsg}`,
         meta
       );
     }

src/cli_plugin/install/install.js

@@ -6,6 +6,7 @@ import { extract, getPackData } from './pack';
 import { renamePlugin } from './rename';
 import { sync as rimrafSync } from 'rimraf';
 import { existingInstall, rebuildCache, assertVersion } from './kibana';
+import { prepareExternalProjectDependencies } from '@kbn/build';
 import mkdirp from 'mkdirp';
 
 const mkdir = Promise.promisify(mkdirp);
@@ -28,6 +29,8 @@ export default async function install(settings, logger) {
 
     assertVersion(settings);
 
+    await prepareExternalProjectDependencies(settings.workingPath);
+
     await renamePlugin(settings.workingPath, path.join(settings.pluginDir, settings.plugins[0].name));
 
     await rebuildCache(settings, logger);