[plugins] use module ids to import across plugins

packages/kbn-find-used-node-modules/BUILD.bazel

@@ -0,0 +1,122 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-find-used-node-modules"
+PKG_REQUIRE_NAME = "@kbn/find-used-node-modules"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+  "//packages/kbn-babel-preset",
+  "@npm//@babel/core",
+  "@npm//@babel/types",
+  "@npm//@babel/traverse",
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "//packages/kbn-import-resolver:npm_module_types",
+  "@npm//@types/node",
+  "@npm//@types/jest",
+  "@npm//@types/babel__core",
+  "@npm//@babel/traverse",
+  "@npm//@babel/types",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  root_dir = "src",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-find-used-node-modules/README.md

@@ -0,0 +1,7 @@
+# @kbn/find-used-node-modules
+
+Simple abstraction over the `@babel/parser` and the `@babel/traverse` to find the node_modules used by a list of files.
+
+## `findUsedNodeModules(resolver, entryPaths): string[]`
+
+Pass an `ImportResolver` instance from `@kbn/import-resolver` and a list of absolute paths to JS files to get the list of `node_modules` used by those files and the files the import.

packages/kbn-find-used-node-modules/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-find-used-node-modules'],
+};

packages/kbn-find-used-node-modules/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@kbn/find-used-node-modules",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "kibana": {
+    "devOnly": true
+  }
+}

packages/kbn-find-used-node-modules/src/find_used_node_modules.test.ts

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { findUsedNodeModules } from './find_used_node_modules';
+import { ImportResolver } from '@kbn/import-resolver';
+
+jest.mock('./fs');
+
+const FILES: Record<string, string> = {
+  '/foo.js': `
+    require('./bar.js')
+  `,
+  '/bar.js': `
+    require('./box')
+  `,
+  '/box.js': `
+    require('foo')
+  `,
+};
+
+class MockResolver extends ImportResolver {
+  constructor() {
+    super('/', new Map(), new Map());
+  }
+
+  isBazelPackage = jest.fn();
+  resolve = jest.fn();
+}
+
+const RESOLVER = new MockResolver();
+
+beforeEach(() => {
+  jest.resetAllMocks();
+  jest.requireMock('./fs').readFile.mockImplementation((path: string) => {
+    if (Object.hasOwn(FILES, path)) {
+      return FILES[path];
+    }
+
+    const error: any = new Error(`ENOENT, missing file [${path}]`);
+    error.code = 'ENOENT';
+    throw error;
+  });
+});
+
+describe('findUsedNodeModules()', () => {
+  it('excludes built-in modules', async () => {
+    RESOLVER.resolve.mockImplementation(() => ({
+      type: 'built-in',
+    }));
+
+    const results = await findUsedNodeModules({
+      entryPaths: ['/foo.js'],
+      resolver: RESOLVER,
+      findUsedPeers: false,
+    });
+
+    expect(RESOLVER.resolve).toHaveBeenCalledTimes(1);
+    expect(results).toEqual([]);
+  });
+
+  it('returns node_modules found in the source file', async () => {
+    RESOLVER.resolve.mockImplementation((req) => {
+      if (req === './bar.js') {
+        return {
+          type: 'file',
+          nodeModule: '@foo/bar',
+          absolute: '/bar.js',
+        };
+      }
+
+      throw new Error('unexpected request');
+    });
+
+    const results = await findUsedNodeModules({
+      entryPaths: ['/foo.js'],
+      resolver: RESOLVER,
+      findUsedPeers: false,
+    });
+    expect(RESOLVER.resolve).toHaveBeenCalledTimes(1);
+    expect(results).toEqual(['@foo/bar']);
+  });
+
+  it('returns node_modules found in referenced files', async () => {
+    RESOLVER.resolve.mockImplementation((req) => {
+      if (req === './bar.js') {
+        return {
+          type: 'file',
+          absolute: '/bar.js',
+        };
+      }
+
+      if (req === './box') {
+        return {
+          type: 'file',
+          nodeModule: '@foo/box',
+          absolute: '/box.js',
+        };
+      }
+
+      throw new Error('unexpected request');
+    });
+
+    const results = await findUsedNodeModules({
+      entryPaths: ['/foo.js'],
+      resolver: RESOLVER,
+      findUsedPeers: false,
+    });
+    expect(RESOLVER.resolve).toHaveBeenCalledTimes(2);
+    expect(results).toEqual(['@foo/box']);
+  });
+
+  it('does not traverse node_modules', async () => {
+    RESOLVER.resolve.mockImplementation((req) => {
+      if (req === './bar.js') {
+        return {
+          type: 'file',
+          absolute: '/bar.js',
+        };
+      }
+
+      if (req === './box') {
+        return {
+          type: 'file',
+          nodeModule: '@foo/box',
+          absolute: '/box.js',
+        };
+      }
+
+      throw new Error('unexpected request');
+    });
+
+    const results = await findUsedNodeModules({
+      entryPaths: ['/foo.js'],
+      resolver: RESOLVER,
+      findUsedPeers: false,
+    });
+    expect(RESOLVER.resolve).toHaveBeenCalledTimes(2);
+    expect(results).toEqual(['@foo/box']);
+  });
+
+  it('does traverse node_modules which are also bazel packages', async () => {
+    RESOLVER.resolve.mockImplementation((req) => {
+      if (req === './bar.js') {
+        return {
+          type: 'file',
+          absolute: '/bar.js',
+        };
+      }
+
+      if (req === './box') {
+        return {
+          type: 'file',
+          nodeModule: '@foo/box',
+          absolute: '/box.js',
+        };
+      }
+
+      if (req === 'foo') {
+        return {
+          type: 'file',
+          nodeModule: '@foo/core',
+          absolute: '/non-existant',
+        };
+      }
+
+      throw new Error('unexpected request');
+    });
+
+    RESOLVER.isBazelPackage.mockImplementation((pkgId) => {
+      return pkgId === '@foo/box';
+    });
+
+    const results = await findUsedNodeModules({
+      entryPaths: ['/foo.js'],
+      resolver: RESOLVER,
+      findUsedPeers: false,
+    });
+    expect(RESOLVER.resolve).toHaveBeenCalledTimes(3);
+    expect(results).toEqual(['@foo/box', '@foo/core']);
+  });
+});

packages/kbn-find-used-node-modules/src/find_used_node_modules.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { asyncForEachWithLimit } from '@kbn/std';
+import type { ImportResolver } from '@kbn/import-resolver';
+
+import { readFile, readFileSync } from './fs';
+import { getImportRequests } from './get_import_requests';
+
+function isObj(v: any): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null;
+}
+
+function getPeerDeps(thisNodeModule: string) {
+  const pkgPath = require.resolve(`${thisNodeModule}/package.json`);
+  const pkg = JSON.parse(readFileSync(pkgPath));
+
+  if (isObj(pkg) && isObj(pkg.peerDependencies)) {
+    return Object.keys(pkg.peerDependencies);
+  } else {
+    return [];
+  }
+}
+
+interface Options {
+  resolver: ImportResolver;
+  entryPaths: string[];
+  findUsedPeers: boolean;
+  // if we are finding used modules in a node_module, this must be the name of the node_module
+  // we should treat as "this module" rather than "another node module"
+  thisNodeModule?: string;
+}
+
+/**
+ * Parse a list of entry paths and find the node_modules which are required by them. If the
+ * entry path requires/imports a non-node_module then that file is scanned too, deeply, until
+ * all referenced files are scanned.
+ *
+ * Optionally, we can find the used peers of the used node_modules. This will keep track of all
+ * the paths we use to enter a node_module and then traverse from those points, finding the
+ * used modules and comparing those to the `peerDependencies` listed in the node_module's package.json
+ * file. If a used dependeny is in the `peerDependencies` and is used by the node_module it will
+ * be included in the results.
+ *
+ * This was implemented mostly for `@emotion/react` which is used by @elastic/eui but only listed
+ * as a peerDependency. If we didn't keep it in the Kibana package.json then the package would not
+ * be installed and cause an error on startup because `@emotion/react` can't be found. We used to
+ * solve this by scanning the node_modules directory for all the packages which are used but that
+ * was much slower and lead to extra entries in package.json.
+ */
+export async function findUsedNodeModules(options: Options) {
+  const queue = new Set<string>(options.entryPaths);
+  const results = new Set<string>();
+
+  const entryPathsIntoNodeModules = new Map<string, Set<string>>();
+
+  for (const path of queue) {
+    if (Path.extname(path) !== '.js') {
+      continue;
+    }
+
+    const dirname = Path.dirname(path);
+    const code = await readFile(path);
+    const reqs = getImportRequests(code);
+
+    for (const req of reqs) {
+      // resolve the request to it's actual file on dist
+      const result = options.resolver.resolve(req, dirname);
+
+      // ignore non-file resolution results, these represent files which aren't on
+      // the file-system yet (like during the build) built-ins, explicitily ignored
+      // files, and @types only imports
+      if (result?.type !== 'file') {
+        continue;
+      }
+
+      // if the result points to a node_module (or another node_module)...
+      if (result.nodeModule && result.nodeModule !== options.thisNodeModule) {
+        // add it to the results
+        results.add(result.nodeModule);
+
+        // record this absolute path as an entry path into the node module from our entries, if we
+        // need to scan this node_module for used deps we need to know how we access it.
+        const nmEntries = entryPathsIntoNodeModules.get(result.nodeModule);
+        if (!nmEntries) {
+          entryPathsIntoNodeModules.set(result.nodeModule, new Set([result.absolute]));
+        } else {
+          nmEntries.add(result.absolute);
+        }
+      }
+
+      // no need to scan node_modules unless they're bazel packages
+      if (
+        !result.nodeModule ||
+        result.nodeModule === options.thisNodeModule ||
+        options.resolver.isBazelPackage(result.nodeModule)
+      ) {
+        queue.add(result.absolute);
+      }
+    }
+  }
+
+  if (options.findUsedPeers) {
+    await asyncForEachWithLimit(results, 10, async (dep) => {
+      const entryPaths = entryPathsIntoNodeModules.get(dep);
+      if (!entryPaths?.size) {
+        return;
+      }
+
+      const peerDeps = getPeerDeps(dep);
+      if (!peerDeps.length) {
+        return;
+      }
+
+      const usedInside = await findUsedNodeModules({
+        resolver: options.resolver,
+        entryPaths: Array.from(entryPaths),
+        findUsedPeers: false,
+        thisNodeModule: dep,
+      });
+
+      for (const peer of peerDeps) {
+        if (usedInside.includes(peer)) {
+          results.add(peer);
+        }
+      }
+    });
+  }
+
+  return Array.from(results).sort((a, b) => a.localeCompare(b));
+}

packages/kbn-find-used-node-modules/src/fs.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fs from 'fs';
+import Fsp from 'fs/promises';
+
+export function readFileSync(path: string) {
+  return Fs.readFileSync(path, 'utf8');
+}
+
+export function readFile(path: string) {
+  return Fsp.readFile(path, 'utf8');
+}

packages/kbn-find-used-node-modules/src/get_import_requests.test.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getImportRequests } from './get_import_requests';
+
+describe('getImportRequests()', () => {
+  it('should get requests from `require`', () => {
+    const rawCode = `/*foo*/require('dep1'); const bar = 1;`;
+    const foundDeps = getImportRequests(rawCode);
+    expect(foundDeps).toMatchInlineSnapshot(`
+      Array [
+        "dep1",
+      ]
+    `);
+  });
+
+  it('should get requests from `require.resolve`', () => {
+    const rawCode = `/*foo*/require.resolve('dep2'); const bar = 1;`;
+    const foundDeps = getImportRequests(rawCode);
+    expect(foundDeps).toMatchInlineSnapshot(`
+      Array [
+        "dep2",
+      ]
+    `);
+  });
+
+  it('should get requests from `import`', () => {
+    const rawCode = `/*foo*/import dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`;
+    const foundDeps = getImportRequests(rawCode);
+    expect(foundDeps).toMatchInlineSnapshot(`
+      Array [
+        "dep1",
+        "dep2",
+      ]
+    `);
+  });
+
+  it('should get requests from `export from`', () => {
+    const rawCode = `/*foo*/export dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`;
+    const foundDeps = getImportRequests(rawCode);
+    expect(foundDeps).toMatchInlineSnapshot(`
+      Array [
+        "dep1",
+        "dep2",
+      ]
+    `);
+  });
+
+  it('should get requests from `export * from`', () => {
+    const rawCode = `/*foo*/export * from 'dep1'; export dep2 from 'dep2';const bar = 1;`;
+    const foundDeps = getImportRequests(rawCode);
+    expect(foundDeps).toMatchInlineSnapshot(`
+      Array [
+        "dep1",
+        "dep2",
+      ]
+    `);
+  });
+});

packages/kbn-find-used-node-modules/src/get_import_requests.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as parser from '@babel/parser';
+import traverse from '@babel/traverse';
+// @ts-expect-error Not available with types
+import babelParserOptions from '@kbn/babel-preset/common_babel_parser_options';
+
+import { importVisitor } from './import_visitor';
+
+/**
+ * Parse the code and return an array of all the import requests in that file
+ */
+export function getImportRequests(code: string) {
+  const importRequests: string[] = [];
+
+  // Parse and get the code AST
+  const ast = parser.parse(code, babelParserOptions);
+
+  // Loop through the code AST with
+  // the defined visitors
+  traverse(ast, importVisitor(importRequests));
+
+  return importRequests;
+}

packages/kbn-find-used-node-modules/src/import_visitor.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as T from '@babel/types';
+import type { Visitor } from '@babel/core';
+
+/**
+ * @notice
+ *
+ * This product has relied on ASTExplorer that is licensed under MIT.
+ */
+
+// AST check for require expressions
+const isRequire = ({ callee }: T.CallExpression) =>
+  T.isIdentifier(callee) && callee.name === 'require';
+
+// AST check for require.resolve expressions
+const isRequireResolve = ({ callee }: T.CallExpression) =>
+  T.isMemberExpression(callee) &&
+  T.isIdentifier(callee.object) &&
+  callee.object.name === 'require' &&
+  T.isIdentifier(callee.property) &&
+  callee.property.name === 'resolve';
+
+/**
+ * Create a Babel AST visitor that will write import requests into the passed array
+ */
+export function importVisitor(importRequests: string[]): Visitor {
+  // This was built with help on an ast explorer and some ESTree docs
+  // like the babel parser ast spec and the main docs for the Esprima
+  // which is a complete and useful docs for the ESTree spec.
+  //
+  // https://astexplorer.net
+  // https://github.com/babel/babel/blob/master/packages/babel-parser/ast/spec.md
+  // https://esprima.readthedocs.io/en/latest/syntax-tree-format.html
+  // https://github.com/estree/estree
+
+  // Visitors to traverse and find dependencies
+  return {
+    // raw values on require + require.resolve
+    CallExpression: ({ node }) => {
+      if (isRequire(node) || isRequireResolve(node)) {
+        const nodeArguments = node.arguments;
+        const reqArg = Array.isArray(nodeArguments) ? nodeArguments.shift() : null;
+
+        if (!reqArg) {
+          return;
+        }
+
+        if (reqArg.type === 'StringLiteral') {
+          importRequests.push(reqArg.value);
+        }
+      }
+    },
+
+    // raw values on import
+    ImportDeclaration: ({ node }) => {
+      // Get string values from import expressions
+      const importSource = node.source;
+      importRequests.push(importSource.value);
+    },
+
+    // raw values on export from
+    ExportNamedDeclaration: ({ node }) => {
+      // Get string values from export from expressions
+      if (node.source) {
+        importRequests.push(node.source.value);
+      }
+    },
+
+    // raw values on export * from
+    ExportAllDeclaration: ({ node }) => {
+      const exportAllFromSource = node.source;
+      importRequests.push(exportAllFromSource.value);
+    },
+  };
+}

packages/kbn-find-used-node-modules/src/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { findUsedNodeModules } from './find_used_node_modules';

packages/kbn-find-used-node-modules/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "rootDir": "src",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ]
+}