Implement package linter (#148496)

This PR implements a linter like the TS Project linter, except for
packages in the repo. It does this by extracting the reusable bits from
the TS Project linter and reusing them for the project linter. The only
rule that exists for packages right now is that the "name" in the
package.json file matches the "id" in Kibana.jsonc. The goal is to use a
rule to migrate kibana.json files on the future.

Additionally, a new rule for validating the indentation of tsconfig.json
files was added.

Validating and fixing violations is what has triggered review by so many
teams, but we plan to treat those review requests as notifications of
the changes and not as blockers for merging.

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

kbn_pm/src/commands/bootstrap/bootstrap_command.mjs

@@ -14,9 +14,10 @@ import { haveNodeModulesBeenManuallyDeleted, removeYarnIntegrityFileIfExists } f
 import { setupRemoteCache } from './setup_remote_cache.mjs';
 import { sortPackageJson } from './sort_package_json.mjs';
 import { regeneratePackageMap } from './regenerate_package_map.mjs';
+import { regenerateTsconfigPaths } from './regenerate_tsconfig_paths.mjs';
 import { regenerateBaseTsconfig } from './regenerate_base_tsconfig.mjs';
-import { packageDiscovery, pluginDiscovery } from './discovery.mjs';
-import { validatePackageJson } from './validate_package_json.mjs';
+import { discovery } from './discovery.mjs';
+import { updatePackageJson } from './update_package_json.mjs';
 
 /** @type {import('../../lib/command').Command} */
 export const command = {
@@ -61,13 +62,33 @@ export const command = {
     const forceInstall =
       args.getBooleanValue('force-install') ?? (await haveNodeModulesBeenManuallyDeleted());
 
-    await Bazel.tryRemovingBazeliskFromYarnGlobal(log);
+    const [{ packages, plugins, tsConfigsPaths }] = await Promise.all([
+      // discover the location of packages, plugins, etc
+      await time('discovery', discovery),
 
-    // Install bazel machinery tools if needed
-    await Bazel.ensureInstalled(log);
+      (async () => {
+        await Bazel.tryRemovingBazeliskFromYarnGlobal(log);
 
-    // Setup remote cache settings in .bazelrc.cache if needed
-    await setupRemoteCache(log);
+        // Install bazel machinery tools if needed
+        await Bazel.ensureInstalled(log);
+
+        // Setup remote cache settings in .bazelrc.cache if needed
+        await setupRemoteCache(log);
+      })(),
+    ]);
+
+    // generate the package map and package.json file, if necessary
+    await Promise.all([
+      time('regenerate package map', async () => {
+        await regeneratePackageMap(packages, plugins, log);
+      }),
+      time('regenerate tsconfig map', async () => {
+        await regenerateTsconfigPaths(tsConfigsPaths, log);
+      }),
+      time('update package json', async () => {
+        await updatePackageJson(packages, log);
+      }),
+    ]);
 
     // Bootstrap process for Bazel packages
     // Bazel is now managing dependencies so yarn install
@@ -85,34 +106,16 @@ export const command = {
       });
     }
 
-    // discover the location of packages and plugins
-    const [plugins, packages] = await Promise.all([
-      time('plugin discovery', pluginDiscovery),
-      time('package discovery', packageDiscovery),
-    ]);
-
-    // generate the package map which powers the resolver and several other features
-    // needed as an input to the bazel builds
-    await time('regenerate package map', async () => {
-      await regeneratePackageMap(packages, plugins, log);
-    });
-
     await time('pre-build webpack bundles for packages', async () => {
       await Bazel.buildWebpackBundles(log, { offline, quiet });
     });
 
-    await time('regenerate tsconfig.base.json', async () => {
-      await regenerateBaseTsconfig();
-    });
-
     await Promise.all([
-      time('sort package json', async () => {
-        await sortPackageJson();
+      time('regenerate tsconfig.base.json', async () => {
+        await regenerateBaseTsconfig();
       }),
-      time('validate package json', async () => {
-        // now that deps are installed we can import `@kbn/yarn-lock-validator`
-        const { kibanaPackageJson } = External['@kbn/repo-info']();
-        await validatePackageJson(kibanaPackageJson, log);
+      time('sort package json', async () => {
+        await sortPackageJson(log);
       }),
       validate
         ? time('validate dependencies', async () => {

kbn_pm/src/commands/bootstrap/discovery.mjs

@@ -6,33 +6,84 @@
  * Side Public License, v 1.
  */
 
+import Path from 'path';
+import Fs from 'fs';
+import ChildProcess from 'child_process';
+import { promisify } from 'util';
+
 import { REPO_ROOT } from '../../lib/paths.mjs';
+const execAsync = promisify(ChildProcess.execFile);
 
-// we need to run these in order to generate the pkg map which is used by things
-// like `@kbn/babel-register`, so we have to import the JS files directory and can't
-// rely on `@kbn/babel-register`.
-
-export async function packageDiscovery() {
-  const { discoverBazelPackages } = await import(
-    // eslint-disable-next-line @kbn/imports/uniform_imports
-    '../../../../packages/kbn-bazel-packages/index.js'
-  );
-
-  return await discoverBazelPackages(REPO_ROOT);
-}
-
-export async function pluginDiscovery() {
+export async function discovery() {
   const { getPluginSearchPaths, simpleKibanaPlatformPluginDiscovery } = await import(
     // eslint-disable-next-line @kbn/imports/uniform_imports
     '../../../../packages/kbn-plugin-discovery/index.js'
   );
 
-  const searchPaths = getPluginSearchPaths({
-    rootDir: REPO_ROOT,
-    examples: true,
-    oss: false,
-    testPlugins: true,
+  const { Package } = await import(
+    // we need to run this before we install node modules, so it can't rely on @kbn/* imports
+    // eslint-disable-next-line @kbn/imports/uniform_imports
+    '../../../../packages/kbn-repo-packages/index.js'
+  );
+
+  const proc = await execAsync('git', ['ls-files', '-comt', '--exclude-standard'], {
+    cwd: REPO_ROOT,
+    encoding: 'utf8',
+    maxBuffer: Infinity,
   });
 
-  return simpleKibanaPlatformPluginDiscovery(searchPaths, []);
+  const paths = new Map();
+  /** @type {Map<string, Set<string>>} */
+  const filesByName = new Map();
+
+  for (const raw of proc.stdout.split('\n')) {
+    const line = raw.trim();
+    if (!line) {
+      continue;
+    }
+
+    const repoRel = line.slice(2); // trim the single char status and separating space from the line
+    const name = repoRel.split('/').pop();
+    if (name !== 'kibana.jsonc' && name !== 'tsconfig.json') {
+      continue;
+    }
+
+    const existingPath = paths.get(repoRel);
+    const path = existingPath ?? Path.resolve(REPO_ROOT, repoRel);
+    if (!existingPath) {
+      paths.set(repoRel, path);
+    }
+
+    let files = filesByName.get(name);
+    if (!files) {
+      files = new Set();
+      filesByName.set(name, files);
+    }
+
+    if (line.startsWith('C ')) {
+      // this line indicates that the previous path is changed in the working
+      // tree, so we need to determine if it was deleted and remove it if so
+      if (!Fs.existsSync(path)) {
+        files.delete(path);
+      }
+    } else {
+      files.add(path);
+    }
+  }
+
+  return {
+    plugins: simpleKibanaPlatformPluginDiscovery(
+      getPluginSearchPaths({
+        rootDir: REPO_ROOT,
+        examples: true,
+        oss: false,
+        testPlugins: true,
+      }),
+      []
+    ),
+    tsConfigsPaths: Array.from(filesByName.get('tsconfig.json') ?? new Set()),
+    packages: Array.from(filesByName.get('kibana.jsonc') ?? new Set())
+      .map((path) => Package.fromManifest(REPO_ROOT, path))
+      .sort((a, b) => a.id.localeCompare(b.id)),
+  };
 }

kbn_pm/src/commands/bootstrap/regenerate_base_tsconfig.mjs

@@ -13,7 +13,7 @@ import { REPO_ROOT } from '../../lib/paths.mjs';
 import External from '../../lib/external_packages.js';
 
 export async function regenerateBaseTsconfig() {
-  const pkgMap = External['@kbn/package-map']().readPackageMap();
+  const pkgMap = External['@kbn/repo-packages']().readPackageMap();
   const tsconfigPath = Path.resolve(REPO_ROOT, 'tsconfig.base.json');
   const lines = (await Fsp.readFile(tsconfigPath, 'utf-8')).split('\n');
 

kbn_pm/src/commands/bootstrap/regenerate_package_map.mjs

@@ -16,18 +16,12 @@ import { REPO_ROOT } from '../../lib/paths.mjs';
 
 /**
  *
- * @param {import('@kbn/bazel-packages').BazelPackage[]} packages
+ * @param {import('@kbn/repo-packages').Package[]} packages
  * @param {import('@kbn/plugin-discovery').KibanaPlatformPlugin[]} plugins
  * @param {import('@kbn/some-dev-log').SomeDevLog} log
  */
 export async function regeneratePackageMap(packages, plugins, log) {
-  // clean up old version of package map package
-  Fs.rmSync(Path.resolve(REPO_ROOT, 'packages/kbn-synthetic-package-map'), {
-    recursive: true,
-    force: true,
-  });
-
-  const path = Path.resolve(REPO_ROOT, 'packages/kbn-package-map/package-map.json');
+  const path = Path.resolve(REPO_ROOT, 'packages/kbn-repo-packages/package-map.json');
   const existingContent = Fs.existsSync(path) ? await Fsp.readFile(path, 'utf8') : undefined;
 
   /** @type {Array<[string, string]>} */
@@ -52,6 +46,6 @@ export async function regeneratePackageMap(packages, plugins, log) {
 
   if (content !== existingContent) {
     await Fsp.writeFile(path, content);
-    log.warning('updated package map, many caches may be invalidated');
+    log.warning('updated package map');
   }
 }

kbn_pm/src/commands/bootstrap/regenerate_tsconfig_paths.mjs

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+import Fsp from 'fs/promises';
+
+import { REPO_ROOT } from '../../lib/paths.mjs';
+
+/**
+ * @param {string[]} tsconfigPaths
+ * @param {import('@kbn/some-dev-log').SomeDevLog} log
+ */
+export async function regenerateTsconfigPaths(tsconfigPaths, log) {
+  const path = Path.resolve(REPO_ROOT, 'packages/kbn-ts-projects/config-paths.json');
+  const existingContent = Fs.existsSync(path) ? await Fsp.readFile(path, 'utf8') : undefined;
+
+  const entries = [...tsconfigPaths]
+    .map((abs) => Path.relative(REPO_ROOT, abs))
+    .sort((a, b) => a.localeCompare(b));
+
+  const content = JSON.stringify(entries, null, 2);
+  if (content !== existingContent) {
+    await Fsp.writeFile(path, content);
+    log.warning('updated tsconfig.json paths');
+  }
+}

kbn_pm/src/commands/bootstrap/sort_package_json.mjs

@@ -7,15 +7,23 @@
  */
 
 import Path from 'path';
-import Fs from 'fs';
+import Fsp from 'fs/promises';
 
 import { REPO_ROOT } from '../../lib/paths.mjs';
 import External from '../../lib/external_packages.js';
 
-export async function sortPackageJson() {
+/**
+ *
+ * @param {import('@kbn/some-dev-log').SomeDevLog} log
+ */
+export async function sortPackageJson(log) {
   const { sortPackageJson } = External['@kbn/sort-package-json']();
 
   const path = Path.resolve(REPO_ROOT, 'package.json');
-  const json = Fs.readFileSync(path, 'utf8');
-  Fs.writeFileSync(path, sortPackageJson(json));
+  const json = await Fsp.readFile(path, 'utf8');
+  const sorted = sortPackageJson(json);
+  if (sorted !== json) {
+    await Fsp.writeFile(path, sorted, 'utf8');
+    log.success('sorted package.json');
+  }
 }

kbn_pm/src/commands/bootstrap/update_package_json.mjs

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fsp from 'fs/promises';
+import Path from 'path';
+
+import { REPO_ROOT } from '../../lib/paths.mjs';
+
+/**
+ * @param {import('@kbn/repo-info').KibanaPackageJson['dependencies']} depsObj
+ * @param {Map<string, string>} actual
+ * @param {Map<string, string>} expected
+ */
+function updatePkgEntries(depsObj, actual, expected) {
+  let changes = false;
+  const keys = new Set([...actual.keys(), ...expected.keys()]);
+  for (const key of keys) {
+    const a = actual.get(key);
+    const e = expected.get(key);
+
+    // if expected and actual match then we don't need to do anything
+    if (a === e) {
+      continue;
+    }
+
+    changes = true;
+
+    // if expected is undefined then this key shouldn't be set
+    if (e === undefined) {
+      delete depsObj[key];
+      continue;
+    }
+
+    // otherwise we just need to update/add this key/value
+    depsObj[key] = e;
+  }
+  return changes;
+}
+
+/**
+ * @param {import('@kbn/repo-packages').Package[]} pkgs
+ * @param {import('@kbn/some-dev-log').SomeDevLog} log
+ */
+export async function updatePackageJson(pkgs, log) {
+  const path = Path.resolve(REPO_ROOT, 'package.json');
+  /** @type {import('@kbn/repo-info').KibanaPackageJson} */
+  const pkgJson = JSON.parse(await Fsp.readFile(path, 'utf8'));
+
+  let changes = false;
+  const typesInProd = Object.keys(pkgJson.dependencies).filter((id) => id.startsWith('@types/'));
+  for (const t of typesInProd) {
+    changes = true;
+    pkgJson.devDependencies[t] = pkgJson.dependencies[t];
+    delete pkgJson.dependencies[t];
+  }
+
+  changes ||= updatePkgEntries(
+    pkgJson.dependencies,
+    new Map(Object.entries(pkgJson.dependencies).filter(([k]) => k.startsWith('@kbn/'))),
+    new Map(
+      pkgs
+        .filter((p) => !p.isDevOnly)
+        .map((p) => [p.manifest.id, `link:${p.normalizedRepoRelativeDir}`])
+    )
+  );
+
+  changes ||= updatePkgEntries(
+    pkgJson.devDependencies,
+    new Map(Object.entries(pkgJson.devDependencies).filter(([k]) => k.startsWith('@kbn/'))),
+    new Map(
+      pkgs
+        .filter((p) => p.isDevOnly)
+        .map((p) => [p.manifest.id, `link:${p.normalizedRepoRelativeDir}`])
+    )
+  );
+
+  if (changes) {
+    await Fsp.writeFile(path, JSON.stringify(pkgJson, null, 2));
+    log.warning('updated package.json');
+  }
+}

kbn_pm/src/commands/run_in_packages_command.mjs

@@ -40,8 +40,8 @@ export const command = {
     const exclude = args.getStringValues('exclude') ?? [];
     const include = args.getStringValues('include') ?? [];
 
-    const { discoverBazelPackages } = External['@kbn/bazel-packages']();
-    const packages = await discoverBazelPackages(REPO_ROOT);
+    const { getPackages } = External['@kbn/repo-packages']();
+    const packages = getPackages(REPO_ROOT);
     for (const { manifest, pkg, normalizedRepoRelativeDir } of packages) {
       if (
         exclude.includes(manifest.id) ||

kbn_pm/src/lib/external_packages.js

@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 module.exports = {
-  ['@kbn/bazel-packages']() {
+  ['@kbn/repo-packages']() {
     require('@kbn/babel-register').install();
-    return require('@kbn/bazel-packages');
+    return require('@kbn/repo-packages');
   },
 
   ['@kbn/ci-stats-reporter']() {
@@ -26,11 +26,6 @@ module.exports = {
     return require('@kbn/sort-package-json');
   },
 
-  ['@kbn/package-map']() {
-    require('@kbn/babel-register').install();
-    return require('@kbn/package-map');
-  },
-
   ['@kbn/get-repo-files']() {
     require('@kbn/babel-register').install();
     return require('@kbn/get-repo-files');

kbn_pm/src/lib/find_clean_paths.mjs

@@ -16,11 +16,11 @@ import External from './external_packages.js';
  * Attempt to load the package map, if bootstrap hasn't run successfully
  * this might fail.
  * @param {import('@kbn/some-dev-log').SomeDevLog} log
- * @returns {Promise<import('@kbn/package-map').PackageMap>}
+ * @returns {Promise<import('@kbn/repo-packages').PackageMap>}
  */
 async function tryToGetPackageMap(log) {
   try {
-    const { readPackageMap } = External['@kbn/package-map']();
+    const { readPackageMap } = External['@kbn/repo-packages']();
     return readPackageMap();
   } catch (error) {
     log.warning('unable to load package map, unable to clean target directories in packages');

kbn_pm/tsconfig.json

@@ -15,14 +15,13 @@
   ],
   "kbn_references": [
     "@kbn/babel-register",
-    "@kbn/bazel-packages",
     "@kbn/repo-info",
     "@kbn/yarn-lock-validator",
     "@kbn/get-repo-files",
     "@kbn/sort-package-json",
     { "path": "../src/dev/tsconfig.json" },
     "@kbn/ci-stats-reporter",
-    "@kbn/package-map",
-    "@kbn/ts-projects"
+    "@kbn/ts-projects",
+    "@kbn/repo-packages"
   ]
 }