[tsProjects] auto-rebuild tsconfig-paths.json cache (#148743)

Updates the TsProject loading logic to automatically rebuild the
tsconfig-paths.json cache of tsconfig file locations which was
previously only updated by bootstrap. This means that people will no
longer be stopped when they are running tasks which didn't used to
strictly require a bootstrap, and told to bootstrap the repo. This is
specifically usable in this context because we are only populating the
cache in bootstrap because we are already scanning all the files in the
repo at that point. It's a quick operation (though one we ideally
wouldn't be executing constantly), so adding it here too just provides a
better DX.

Additionally, this PR adds a check to the start of the precommit hook
which will bootstrap the repo if the user just finished resolving a
merge. This puts the repo in a more correct state before running code to
validate code via eslint, etc.

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-dev-utils/src/precommit_hook/script_source.ts

@@ -18,8 +18,8 @@ export const SCRIPT_SOURCE = `#!/usr/bin/env bash
 # ** PLEASE DO NOT CHANGE IT MANUALLY **
 #
 # GENERATED BY \`node scripts/register_git_hook\`
-# IF YOU WANNA CHANGE SOMETHING IN THIS SCRIPT
-# PLEASE RE-RUN 'yarn kbn bootstrap' or 'node scripts/register_git_hook'
+# IF YOU CHANGE SOMETHING IN THIS SCRIPT IT WILL
+# BE REWRITTEN BY 'node scripts/register_git_hook'
 
 # pre-commit script takes zero arguments: https://git-scm.com/docs/githooks#_pre_commit
 

packages/kbn-get-repo-files/index.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { getRepoFiles } from './src/get_repo_files';
+export { getRepoFiles, getRepoFilesSync } from './src/get_repo_files';

packages/kbn-get-repo-files/src/get_repo_files.ts

@@ -13,28 +13,11 @@ import execa from 'execa';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { RepoPath } from '@kbn/repo-path';
 
-/**
- * List the files in the repo, only including files which are manged by version
- * control or "untracked" (new, not committed, and not ignored).
- * @param include limit the list to specfic absolute paths
- * @param exclude exclude specific absolute paths
- */
-export async function getRepoFiles(include?: string[], exclude?: string[]) {
-  const flags = [
-    include?.map((p) => Path.relative(REPO_ROOT, p)) ?? [],
-    exclude?.map((p) => `--exclude=${Path.relative(REPO_ROOT, p)}`) ?? [],
-  ].flat();
-
-  const proc = await execa('git', ['ls-files', '-comt', '--exclude-standard', ...flags], {
-    cwd: REPO_ROOT,
-    stdio: ['ignore', 'pipe', 'pipe'],
-    buffer: true,
-  });
-
+function parseLsFilesOutput(output: string) {
   const paths = new Map<string, RepoPath>();
   const files = new Set<RepoPath>();
 
-  for (const line of proc.stdout.split('\n').map((l) => l.trim())) {
+  for (const line of output.split('\n').map((l) => l.trim())) {
     if (!line) {
       continue;
     }
@@ -59,3 +42,45 @@ export async function getRepoFiles(include?: string[], exclude?: string[]) {
 
   return files;
 }
+
+function getGitFlags(include?: string[], exclude?: string[]) {
+  return [
+    'ls-files',
+    '-comt',
+    '--exclude-standard',
+    include?.map((p) => Path.relative(REPO_ROOT, p)) ?? [],
+    exclude?.map((p) => `--exclude=${Path.relative(REPO_ROOT, p)}`) ?? [],
+  ].flat();
+}
+
+/**
+ * List the files in the repo, only including files which are manged by version
+ * control or "untracked" (new, not committed, and not ignored).
+ * @param include limit the list to specfic absolute paths
+ * @param exclude exclude specific absolute paths
+ */
+export async function getRepoFiles(include?: string[], exclude?: string[]) {
+  const proc = await execa('git', getGitFlags(include, exclude), {
+    cwd: REPO_ROOT,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    buffer: true,
+  });
+
+  return parseLsFilesOutput(proc.stdout);
+}
+
+/**
+ * Synchronously list the files in the repo, only including files which are manged by version
+ * control or "untracked" (new, not committed, and not ignored).
+ * @param include limit the list to specfic absolute paths
+ * @param exclude exclude specific absolute paths
+ */
+export function getRepoFilesSync(include?: string[], exclude?: string[]) {
+  const proc = execa.sync('git', getGitFlags(include, exclude), {
+    cwd: REPO_ROOT,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    buffer: true,
+  });
+
+  return parseLsFilesOutput(proc.stdout);
+}

packages/kbn-ts-projects/ts_project.ts

@@ -12,6 +12,8 @@ import Fs from 'fs';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { makeMatcher } from '@kbn/picomatcher';
 import { findPackageInfoForPath } from '@kbn/repo-packages';
+import { createFailError } from '@kbn/dev-cli-errors';
+import { getRepoFilesSync } from '@kbn/get-repo-files';
 
 import { readTsConfig, parseTsConfig, TsConfig } from './ts_configfile';
 
@@ -56,7 +58,11 @@ function expand(name: string, patterns: string[], knownPaths: string[]) {
 }
 
 export class TsProject {
-  static loadAll(options: { ignore: string[]; disableTypeCheck: string[] }) {
+  static loadAll(options: {
+    ignore: string[];
+    disableTypeCheck: string[];
+    noTsconfigPathsRefresh?: boolean;
+  }): TsProject[] {
     const mapPath = Path.resolve(__dirname, 'config-paths.json');
     if (!Fs.existsSync(mapPath)) {
       throw new Error('missing config-paths.json file, make sure you run `yarn kbn bootstrap`');
@@ -68,23 +74,37 @@ export class TsProject {
     const disableTypeCheck = expand('disableTypeCheck', options.disableTypeCheck, tsConfigRepoRels);
 
     const cache = new Map();
-    const projects = tsConfigRepoRels.flatMap((repoRel) => {
+    const projects: TsProject[] = [];
+    for (const repoRel of tsConfigRepoRels) {
       if (ignores.has(repoRel)) {
-        return [];
+        continue;
       }
 
       const proj = TsProject.createFromCache(cache, Path.resolve(REPO_ROOT, repoRel), {
         disableTypeCheck: disableTypeCheck.has(repoRel),
       });
 
-      if (!proj) {
-        throw new Error(
-          `The tsconfig at ${repoRel} has been removed, please run "yarn kbn bootstrap" to update map of tsconfig files in the repository`
+      if (proj) {
+        projects.push(proj);
+        continue;
+      }
+
+      if (options.noTsconfigPathsRefresh) {
+        throw createFailError(
+          `Run "yarn kbn bootstrap" to update the tsconfig.json path cache. ${repoRel} no longer exists.`
         );
       }
 
-      return proj;
-    });
+      // rebuild the tsconfig.json path cache
+      const tsConfigPaths = Array.from(getRepoFilesSync()).flatMap((r) =>
+        r.basename === 'tsconfig.json' ? r.repoRel : []
+      );
+      Fs.writeFileSync(mapPath, JSON.stringify(tsConfigPaths, null, 2));
+      return TsProject.loadAll({
+        ...options,
+        noTsconfigPathsRefresh: true,
+      });
+    }
 
     return projects;
   }

packages/kbn-ts-projects/tsconfig.json

@@ -17,5 +17,7 @@
     "@kbn/repo-info",
     "@kbn/repo-packages",
     "@kbn/picomatcher",
+    "@kbn/dev-cli-errors",
+    "@kbn/get-repo-files",
   ]
 }