[spacetime] Improve scripts/build_api_docs stats output speed. (#157129)

The existing `build_api_docs` script takes about ~10m to run and
generate the API docs. That's not too bad for the daily CI task that
generates the docs. However, even if you're just interested in getting
some stats on a single plugin/package it runs the full script. We often
have to run this script locally to check the stats in detail to improve
JSDoc and missing exports which is quite cumbersome if it takes this
long to run after each code change.

This PR updates the script in the following way: If both the options
`--stats` and `--plugin` (with a single plugin/package) are set, it will
not generate a TS project for the whole Kibana codebase but just the
code related to that plugin/package. API docs will then not be written
to avoid inconsistencies and just the stats will be logged.

Depending on the size of the package/plugin this can reduce the time
needed to run the script down to a few seconds.

packages/kbn-docs-utils/src/build_api_docs_cli.ts

@@ -10,11 +10,14 @@ import Fs from 'fs';
 import Fsp from 'fs/promises';
 import Path from 'path';
 
+import apm, { type Transaction } from 'elastic-apm-node';
+import { Project } from 'ts-morph';
+
 import { run } from '@kbn/dev-cli-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
 import { CiStatsReporter } from '@kbn/ci-stats-reporter';
 import { REPO_ROOT } from '@kbn/repo-info';
-import { Project } from 'ts-morph';
+import { initApm } from '@kbn/apm-config-loader';
 
 import { writePluginDocs } from './mdx/write_plugin_mdx_docs';
 import { ApiDeclaration, ApiStats, PluginMetaInfo } from './types';
@@ -35,9 +38,23 @@ function isStringArray(arr: unknown | string[]): arr is string[] {
   return Array.isArray(arr) && arr.every((p) => typeof p === 'string');
 }
 
+const rootDir = Path.join(__dirname, '../../..');
+initApm(process.argv, rootDir, false, 'build_api_docs_cli');
+
+async function endTransactionWithFailure(transaction: Transaction | null) {
+  if (transaction !== null) {
+    transaction.setOutcome('failure');
+    transaction.end();
+    await apm.flush();
+  }
+}
+
 export function runBuildApiDocsCli() {
   run(
     async ({ log, flags }) => {
+      const transaction = apm.startTransaction('build-api-docs', 'kibana-cli');
+      const spanSetup = transaction?.startSpan('build_api_docs.setup', 'setup');
+
       const collectReferences = flags.references as boolean;
       const stats = flags.stats && typeof flags.stats === 'string' ? [flags.stats] : flags.stats;
       const pluginFilter =
@@ -46,6 +63,7 @@ export function runBuildApiDocsCli() {
           : (flags.plugin as string[] | undefined);
 
       if (pluginFilter && !isStringArray(pluginFilter)) {
+        await endTransactionWithFailure(transaction);
         throw createFlagError('expected --plugin must only contain strings');
       }
 
@@ -55,6 +73,7 @@ export function runBuildApiDocsCli() {
           stats.find((s) => s !== 'any' && s !== 'comments' && s !== 'exports')) ||
         (stats && !isStringArray(stats))
       ) {
+        await endTransactionWithFailure(transaction);
         throw createFlagError(
           'expected --stats must only contain `any`, `comments` and/or `exports`'
         );
@@ -62,14 +81,45 @@ export function runBuildApiDocsCli() {
 
       const outputFolder = Path.resolve(REPO_ROOT, 'api_docs');
 
+      spanSetup?.end();
+      const spanInitialDocIds = transaction?.startSpan('build_api_docs.initialDocIds', 'setup');
+
       const initialDocIds =
         !pluginFilter && Fs.existsSync(outputFolder)
           ? await getAllDocFileIds(outputFolder)
           : undefined;
 
-      const project = getTsProject(REPO_ROOT);
+      spanInitialDocIds?.end();
+      const spanPlugins = transaction?.startSpan('build_api_docs.findPlugins', 'setup');
 
-      const plugins = findPlugins();
+      const plugins = findPlugins(stats && pluginFilter ? pluginFilter : undefined);
+
+      if (stats && Array.isArray(pluginFilter) && pluginFilter.length !== plugins.length) {
+        await endTransactionWithFailure(transaction);
+        throw createFlagError('expected --plugin was not found');
+      }
+
+      spanPlugins?.end();
+
+      const spanPathsByPackage = transaction?.startSpan(
+        'build_api_docs.getPathsByPackage',
+        'setup'
+      );
+
+      const pathsByPlugin = await getPathsByPackage(plugins);
+
+      spanPathsByPackage?.end();
+
+      const spanProject = transaction?.startSpan('build_api_docs.getTsProject', 'setup');
+
+      const project = getTsProject(
+        REPO_ROOT,
+        stats && pluginFilter && plugins.length === 1 ? plugins[0].directory : undefined
+      );
+
+      spanProject?.end();
+
+      const spanFolders = transaction?.startSpan('build_api_docs.check-folders', 'setup');
 
       // if the output folder already exists, and we don't have a plugin filter, delete all the files in the output folder
       if (Fs.existsSync(outputFolder) && !pluginFilter) {
@@ -81,6 +131,9 @@ export function runBuildApiDocsCli() {
         await Fsp.mkdir(outputFolder, { recursive: true });
       }
 
+      spanFolders?.end();
+      const spanPluginApiMap = transaction?.startSpan('build_api_docs.getPluginApiMap', 'setup');
+
       const {
         pluginApiMap,
         missingApiItems,
@@ -89,12 +142,23 @@ export function runBuildApiDocsCli() {
         adoptionTrackedAPIs,
       } = getPluginApiMap(project, plugins, log, { collectReferences, pluginFilter });
 
+      spanPluginApiMap?.end();
+
       const reporter = CiStatsReporter.fromEnv(log);
-      const pathsByPlugin = await getPathsByPackage(plugins);
 
       const allPluginStats: { [key: string]: PluginMetaInfo & ApiStats & EslintDisableCounts } = {};
       for (const plugin of plugins) {
         const id = plugin.id;
+
+        if (stats && pluginFilter && !pluginFilter.includes(plugin.id)) {
+          continue;
+        }
+
+        const spanApiStatsForPlugin = transaction?.startSpan(
+          `build_api_docs.collectApiStatsForPlugin-${id}`,
+          'stats'
+        );
+
         const pluginApi = pluginApiMap[id];
         const paths = pathsByPlugin.get(plugin) ?? [];
 
@@ -110,9 +174,20 @@ export function runBuildApiDocsCli() {
           description: plugin.manifest.description,
           isPlugin: plugin.isPlugin,
         };
+
+        spanApiStatsForPlugin?.end();
       }
 
-      await writePluginDirectoryDoc(outputFolder, pluginApiMap, allPluginStats, log);
+      if (!stats) {
+        const spanWritePluginDirectoryDoc = transaction?.startSpan(
+          'build_api_docs.writePluginDirectoryDoc',
+          'write'
+        );
+
+        await writePluginDirectoryDoc(outputFolder, pluginApiMap, allPluginStats, log);
+
+        spanWritePluginDirectoryDoc?.end();
+      }
 
       for (const plugin of plugins) {
         // Note that the filtering is done here, and not above because the entire public plugin API has to
@@ -127,6 +202,11 @@ export function runBuildApiDocsCli() {
         const pluginStats = allPluginStats[id];
         const pluginTeam = plugin.manifest.owner.name;
 
+        const spanMetrics = transaction?.startSpan(
+          `build_api_docs.collectApiStatsForPlugin-${id}`,
+          'stats'
+        );
+
         reporter.metrics([
           {
             id,
@@ -283,20 +363,56 @@ export function runBuildApiDocsCli() {
           }
         }
 
-        if (pluginStats.apiCount > 0) {
-          log.info(`Writing public API doc for plugin ${pluginApi.id}.`);
-          await writePluginDocs(outputFolder, { doc: pluginApi, plugin, pluginStats, log });
-        } else {
-          log.info(`Plugin ${pluginApi.id} has no public API.`);
+        spanMetrics?.end();
+
+        if (!stats) {
+          if (pluginStats.apiCount > 0) {
+            log.info(`Writing public API doc for plugin ${pluginApi.id}.`);
+
+            const spanWritePluginDocs = transaction?.startSpan(
+              'build_api_docs.writePluginDocs',
+              'write'
+            );
+
+            await writePluginDocs(outputFolder, { doc: pluginApi, plugin, pluginStats, log });
+
+            spanWritePluginDocs?.end();
+          } else {
+            log.info(`Plugin ${pluginApi.id} has no public API.`);
+          }
+
+          const spanWriteDeprecationDocByPlugin = transaction?.startSpan(
+            'build_api_docs.writeDeprecationDocByPlugin',
+            'write'
+          );
+
+          await writeDeprecationDocByPlugin(outputFolder, referencedDeprecations, log);
+
+          spanWriteDeprecationDocByPlugin?.end();
+
+          const spanWriteDeprecationDueByTeam = transaction?.startSpan(
+            'build_api_docs.writeDeprecationDueByTeam',
+            'write'
+          );
+
+          await writeDeprecationDueByTeam(outputFolder, referencedDeprecations, plugins, log);
+
+          spanWriteDeprecationDueByTeam?.end();
+
+          const spanWriteDeprecationDocByApi = transaction?.startSpan(
+            'build_api_docs.writeDeprecationDocByApi',
+            'write'
+          );
+
+          await writeDeprecationDocByApi(
+            outputFolder,
+            referencedDeprecations,
+            unreferencedDeprecations,
+            log
+          );
+
+          spanWriteDeprecationDocByApi?.end();
         }
-        await writeDeprecationDocByPlugin(outputFolder, referencedDeprecations, log);
-        await writeDeprecationDueByTeam(outputFolder, referencedDeprecations, plugins, log);
-        await writeDeprecationDocByApi(
-          outputFolder,
-          referencedDeprecations,
-          unreferencedDeprecations,
-          log
-        );
       }
 
       if (Object.values(pathsOutsideScopes).length > 0) {
@@ -307,6 +423,8 @@ export function runBuildApiDocsCli() {
       if (initialDocIds) {
         await trimDeletedDocsFromNav(log, initialDocIds, outputFolder);
       }
+
+      transaction?.end();
     },
     {
       log: {
@@ -316,26 +434,36 @@ export function runBuildApiDocsCli() {
         string: ['plugin', 'stats'],
         boolean: ['references'],
         help: `
-          --plugin             Optionally, run for only a specific plugin
-          --stats              Optionally print API stats. Must be one or more of: any, comments or exports.
-          --references         Collect references for API items
+          --plugin           Optionally, run for only a specific plugin
+          --stats            Optionally print API stats. Must be one or more of: any, comments or exports.
+                             In combination with a single plugin filter this option will skip writing any
+                             API docs as a tradeoff to just produce the stats output more quickly.
+          --references       Collect references for API items
         `,
       },
     }
   );
 }
 
-function getTsProject(repoPath: string) {
-  const xpackTsConfig = `${repoPath}/tsconfig.json`;
+function getTsProject(repoPath: string, overridePath?: string) {
+  const xpackTsConfig = !overridePath
+    ? `${repoPath}/tsconfig.json`
+    : `${overridePath}/tsconfig.json`;
+
   const project = new Project({
     tsConfigFilePath: xpackTsConfig,
     // We'll use the files added below instead.
     skipAddingFilesFromTsConfig: true,
   });
-  project.addSourceFilesAtPaths([`${repoPath}/x-pack/plugins/**/*.ts`, '!**/*.d.ts']);
-  project.addSourceFilesAtPaths([`${repoPath}/x-pack/packages/**/*.ts`, '!**/*.d.ts']);
-  project.addSourceFilesAtPaths([`${repoPath}/src/plugins/**/*.ts`, '!**/*.d.ts']);
-  project.addSourceFilesAtPaths([`${repoPath}/packages/**/*.ts`, '!**/*.d.ts']);
+
+  if (!overridePath) {
+    project.addSourceFilesAtPaths([`${repoPath}/x-pack/plugins/**/*.ts`, '!**/*.d.ts']);
+    project.addSourceFilesAtPaths([`${repoPath}/x-pack/packages/**/*.ts`, '!**/*.d.ts']);
+    project.addSourceFilesAtPaths([`${repoPath}/src/plugins/**/*.ts`, '!**/*.d.ts']);
+    project.addSourceFilesAtPaths([`${repoPath}/packages/**/*.ts`, '!**/*.d.ts']);
+  } else {
+    project.addSourceFilesAtPaths([`${overridePath}/**/*.ts`, '!**/*.d.ts']);
+  }
   project.resolveSourceFileDependencies();
   return project;
 }

packages/kbn-docs-utils/src/find_plugins.ts

@@ -52,7 +52,7 @@ function toPluginOrPackage(pkg: Package): PluginOrPackage {
   };
 }
 
-export function findPlugins(): PluginOrPackage[] {
+export function findPlugins(pluginOrPackageFilter?: string[]): PluginOrPackage[] {
   const packages = getPackages(REPO_ROOT);
   const plugins = packages.filter(
     getPluginPackagesFilter({
@@ -66,14 +66,30 @@ export function findPlugins(): PluginOrPackage[] {
     throw new Error('unable to find @kbn/core');
   }
 
-  return [...[core, ...plugins].map(toPluginOrPackage), ...findPackages()];
+  if (!pluginOrPackageFilter) {
+    return [...[core, ...plugins].map(toPluginOrPackage), ...findPackages()];
+  } else {
+    return [
+      ...plugins
+        .filter((p) => pluginOrPackageFilter.includes(p.manifest.plugin.id))
+        .map(toPluginOrPackage),
+      ...findPackages(pluginOrPackageFilter),
+    ];
+  }
 }
 
 /**
  * Helper to find packages.
  */
-export function findPackages(): PluginOrPackage[] {
+export function findPackages(packageFilter?: string[]): PluginOrPackage[] {
   return getPackages(REPO_ROOT)
     .filter((p) => !p.isPlugin())
+    .filter((p) => {
+      if (!Array.isArray(packageFilter)) {
+        return true;
+      } else {
+        return packageFilter.includes(p.manifest.id);
+      }
+    })
     .map(toPluginOrPackage);
 }

packages/kbn-docs-utils/tsconfig.json

@@ -23,5 +23,6 @@
     "@kbn/std",
     "@kbn/get-repo-files",
     "@kbn/repo-packages",
+    "@kbn/apm-config-loader",
   ]
 }