[api-docs] automatically strip deleted doc files from the nav (#139878)

packages/kbn-docs-utils/BUILD.bazel

@@ -49,10 +49,13 @@ TYPES_DEPS = [
   "//packages/kbn-plugin-discovery:npm_module_types",
   "//packages/kbn-dev-utils:npm_module_types",
   "//packages/kbn-utils:npm_module_types",
+  "//packages/kbn-tooling-log:npm_module_types",
   "@npm//ts-morph",
   "@npm//@types/dedent",
   "@npm//@types/jest",
+  "@npm//@types/js-yaml",
   "@npm//@types/node",
+  "@npm//globby",
 ]
 
 jsts_transpiler(

packages/kbn-docs-utils/src/api_docs/build_api_docs_cli.ts

@@ -7,6 +7,7 @@
  */
 
 import Fs from 'fs';
+import Fsp from 'fs/promises';
 import Path from 'path';
 
 import { run } from '@kbn/dev-cli-runner';
@@ -26,6 +27,8 @@ import { writePluginDirectoryDoc } from './mdx/write_plugin_directory_doc';
 import { collectApiStatsForPlugin } from './stats';
 import { countEslintDisableLine, EslintDisableCounts } from './count_eslint_disable';
 import { writeDeprecationDueByTeam } from './mdx/write_deprecations_due_by_team';
+import { trimDeletedDocsFromNav } from './trim_deleted_docs_from_nav';
+import { getAllDocFileIds } from './mdx/get_all_doc_file_ids';
 
 function isStringArray(arr: unknown | string[]): arr is string[] {
   return Array.isArray(arr) && arr.every((p) => typeof p === 'string');
@@ -53,30 +56,27 @@ export function runBuildApiDocsCli() {
         );
       }
 
+      const outputFolder = Path.resolve(REPO_ROOT, 'api_docs');
+
+      const initialDocIds =
+        !pluginFilter && Fs.existsSync(outputFolder)
+          ? await getAllDocFileIds(outputFolder)
+          : undefined;
+
       const project = getTsProject(REPO_ROOT);
 
       const plugins = findPlugins();
 
-      const outputFolder = Path.resolve(REPO_ROOT, 'api_docs');
-      if (!Fs.existsSync(outputFolder)) {
-        Fs.mkdirSync(outputFolder);
-
-        // Don't delete all the files if a plugin filter is being used.
-      } else if (!pluginFilter) {
-        // Delete all files except the README that warns about the auto-generated nature of
-        // the folder.
-        const files = Fs.readdirSync(outputFolder);
-        await Promise.all(
-          files
-            .filter((file) => file.indexOf('README.md') < 0)
-            .map(
-              (file) =>
-                new Promise<void>((resolve, reject) =>
-                  Fs.rm(Path.resolve(outputFolder, file), (err) => (err ? reject(err) : resolve()))
-                )
-            )
-        );
+      // if the output folder already exists and we don't have a plugin filter, delete all the files in the output folder
+      if (Fs.existsSync(outputFolder) && !pluginFilter) {
+        await Fsp.rm(outputFolder, { recursive: true });
       }
+
+      // if the output folder doesn't exist, create it
+      if (!Fs.existsSync(outputFolder)) {
+        await Fsp.mkdir(outputFolder, { recursive: true });
+      }
+
       const collectReferences = flags.references as boolean;
 
       const { pluginApiMap, missingApiItems, unreferencedDeprecations, referencedDeprecations } =
@@ -264,10 +264,15 @@ export function runBuildApiDocsCli() {
           log
         );
       });
+
       if (Object.values(pathsOutsideScopes).length > 0) {
         log.warning(`Found paths outside of normal scope folders:`);
         log.warning(pathsOutsideScopes);
       }
+
+      if (initialDocIds) {
+        await trimDeletedDocsFromNav(log, initialDocIds, outputFolder);
+      }
     },
     {
       log: {

packages/kbn-docs-utils/src/api_docs/mdx/get_all_doc_file_ids.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fsp from 'fs/promises';
+
+import globby from 'globby';
+import { asyncMapWithLimit } from '@kbn/std';
+import Yaml from 'js-yaml';
+
+const FM_SEP_RE = /^---$/m;
+
+export async function getAllDocFileIds(outputDir: string) {
+  const paths = await globby(['**/*.mdx'], {
+    cwd: outputDir,
+    absolute: true,
+    unique: true,
+  });
+
+  const ids = await asyncMapWithLimit(paths, 20, async (path) => {
+    const content = await Fsp.readFile(path, 'utf8');
+
+    const fmStart = content.match(FM_SEP_RE);
+    if (fmStart?.index === undefined) {
+      throw new Error(`unable to find start of frontmatter in ${path}`);
+    }
+    const fmYaml = content.slice(fmStart.index + fmStart[0].length);
+
+    const fmEnd = fmYaml.match(FM_SEP_RE);
+    if (fmEnd?.index === undefined) {
+      throw new Error(`unable to find end of frontmatter in ${path}`);
+    }
+
+    let fm;
+    try {
+      fm = Yaml.safeLoad(fmYaml.slice(0, fmEnd.index));
+      if (typeof fm !== 'object' || fm === null) {
+        throw new Error('expected yaml to produce an object');
+      }
+    } catch (err) {
+      throw new Error(`unable to parse frontmatter in ${path}: ${err.message}`);
+    }
+
+    const id = fm.id;
+    if (typeof id !== 'string') {
+      throw new Error(`missing "id" in frontmatter in ${path}`);
+    }
+
+    return id;
+  });
+
+  return ids.flat();
+}

packages/kbn-docs-utils/src/api_docs/trim_deleted_docs_from_nav.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fsp from 'fs/promises';
+
+import { REPO_ROOT } from '@kbn/utils';
+import { ToolingLog } from '@kbn/tooling-log';
+
+import { getAllDocFileIds } from './mdx/get_all_doc_file_ids';
+
+interface NavEntry {
+  id?: string;
+  items?: NavEntry[];
+}
+
+export async function trimDeletedDocsFromNav(
+  log: ToolingLog,
+  initialDocIds: string[],
+  outputDir: string
+) {
+  const generatedDocIds = await getAllDocFileIds(outputDir);
+  const deleted = initialDocIds.filter((id) => !generatedDocIds.includes(id));
+  if (!deleted.length) {
+    log.info('no deleted doc files detected');
+  }
+
+  const navPath = Path.resolve(REPO_ROOT, 'nav-kibana-dev.docnav.json');
+
+  let navJson;
+  try {
+    navJson = await Fsp.readFile(navPath, 'utf8');
+  } catch (error) {
+    throw new Error(`unable to read dev-docs nav at ${navPath}: ${error.message}`);
+  }
+
+  let nav;
+  try {
+    nav = JSON.parse(navJson);
+  } catch (error) {
+    throw new Error(`unable to parse nav at ${navPath}: ${error.message}`);
+  }
+
+  let updatedNav = false;
+  (function recurse(entry: NavEntry, parent?: NavEntry): void {
+    if (parent && typeof entry.id === 'string' && deleted.includes(entry.id)) {
+      updatedNav = true;
+      parent.items = parent.items?.filter((i) => i !== entry);
+      return;
+    }
+
+    if (entry.items) {
+      for (const item of entry.items) {
+        recurse(item, entry);
+      }
+    }
+  })(nav);
+
+  if (updatedNav) {
+    log.info('updating docs nav to remove references to deleted pages');
+    await Fsp.writeFile(
+      navPath,
+      JSON.stringify(nav, null, 2) + (navJson.endsWith('\n') ? '\n' : '')
+    );
+  }
+}