SK: Relocate Script v7.2 (#207081)

## Summary

* Added a few transforms to simplify package paths.
* Fixed typo causing `.mdx` files to not be processed when replacing
references.
* Added preliminary support for `--healthcheck` (to check for broken
references to files and links).

packages/kbn-relocate/constants.ts

@@ -24,7 +24,7 @@ export const EXTENSIONS = [
   'lock',
   'bazel',
   'md',
-  'mdz',
+  'mdx',
   'asciidoc',
   'sh',
   'snap',
@@ -42,6 +42,10 @@ export const EXCLUDED_FOLDERS = [
   './.es',
   './.git',
   // './.github',
+  './bazel-bin',
+  './bazel-kibana',
+  './bazel-out',
+  './bazel-testlogs',
   './.native_modules',
   './.node_binaries',
   './.vscode',
@@ -56,6 +60,8 @@ export const EXCLUDED_FOLDERS = [
   './trash',
 ];
 
+export const EXCLUDED_FOLDER_NAMES = ['target'];
+
 export const NO_GREP = EXCLUDED_FOLDERS.map((f) => `--exclude-dir "${f}"`).join(' ');
 
 // These two constants are singletons, used and updated throughout the process

packages/kbn-relocate/healthcheck.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ToolingLog } from '@kbn/tooling-log';
+import fs from 'fs';
+import path, { join } from 'path';
+import { getPackages } from '@kbn/repo-packages';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { EXCLUDED_FOLDERS, EXCLUDED_FOLDER_NAMES, EXTENSIONS } from './constants';
+import { BASE_FOLDER } from './constants';
+
+const findPaths = (content: string): string[] => {
+  const regex = /([\.]{1,2}(\/[^\s)\]\['`#"]+)+)/g;
+  return content.match(regex) || [];
+};
+
+const findUrls = (content: string): string[] => {
+  const regex = /http(s)?:\/\/([^\s)\]\['`#"]+)/g;
+  return content.match(regex) || [];
+};
+
+const checkUrlExists = async (url: string, tries = 3): Promise<boolean> => {
+  try {
+    const response = await fetch(url, { method: 'GET' });
+    return response.ok;
+  } catch (error) {
+    return tries > 0 ? checkUrlExists(url, tries - 1) : false;
+  }
+};
+
+const isModuleReference = (moduleNames: string[], reference: string): boolean => {
+  return (
+    reference.includes('/packages/') || reference.includes('/plugins/') // ||
+    // moduleNames.some((name) => reference.includes(name))
+  );
+};
+
+const checkIfResourceExists = (baseDir: string, reference: string): boolean => {
+  const filePath = join(baseDir, reference);
+  const rootReference = join(BASE_FOLDER, reference);
+  const fatherReference = join(baseDir, '..', reference);
+
+  return (
+    filePath.includes('/target') || // ignore target folders
+    filePath.includes('{') || // ignore paths with variables
+    filePath.includes('(') || // ignore paths with regexps / vars
+    filePath.includes('*') || // assume wildcard patterns exist
+    fs.existsSync(filePath) ||
+    fs.existsSync(`${filePath}.ts`) ||
+    fs.existsSync(`${filePath}.tsx`) ||
+    fs.existsSync(`${filePath}.d.ts`) ||
+    fs.existsSync(`${filePath}.js`) ||
+    fs.existsSync(`${filePath}/index.ts`) ||
+    fs.existsSync(`${filePath}/index.js`) ||
+    fs.existsSync(rootReference) ||
+    fs.existsSync(`${rootReference}.js`) ||
+    fs.existsSync(fatherReference)
+  );
+};
+
+const getAllFiles = (
+  dirPath: string,
+  arrayOfFiles: fs.Dirent[] = [],
+  extensions?: string[]
+): fs.Dirent[] => {
+  const files = fs.readdirSync(dirPath, { withFileTypes: true });
+
+  files.forEach((file) => {
+    const filePath = path.join(dirPath, file.name);
+    if (
+      !EXCLUDED_FOLDERS.some((folder) => filePath.startsWith(join(BASE_FOLDER, folder))) &&
+      !EXCLUDED_FOLDER_NAMES.includes(file.name)
+    ) {
+      if (fs.statSync(filePath).isDirectory()) {
+        arrayOfFiles = getAllFiles(filePath, arrayOfFiles);
+      } else {
+        if (!extensions || extensions.find((ext) => file.name.endsWith(ext))) {
+          arrayOfFiles.push(file);
+        }
+      }
+    }
+  });
+
+  return arrayOfFiles;
+};
+
+export const findBrokenReferences = async (log: ToolingLog) => {
+  const packages = getPackages(REPO_ROOT);
+  const moduleNames = packages.map((pkg) => pkg.directory.split('/').pop()!);
+  const files = getAllFiles(BASE_FOLDER, [], EXTENSIONS);
+
+  for (const file of files) {
+    const fileBrokenReferences = [];
+    const filePath = join(file.path, file.name);
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const references = findPaths(content);
+
+    for (const ref of references) {
+      if (isModuleReference(moduleNames, ref) && !checkIfResourceExists(file.path, ref)) {
+        fileBrokenReferences.push(ref);
+      }
+    }
+
+    if (fileBrokenReferences.length > 0) {
+      log.info(filePath, fileBrokenReferences);
+    }
+  }
+};
+
+export const findBrokenLinks = async (log: ToolingLog) => {
+  const files = getAllFiles(BASE_FOLDER);
+
+  for (const file of files) {
+    const fileBrokenLinks = [];
+    const filePath = join(file.path, file.name);
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const references = findUrls(content);
+
+    for (const ref of references) {
+      if (
+        ref.includes('github.com/elastic/kibana') &&
+        ref.includes('blob') &&
+        !(await checkUrlExists(ref))
+      ) {
+        fileBrokenLinks.push(ref);
+      }
+    }
+
+    if (fileBrokenLinks.length > 0) {
+      log.info(filePath, fileBrokenLinks);
+    }
+  }
+};

packages/kbn-relocate/index.ts

@@ -10,6 +10,7 @@
 import { run } from '@kbn/dev-cli-runner';
 import { findAndRelocateModules, findAndMoveModule } from './relocate';
 import { listModules } from './list';
+import { findBrokenLinks, findBrokenReferences } from './healthcheck';
 
 const toStringArray = (flag: string | boolean | string[] | undefined): string[] => {
   if (typeof flag === 'string') {
@@ -42,11 +43,19 @@ const toOptString = (
 export const runKbnRelocateCli = () => {
   run(
     async ({ log, flags }) => {
-      if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) {
+      if (typeof flags.list === 'string' && flags.list.length > 0) {
+        await listModules(flags.list, log);
+      } else if (typeof flags.healthcheck === 'string' && flags.healthcheck.length > 0) {
+        if (flags.healthcheck === 'references') {
+          await findBrokenReferences(log);
+        } else if (flags.healthcheck === 'links') {
+          await findBrokenLinks(log);
+        } else {
+          log.error(`Unknown --healthcheck option: ${flags.healthcheck}`);
+        }
+      } else if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) {
         log.info('When using --moveOnly flag, the rest of flags are ignored.');
         await findAndMoveModule(flags.moveOnly, log);
-      } else if (typeof flags.list === 'string' && flags.list.length > 0) {
-        await listModules(flags.list, log);
       } else {
         const { pr, team, path, include, exclude, baseBranch } = flags;
         await findAndRelocateModules(
@@ -67,10 +76,25 @@ export const runKbnRelocateCli = () => {
         defaultLevel: 'info',
       },
       flags: {
-        string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly', 'list'],
+        string: [
+          'healthcheck',
+          'pr',
+          'team',
+          'path',
+          'include',
+          'exclude',
+          'baseBranch',
+          'moveOnly',
+          'list',
+        ],
         help: `
           Usage: node scripts/relocate [options]
 
+          --list "all" List all Kibana modules
+          --list "uncategorised" List Kibana modules that are lacking 'group' or 'visibility' information
+          --list "incorrect" List Kibana modules that are not in the correct folder (aka folder does not match group/visibility in the manifest)
+          --healthcheck "references" Find broken references in Kibana codebase (Beta)
+          --healthcheck "links" Find broken links in Kibana codebase (Beta)
           --moveOnly <moduleId> Only move the specified module in the current branch (no cleanup, no branching, no commit)
           --pr <number> Use the given PR number instead of creating a new one
           --team <owner> Include all modules (packages and plugins) belonging to the specified owner (can specify multiple teams)
@@ -78,9 +102,6 @@ export const runKbnRelocateCli = () => {
           --include <id> Include the specified module in the relocation (can specify multiple modules)
           --exclude <id> Exclude the specified module from the relocation (can use multiple times)
           --baseBranch <name> Use a branch different than 'main' (e.g. "8.x")
-          --list "all" List all Kibana modules
-          --list "uncategorised" List Kibana modules that are lacking 'group' or 'visibility' information
-          --list "incorrect" List Kibana modules that are not in the correct folder (aka folder does not match group/visibility in the manifest)
 
           E.g. relocate all modules owned by Core team and also modules owned by Operations team, excluding 'foo-module-id'. Force push into PR 239847:
           node scripts/relocate --pr 239847 --team @elastic/kibana-core --team @elastic/kibana-operations --exclude @kbn/foo-module-id

packages/kbn-relocate/utils/transforms.ts

@@ -13,7 +13,10 @@ type TransformFunction = (param: string) => string;
 const TRANSFORMS: Record<string, string | TransformFunction> = {
   'src/platform/packages/shared/chart_expressions/common':
     'src/platform/packages/shared/chart-expressions-common',
+  'x-pack/solutions/search/packages/search/shared_ui': 'x-pack/solutions/search/packages/shared_ui',
   'x-pack/solutions/security/packages/security-solution/': 'x-pack/solutions/security/packages/',
+  'x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant':
+    'x-pack/platform/plugins/shared/observability_ai_assistant',
   'x-pack/solutions/observability/plugins/observability_solution/':
     'x-pack/solutions/observability/plugins/',
   'x-pack/solutions/observability/packages/observability/observability_utils/observability_':