Sustainable Kibana Architecture: Relocate script v4 (#204383)

## Summary

* Auto-detect "upstream" and "origin" remotes (instead of assuming their
names).
* Allow relocating modules that are already in a "sustainable" folder.
  * Filter out modules that are in the correct locations.
* Update the list of _modules to relocate_ to show only those modules
that are actually moved.

---------

Co-authored-by: Alejandro Fernández Haro <afharo@gmail.com>

packages/kbn-relocate/README.md

@@ -8,12 +8,9 @@ You must have `gh` CLI tool installed. You can install it by running:
 
 ```sh
 brew install gh
+gh auth login
 ```
 
-You must have `elastic/kibana` remote configured under the name `upstream`.
-
-You must have a remote named `origin` pointing to your fork of the Kibana repo.
-
 ## Usage
 
 First of all, you need to decide whether you want to contribute to an existing PR or to create a new one. Use the `--pr` flag to specify the PR you are trying to update:

packages/kbn-relocate/constants.ts

@@ -13,13 +13,26 @@ export const BASE_FOLDER = process.cwd() + '/';
 export const BASE_FOLDER_DEPTH = process.cwd().split('/').length;
 export const KIBANA_FOLDER = process.cwd().split('/').pop()!;
 export const EXCLUDED_MODULES = ['@kbn/core'];
-export const TARGET_FOLDERS = [
-  'src/platform/plugins/',
-  'src/platform/packages/',
-  'x-pack/platform/plugins/',
-  'x-pack/platform/packages/',
-  'x-pack/solutions/',
-];
+export const TARGET_FOLDERS: Record<string, string[]> = {
+  'platform:private': [
+    'src/platform/packages/private/',
+    'src/platform/plugins/private/',
+    'x-pack/platform/packages/private/',
+    'x-pack/platform/plugins/private/',
+  ],
+  'platform:shared': [
+    'src/platform/packages/shared/',
+    'src/platform/plugins/shared/',
+    'x-pack/platform/packages/shared/',
+    'x-pack/platform/plugins/shared/',
+  ],
+  'observability:private': [
+    'x-pack/solutions/observability/packages/',
+    'x-pack/solutions/observability/plugins/',
+  ],
+  'search:private': ['x-pack/solutions/search/packages/', 'x-pack/solutions/search/plugins/'],
+  'security:private': ['x-pack/solutions/security/packages/', 'x-pack/solutions/security/plugins/'],
+};
 export const EXTENSIONS = [
   'eslintignore',
   'gitignore',

packages/kbn-relocate/index.ts

@@ -46,15 +46,17 @@ export const runKbnRelocateCli = () => {
         await findAndMoveModule(flags.moveOnly, log);
       } else {
         const { pr, team, path, include, exclude, baseBranch } = flags;
-        await findAndRelocateModules({
-          prNumber: toOptString('prNumber', pr),
-          baseBranch: toOptString('baseBranch', baseBranch, 'main')!,
-          teams: toStringArray(team),
-          paths: toStringArray(path),
-          included: toStringArray(include),
-          excluded: toStringArray(exclude),
-          log,
-        });
+        await findAndRelocateModules(
+          {
+            prNumber: toOptString('prNumber', pr),
+            baseBranch: toOptString('baseBranch', baseBranch, 'main')!,
+            teams: toStringArray(team),
+            paths: toStringArray(path),
+            included: toStringArray(include),
+            excluded: toStringArray(exclude),
+          },
+          log
+        );
       }
     },
     {

packages/kbn-relocate/relocate.ts

@@ -16,22 +16,17 @@ import type { ToolingLog } from '@kbn/tooling-log';
 import { getPackages } from '@kbn/repo-packages';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { Package } from './types';
-import {
-  DESCRIPTION,
-  EXCLUDED_MODULES,
-  KIBANA_FOLDER,
-  NEW_BRANCH,
-  TARGET_FOLDERS,
-} from './constants';
+import { DESCRIPTION, EXCLUDED_MODULES, KIBANA_FOLDER, NEW_BRANCH } from './constants';
 import {
   belongsTo,
   calculateModuleTargetFolder,
+  isInTargetFolder,
   replaceReferences,
   replaceRelativePaths,
-} from './utils.relocate';
-import { safeExec } from './utils.exec';
-import { relocatePlan, relocateSummary } from './utils.logging';
-import { checkoutBranch, checkoutResetPr } from './utils.git';
+} from './utils/relocate';
+import { safeExec } from './utils/exec';
+import { relocatePlan, relocateSummary } from './utils/logging';
+import { checkoutBranch, checkoutResetPr, findGithubLogin, findRemoteName } from './utils/git';
 
 const moveModule = async (module: Package, log: ToolingLog) => {
   const destination = calculateModuleTargetFolder(module);
@@ -52,11 +47,6 @@ const relocateModules = async (toMove: Package[], log: ToolingLog): Promise<numb
   for (let i = 0; i < toMove.length; ++i) {
     const module = toMove[i];
 
-    if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder))) {
-      log.warning(`The module ${module.id} is already in a "sustainable" folder. Skipping`);
-      // skip modules that are already moved
-      continue;
-    }
     log.info('');
     log.info('--------------------------------------------------------------------------------');
     log.info(`\t${module.id} (${i + 1} of ${toMove.length})`);
@@ -93,10 +83,9 @@ export interface RelocateModulesParams {
   paths: string[];
   included: string[];
   excluded: string[];
-  log: ToolingLog;
 }
 
-const findModules = ({ teams, paths, included, excluded }: FindModulesParams) => {
+const findModules = ({ teams, paths, included, excluded }: FindModulesParams, log: ToolingLog) => {
   // get all modules
   const modules = getPackages(REPO_ROOT);
 
@@ -123,13 +112,14 @@ const findModules = ({ teams, paths, included, excluded }: FindModulesParams) =>
           paths.some((path) => module.directory.includes(path))
       )
       // the module is not explicitly excluded
-      .filter(({ id }) => !excluded.includes(id)),
-    'id'
+      .filter(({ id }) => !excluded.includes(id))
+      // exclude modules that are in the correct folder
+      .filter((module) => !isInTargetFolder(module, log))
   );
 };
 
 export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => {
-  const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] });
+  const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] }, log);
   if (!modules.length) {
     log.warning(`Cannot move ${moduleId}, either not found or not allowed!`);
   } else {
@@ -137,10 +127,24 @@ export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => {
   }
 };
 
-export const findAndRelocateModules = async (params: RelocateModulesParams) => {
-  const { prNumber, log, baseBranch, ...findParams } = params;
+export const findAndRelocateModules = async (params: RelocateModulesParams, log: ToolingLog) => {
+  const upstream = await findRemoteName('elastic/kibana');
+  if (!upstream) {
+    log.error(
+      'This repository does not have a remote pointing to the elastic/kibana repository. Aborting'
+    );
+    return;
+  }
 
-  const toMove = findModules(findParams);
+  const origin = await findRemoteName(`${await findGithubLogin()}/kibana`);
+  if (!origin) {
+    log.error('This repository does not have a remote pointing to your Kibana fork. Aborting');
+    return;
+  }
+
+  const { prNumber, baseBranch, ...findParams } = params;
+
+  const toMove = findModules(findParams, log);
   if (!toMove.length) {
     log.info(
       `No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags`
@@ -164,7 +168,7 @@ export const findAndRelocateModules = async (params: RelocateModulesParams) => {
   await safeExec(`git restore --staged .`);
   await safeExec(`git restore .`);
   await safeExec(`git clean -f -d`);
-  await safeExec(`git checkout ${baseBranch} && git pull upstream ${baseBranch}`);
+  await safeExec(`git checkout ${baseBranch} && git pull ${upstream} ${baseBranch}`);
 
   if (prNumber) {
     // checkout existing PR, reset all commits, rebase from baseBranch
@@ -204,7 +208,7 @@ export const findAndRelocateModules = async (params: RelocateModulesParams) => {
 
   const pushCmd = prNumber
     ? `git push --force-with-lease`
-    : `git push --set-upstream origin ${NEW_BRANCH}`;
+    : `git push --set-upstream ${origin} ${NEW_BRANCH}`;
 
   if (!res2.pushBranch) {
     log.info(`Remember to push changes with "${pushCmd}"`);

packages/kbn-relocate/utils/git.ts

@@ -8,8 +8,26 @@
  */
 
 import inquirer from 'inquirer';
-import type { Commit, PullRequest } from './types';
-import { safeExec } from './utils.exec';
+import type { Commit, PullRequest } from '../types';
+import { safeExec } from './exec';
+
+export const findRemoteName = async (repo: string) => {
+  const res = await safeExec('git remote -v');
+  const remotes = res.stdout.split('\n').map((line) => line.split(/\t| /).filter(Boolean));
+  return remotes.find(([_, url]) => url.includes(`github.com/${repo}`))?.[0];
+};
+
+export const findGithubLogin = async () => {
+  const res = await safeExec('gh auth status');
+  // e.g. ✓ Logged in to github.com account gsoldevila (/Users/gsoldevila/.config/gh/hosts.yml)
+  const loginLine = res.stdout
+    .split('\n')
+    .find((line) => line.includes('Logged in'))
+    ?.split(/\t| /)
+    .filter(Boolean);
+
+  return loginLine?.[loginLine?.findIndex((fragment) => fragment === 'account') + 1];
+};
 
 export const findPr = async (number: string): Promise<PullRequest> => {
   const res = await safeExec(`gh pr view ${number} --json commits,headRefName`);

packages/kbn-relocate/utils/logging.ts

@@ -10,8 +10,8 @@
 import type { ToolingLog } from '@kbn/tooling-log';
 import { appendFileSync, writeFileSync } from 'fs';
 import dedent from 'dedent';
-import type { Package } from './types';
-import { calculateModuleTargetFolder } from './utils.relocate';
+import type { Package } from '../types';
+import { calculateModuleTargetFolder } from './relocate';
 import {
   BASE_FOLDER,
   DESCRIPTION,
@@ -19,7 +19,7 @@ import {
   SCRIPT_ERRORS,
   UPDATED_REFERENCES,
   UPDATED_RELATIVE_PATHS,
-} from './constants';
+} from '../constants';
 
 export const relocatePlan = (modules: Package[], log: ToolingLog) => {
   const plugins = modules.filter((module) => module.manifest.type === 'plugin');

packages/kbn-relocate/utils/relocate.ts

@@ -10,7 +10,7 @@
 import { join } from 'path';
 import type { ToolingLog } from '@kbn/tooling-log';
 import { orderBy } from 'lodash';
-import type { Package } from './types';
+import type { Package } from '../types';
 import { applyTransforms } from './transforms';
 import {
   BASE_FOLDER,
@@ -22,8 +22,8 @@ import {
   TARGET_FOLDERS,
   UPDATED_REFERENCES,
   UPDATED_RELATIVE_PATHS,
-} from './constants';
-import { quietExec, safeExec } from './utils.exec';
+} from '../constants';
+import { quietExec, safeExec } from './exec';
 
 export const belongsTo = (module: Package, owner: string): boolean => {
   return Array.from(module.manifest.owner)[0] === owner;
@@ -40,11 +40,18 @@ export const calculateModuleTargetFolder = (module: Package): string => {
   const isPlugin = module.manifest.type === 'plugin';
   const fullPath = join(BASE_FOLDER, module.directory);
   let moduleDelimiter = isPlugin ? '/plugins/' : '/packages/';
-  if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder)) && group === 'platform') {
-    // if a platform module has already been relocated, strip the /private/ or /shared/ part too
-    moduleDelimiter += `${module.visibility}/`;
+
+  // for platform modules that are in a sustainable folder, strip the /private/ or /shared/ part too
+  if (module.directory.includes(`${moduleDelimiter}private/`)) {
+    moduleDelimiter += 'private/';
+  } else if (module.directory.includes(`${moduleDelimiter}shared/`)) {
+    moduleDelimiter += 'shared/';
   }
-  const moduleFolder = fullPath.split(moduleDelimiter).pop()!;
+
+  const chunks = fullPath.split(moduleDelimiter);
+  chunks.shift(); // remove the base path up to '/packages/' or '/plugins/'
+  const moduleFolder = chunks.join(moduleDelimiter); // in case there's an extra /packages/ or /plugins/ folder
+
   let path: string;
 
   if (group === 'platform') {
@@ -79,6 +86,26 @@ export const calculateModuleTargetFolder = (module: Package): string => {
   return applyTransforms(module, path);
 };
 
+export const isInTargetFolder = (module: Package, log: ToolingLog): boolean => {
+  if (!module.group || !module.visibility) {
+    log.warning(`The module '${module.id}' is missing the group/visibility information`);
+    return true;
+  }
+
+  const baseTargetFolders = TARGET_FOLDERS[`${module.group}:${module.visibility}`];
+  const baseTargetFolder = baseTargetFolders.find((candidate) => {
+    return module.directory.includes(candidate);
+  });
+  if (baseTargetFolder) {
+    log.info(
+      `The module ${module.id} is already in the correct folder: '${baseTargetFolder}'. Skipping`
+    );
+    return true;
+  }
+
+  return false;
+};
+
 export const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => {
   const dir = module.directory;
   const source =

packages/kbn-relocate/utils/transforms.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type { Package } from './types';
+import type { Package } from '../types';
 
 type TransformFunction = (param: string) => string;
 const TRANSFORMS: Record<string, string | TransformFunction> = {