[8.x] Sustainable Kibana Architecture: Relocate script v5 (#204522)

## Summary

Backport https://github.com/elastic/kibana/pull/204461 to `8.x` branch

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Gerard Soldevila 2024-12-17 13:57:27 +01:00 committed by GitHub
parent 719342892d
commit e3c5e8ddb1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 1228 additions and 0 deletions

1
.github/CODEOWNERS vendored
View file

@ -721,6 +721,7 @@ packages/react/kibana_context/theme @elastic/appex-sharedux
packages/react/kibana_mount @elastic/appex-sharedux
packages/kbn-react-mute-legacy-root-warning @elastic/appex-sharedux
packages/kbn-recently-accessed @elastic/appex-sharedux
packages/kbn-relocate @elastic/kibana-core
x-pack/platform/plugins/private/remote_clusters @elastic/kibana-management
test/plugin_functional/plugins/rendering_plugin @elastic/kibana-core
packages/kbn-repo-file-maps @elastic/kibana-operations

View file

@ -1488,6 +1488,7 @@
"@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
"@kbn/plugin-helpers": "link:packages/kbn-plugin-helpers",
"@kbn/product-doc-artifact-builder": "link:x-pack/packages/ai-infra/product-doc-artifact-builder",
"@kbn/relocate": "link:packages/kbn-relocate",
"@kbn/repo-file-maps": "link:packages/kbn-repo-file-maps",
"@kbn/repo-linter": "link:packages/kbn-repo-linter",
"@kbn/repo-path": "link:packages/kbn-repo-path",

View file

@ -0,0 +1,57 @@
# @kbn/relocate
This package contains a CLI tool to help move modules (plugins and packages) into their intended folders, according to the _Sustainable Kibana Architecture.
## Prerequisites
You must have `gh` CLI tool installed. You can install it by running:
```sh
brew install gh
gh auth login
```
## Usage
First of all, you need to decide whether you want to contribute to an existing PR or to create a new one. Use the `--pr` flag to specify the PR you are trying to update:
```sh
node scripts/relocate --pr <prNumber>
```
Note that when specifying an existing PR, the logic will undo + rewrite history for that PR, by force-pushing changes.
To relocate modules for a given team, identify the "team handle" (e.g. @elastic/kibana-core), and run the following command from the root of the Kibana repo:
```sh
node scripts/relocate --pr <prNumber> --team <team_handle>
```
You can relocate modules by path, e.g. all modules that are under `x-pack/plugins/observability_solution/`:
```sh
node scripts/relocate --pr <prNumber> --path "x-pack/plugins/observability_solution/"
```
You can specify indivual packages by ID:
```sh
node scripts/relocate --pr <prNumber> --include "@kbn/data-forge" --include "@kbn/deeplinks-observability"
```
You can also specify combinations of the above filters, to include modules that match ANY of the criteria.
Excluding modules explictly is also supported:
```sh
node scripts/relocate --pr <prNumber> --team "@elastic/obs-ux-management-team" --exclude "@kbn/data-forge"
```
## Details
The script generates log / description files of the form `relocate_YYYYMMDDhhmmss_<type>.out`. You can inspect them if you encounter any errors.
In particular, the file `relocate_YYYYMMDDhhmmss_description.out` contains the auto-generated PR description. You can push it to the PR by running:
```sh
gh pr edit <prNumber> -F relocate_YYYYMMDDhhmmss_description.out -R elastic/kibana
```

View file

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import dedent from 'dedent';
export const BASE_FOLDER = process.cwd() + '/';
export const BASE_FOLDER_DEPTH = process.cwd().split('/').length;
export const KIBANA_FOLDER = process.cwd().split('/').pop()!;
export const EXCLUDED_MODULES = ['@kbn/core'];
export const TARGET_FOLDERS: Record<string, string[]> = {
'platform:private': [
'src/platform/packages/private/',
'src/platform/plugins/private/',
'x-pack/platform/packages/private/',
'x-pack/platform/plugins/private/',
],
'platform:shared': [
'src/platform/packages/shared/',
'src/platform/plugins/shared/',
'x-pack/platform/packages/shared/',
'x-pack/platform/plugins/shared/',
],
'observability:private': [
'x-pack/solutions/observability/packages/',
'x-pack/solutions/observability/plugins/',
],
'search:private': ['x-pack/solutions/search/packages/', 'x-pack/solutions/search/plugins/'],
'security:private': ['x-pack/solutions/security/packages/', 'x-pack/solutions/security/plugins/'],
};
export const EXTENSIONS = [
'eslintignore',
'gitignore',
'js',
'mjs',
'txt',
'json',
'lock',
'bazel',
'md',
'mdz',
'asciidoc',
'sh',
'ts',
'jsonc',
'yaml',
'yml',
];
export const EXCLUDED_FOLDERS = [
'./api_docs', // autogenerated daily https://buildkite.com/elastic/kibana-api-docs-daily
'./.chromium',
'./.devcontainer',
'./.es',
'./.git',
// './.github',
'./.native_modules',
'./.node_binaries',
'./.vscode',
'./.yarn-local-mirror',
'./build',
'./core_http.codeql',
'./data',
'./node_modules',
'./target',
'./test.codeql',
'./test2.codeql',
'./trash',
];
export const NO_GREP = EXCLUDED_FOLDERS.map((f) => `--exclude-dir "${f}"`).join(' ');
// These two constants are singletons, used and updated throughout the process
export const UPDATED_REFERENCES = new Set<string>();
export const UPDATED_RELATIVE_PATHS = new Set<string>();
export const SCRIPT_ERRORS: string[] = [];
export const YMDMS = new Date()
.toISOString()
.replace(/[^0-9]/g, '')
.slice(0, -3);
export const DESCRIPTION = `relocate_${YMDMS}_description.out`;
export const NEW_BRANCH = `kbn-team-1309-relocate-${YMDMS}`;
export const GLOBAL_DESCRIPTION = dedent`
## Summary
This PR aims at relocating some of the Kibana modules (plugins and packages) into a new folder structure, according to the _Sustainable Kibana Architecture_ initiative.
> [!IMPORTANT]
> * We kindly ask you to:
> * Manually fix the errors in the error section below (if there are any).
> * Search for the \`packages[\/\\]\` and \`plugins[\/\\]\` patterns in the source code (Babel and Eslint config files), and update them appropriately.
> * Manually review \`.buildkite/scripts/pipelines/pull_request/pipeline.ts\` to ensure that any CI pipeline customizations continue to be correctly applied after the changed path names
> * Review all of the updated files, specially the \`.ts\` and \`.js\` files listed in the sections below, as some of them contain relative paths that have been updated.
> * Think of potential impact of the move, including tooling and configuration files that can be pointing to the relocated modules. E.g.:
> * customised eslint rules
> * docs pointing to source code
> [!NOTE]
> * This PR has been auto-generated.
> * Any manual contributions will be lost if the 'relocate' script is re-run.
> * Try to obtain the missing reviews / approvals before applying manual fixes, and/or keep your changes in a .patch / git stash.
> * Please use [#sustainable_kibana_architecture](https://elastic.slack.com/archives/C07TCKTA22E) Slack channel for feedback.
Are you trying to rebase this PR to solve merge conflicts? Please follow the steps describe [here](https://elastic.slack.com/archives/C07TCKTA22E/p1734019532879269?thread_ts=1734019339.935419&cid=C07TCKTA22E).
`;

View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { run } from '@kbn/dev-cli-runner';
import { findAndRelocateModules, findAndMoveModule } from './relocate';
const toStringArray = (flag: string | boolean | string[] | undefined): string[] => {
if (typeof flag === 'string') {
return [flag].filter(Boolean);
} else if (typeof flag === 'boolean') {
return [];
} else if (Array.isArray(flag)) {
return flag.filter(Boolean);
}
return [];
};
const toOptString = (
flagName: string,
flag: string | boolean | string[] | undefined,
defaultValue?: string
): string | undefined => {
if (typeof flag === 'boolean') {
throw Error(`You must specify a valid string for the --${flagName} flag`);
} else if (Array.isArray(flag)) {
throw Error(`Cannot specify multiple values for --${flagName} flag`);
}
return flag || defaultValue;
};
/**
* A CLI to move Kibana modules into the right folder structure,
* according to the Sustainable Kibana Architecture
*/
export const runKbnRelocateCli = () => {
run(
async ({ log, flags }) => {
if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) {
log.info('When using --moveOnly flag, the rest of flags are ignored.');
await findAndMoveModule(flags.moveOnly, log);
} else {
const { pr, team, path, include, exclude, baseBranch } = flags;
await findAndRelocateModules(
{
prNumber: toOptString('prNumber', pr),
baseBranch: toOptString('baseBranch', baseBranch, 'main')!,
teams: toStringArray(team),
paths: toStringArray(path),
included: toStringArray(include),
excluded: toStringArray(exclude),
},
log
);
}
},
{
log: {
defaultLevel: 'info',
},
flags: {
string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly'],
help: `
Usage: node scripts/relocate [options]
--moveOnly <moduleId> Only move the specified module in the current branch (no cleanup, no branching, no commit)
--pr <number> Use the given PR number instead of creating a new one
--team <owner> Include all modules (packages and plugins) belonging to the specified owner (can specify multiple teams)
--path <path> Include all modules (packages and plugins) under the specified path (can specify multiple paths)
--include <id> Include the specified module in the relocation (can specify multiple modules)
--exclude <id> Exclude the specified module from the relocation (can use multiple times)
--baseBranch <name> Use a branch different than 'main' (e.g. "8.x")
E.g. relocate all modules owned by Core team and also modules owned by Operations team, excluding 'foo-module-id'. Force push into PR 239847:
node scripts/relocate --pr 239847 --team @elastic/kibana-core --team @elastic/kibana-operations --exclude @kbn/foo-module-id
`,
},
}
);
};

View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
module.exports = {
preset: '@kbn/test/jest_node',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-relocate'],
};

View file

@ -0,0 +1,6 @@
{
"type": "shared-server",
"id": "@kbn/relocate",
"owner": "@elastic/kibana-core",
"devOnly": true
}

View file

@ -0,0 +1,6 @@
{
"name": "@kbn/relocate",
"private": true,
"version": "1.0.0",
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
}

View file

@ -0,0 +1,260 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { join } from 'path';
import { existsSync } from 'fs';
import { rename, mkdir, rm } from 'fs/promises';
import inquirer from 'inquirer';
import { orderBy } from 'lodash';
import type { ToolingLog } from '@kbn/tooling-log';
import { getPackages } from '@kbn/repo-packages';
import { REPO_ROOT } from '@kbn/repo-info';
import type { Package, PullRequest } from './types';
import { DESCRIPTION, EXCLUDED_MODULES, KIBANA_FOLDER, NEW_BRANCH } from './constants';
import {
belongsTo,
calculateModuleTargetFolder,
isInTargetFolder,
replaceReferences,
replaceRelativePaths,
} from './utils/relocate';
import { safeExec } from './utils/exec';
import { relocatePlan, relocateSummary } from './utils/logging';
import {
checkoutBranch,
checkoutResetPr,
cherryPickManualCommits,
findGithubLogin,
findPr,
findRemoteName,
getManualCommits,
} from './utils/git';
const moveModule = async (module: Package, log: ToolingLog) => {
const destination = calculateModuleTargetFolder(module);
log.info(`Moving ${module.directory} to ${destination}`);
const chunks = destination.split('/');
chunks.pop(); // discard module folder
if (existsSync(destination)) {
await rm(destination, { recursive: true });
}
await mkdir(join('/', ...chunks), { recursive: true });
await rename(module.directory, destination);
await replaceReferences(module, destination, log);
await replaceRelativePaths(module, destination, log);
};
const relocateModules = async (toMove: Package[], log: ToolingLog): Promise<number> => {
let relocated: number = 0;
for (let i = 0; i < toMove.length; ++i) {
const module = toMove[i];
log.info('');
log.info('--------------------------------------------------------------------------------');
log.info(`\t${module.id} (${i + 1} of ${toMove.length})`);
log.info('--------------------------------------------------------------------------------');
await moveModule(module, log);
// after move operations
await safeExec('yarn kbn bootstrap');
await safeExec('node scripts/build_plugin_list_docs');
await safeExec('node scripts/generate codeowners');
await safeExec('node scripts/lint_packages --fix');
await safeExec('node scripts/eslint --no-cache --fix');
await safeExec('node scripts/precommit_hook --fix');
// single commit per module now
await safeExec(`git add .`);
await safeExec(`git commit --no-verify -m "Relocating module \\\`${module.id}\\\`"`);
++relocated;
}
return relocated;
};
interface FindModulesParams {
teams: string[];
paths: string[];
included: string[];
excluded: string[];
}
export interface RelocateModulesParams {
baseBranch: string;
prNumber?: string;
teams: string[];
paths: string[];
included: string[];
excluded: string[];
}
const findModules = ({ teams, paths, included, excluded }: FindModulesParams, log: ToolingLog) => {
// get all modules
const modules = getPackages(REPO_ROOT);
// find modules selected by user filters
return orderBy(
modules
// exclude devOnly modules (they will remain in /packages)
.filter(({ manifest }) => !manifest.devOnly)
// exclude modules that do not specify a group
.filter(({ manifest }) => manifest.group)
// explicit exclusions
.filter(({ id }) => !EXCLUDED_MODULES.includes(id) && !excluded.includes(id))
// we don't want to move test modules (just yet)
.filter(
({ directory }) =>
!directory.includes(`/${KIBANA_FOLDER}/test/`) &&
!directory.includes(`/${KIBANA_FOLDER}/x-pack/test/`)
)
// the module is under the umbrella specified by the user
.filter(
(module) =>
included.includes(module.id) ||
teams.some((team) => belongsTo(module, team)) ||
paths.some((path) => module.directory.includes(path))
)
// the module is not explicitly excluded
.filter(({ id }) => !excluded.includes(id))
// exclude modules that are in the correct folder
.filter((module) => !isInTargetFolder(module, log))
);
};
export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => {
const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] }, log);
if (!modules.length) {
log.warning(`Cannot move ${moduleId}, either not found or not allowed!`);
} else {
await moveModule(modules[0], log);
}
};
export const findAndRelocateModules = async (params: RelocateModulesParams, log: ToolingLog) => {
const { prNumber, baseBranch, ...findParams } = params;
let pr: PullRequest | undefined;
const upstream = await findRemoteName('elastic/kibana');
if (!upstream) {
log.error(
'This repository does not have a remote pointing to the elastic/kibana repository. Aborting'
);
return;
}
const origin = await findRemoteName(`${await findGithubLogin()}/kibana`);
if (!origin) {
log.error('This repository does not have a remote pointing to your Kibana fork. Aborting');
return;
}
const toMove = findModules(findParams, log);
if (!toMove.length) {
log.info(
`No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags`
);
return;
}
relocatePlan(toMove, log);
const resConfirmPlan = await inquirer.prompt({
type: 'confirm',
name: 'confirmPlan',
message: `The script will RESET CHANGES in this repository, relocate the modules above and update references. Proceed?`,
});
if (!resConfirmPlan.confirmPlan) {
log.info('Aborting');
return;
}
if (prNumber) {
pr = await findPr(prNumber);
if (getManualCommits(pr.commits).length > 0) {
const resOverride = await inquirer.prompt({
type: 'confirm',
name: 'overrideManualCommits',
message: 'Detected manual commits in the PR, do you want to override them?',
});
if (!resOverride.overrideManualCommits) {
return;
}
}
}
// start with a clean repo
await safeExec(`git restore --staged .`);
await safeExec(`git restore .`);
await safeExec(`git clean -f -d`);
await safeExec(`git checkout ${baseBranch} && git pull ${upstream} ${baseBranch}`);
if (pr) {
// checkout existing PR, reset all commits, rebase from baseBranch
try {
await checkoutResetPr(pr, baseBranch);
} catch (error) {
log.error(`Error checking out / resetting PR #${prNumber}:`);
log.error(error);
return;
}
} else {
// checkout new branch
await checkoutBranch(NEW_BRANCH);
}
// push changes in the branch
await inquirer.prompt({
type: 'confirm',
name: 'readyRelocate',
message: `Ready to relocate! You can commit changes previous to the relocation at this point. Confirm to proceed with the relocation`,
});
// relocate modules
await safeExec(`yarn kbn bootstrap`);
const movedCount = await relocateModules(toMove, log);
if (movedCount === 0) {
log.warning(
'No modules were relocated, aborting operation to prevent force-pushing empty changes (this would close the existing PR!)'
);
return;
}
relocateSummary(log);
if (pr) {
await cherryPickManualCommits(pr, log);
}
// push changes in the branch
const resPushBranch = await inquirer.prompt({
type: 'confirm',
name: 'pushBranch',
message: `Relocation finished! You can commit extra changes at this point. Confirm to proceed pushing the current branch`,
});
const pushCmd = prNumber
? `git push --force-with-lease`
: `git push --set-upstream ${origin} ${NEW_BRANCH}`;
if (!resPushBranch.pushBranch) {
log.info(`Remember to push changes with "${pushCmd}"`);
return;
}
await safeExec(pushCmd);
if (prNumber) {
await safeExec(`gh pr edit ${prNumber} -F ${DESCRIPTION} -R elastic/kibana`);
log.info(`Access the PR at: https://github.com/elastic/kibana/pull/${prNumber}`);
} else {
log.info('TIP: Run the following command to quickly create a PR:');
log.info(`$ gh pr create -d -t "<title>" -F ${DESCRIPTION} -R elastic/kibana`);
}
};

View file

@ -0,0 +1,22 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"types": [
"jest",
"node"
]
},
"include": [
"**/*.ts",
],
"exclude": [
"target/**/*"
],
"kbn_references": [
"@kbn/dev-cli-runner",
"@kbn/repo-info",
"@kbn/repo-packages",
"@kbn/tooling-log",
]
}

View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export type { Package } from '@kbn/repo-packages';
export interface CommitAuthor {
login: string;
}
export interface Commit {
oid: string;
messageHeadline: string;
authors: CommitAuthor[];
}
export interface PullRequest {
number: string;
commits: Commit[];
headRefName: string;
}

View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import util from 'util';
import { exec } from 'child_process';
export const execAsync = util.promisify(exec);
export const safeExec = async (command: string, critical = true, log = true) => {
try {
if (log) {
// eslint-disable-next-line no-console
console.log(' >', command);
}
const result = await execAsync(command, { maxBuffer: 1024 * 1024 * 128 });
return result;
} catch (err) {
const message = `Error executing ${command}: ${err}`;
if (critical) {
throw err;
}
return { stdout: '', stderr: message };
}
};
export const quietExec = async (command: string) => {
return await safeExec(command, false, false);
};

View file

@ -0,0 +1,181 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import inquirer from 'inquirer';
import type { ToolingLog } from '@kbn/tooling-log';
import type { Commit, PullRequest } from '../types';
import { safeExec } from './exec';
export const findRemoteName = async (repo: string) => {
const res = await safeExec('git remote -v', true, false);
const remotes = res.stdout
.trim()
.split('\n')
.map((line) => line.split(/\t| /).filter(Boolean))
.filter((chunks) => chunks.length >= 2);
return remotes.find(
([, url]) => url.includes(`github.com/${repo}`) || url.includes(`github.com:${repo}`)
)?.[0];
};
export const findGithubLogin = async () => {
const res = await safeExec('gh auth status', true, false);
// e.g. ✓ Logged in to github.com account gsoldevila (/Users/gsoldevila/.config/gh/hosts.yml)
const loginLine = res.stdout
.split('\n')
.find((line) => line.includes('Logged in'))
?.split(/\t| /)
.filter(Boolean);
return loginLine?.[loginLine?.findIndex((fragment) => fragment === 'account') + 1];
};
export const findPr = async (number: string): Promise<PullRequest> => {
const res = await safeExec(`gh pr view ${number} --json commits,headRefName`);
return { ...JSON.parse(res.stdout), number };
};
export const isManualCommit = (commit: Commit) =>
!commit.messageHeadline.startsWith('Relocating module ') &&
!commit.messageHeadline.startsWith('Moving modules owned by ') &&
!commit.messageHeadline.startsWith('Merge branch ') &&
commit.authors.some(
(author) => author.login !== 'kibanamachine' && author.login !== 'elasticmachine'
);
export function getManualCommits(commits: Commit[]) {
return commits.filter(isManualCommit);
}
export async function getLastCommitMessage() {
return (await safeExec('git log -1 --pretty=%B')).stdout.split('\n')[0];
}
export async function resetAllCommits(numCommits: number) {
await safeExec(`git reset --hard HEAD~${numCommits}`);
let msg = await getLastCommitMessage();
while (msg.startsWith('Relocating module ')) {
await safeExec(`git reset --hard HEAD~1`);
msg = await getLastCommitMessage();
}
await safeExec('git restore --staged .');
await safeExec('git restore .');
await safeExec('git clean -f -d');
}
export async function localBranchExists(branchName: string): Promise<boolean> {
const res = await safeExec('git branch -l');
const branches = res.stdout
.split('\n')
.filter(Boolean)
.map((name) => name.trim());
return branches.includes(branchName);
}
async function deleteBranches(...branchNames: string[]) {
const res = await safeExec('git branch -l');
const branches = res.stdout
.split('\n')
.filter(Boolean)
.map((branchName) => branchName.trim());
await Promise.all(
branchNames
.filter((toDelete) => branches.includes(toDelete))
.map((toDelete) => safeExec(`git branch -D ${toDelete}`).catch(() => {}))
);
}
export const checkoutResetPr = async (pr: PullRequest, baseBranch: string) => {
// delete existing branch
await deleteBranches(pr.headRefName);
// checkout the PR branch
await safeExec(`gh pr checkout ${pr.number}`);
await resetAllCommits(pr.commits.length);
await safeExec(`git rebase ${baseBranch}`);
};
export const checkoutBranch = async (branch: string) => {
// create a new branch / PR
if (await localBranchExists(branch)) {
throw new Error('The local branch already exists, aborting!');
} else {
await safeExec(`git checkout -b ${branch}`);
}
};
export const cherryPickManualCommits = async (pr: PullRequest, log: ToolingLog) => {
const manualCommits = getManualCommits(pr.commits);
if (manualCommits.length) {
log.info(`Found manual commits on https://github.com/elastic/kibana/pull/${pr.number}/commits`);
for (let i = 0; i < manualCommits.length; ++i) {
const { oid, messageHeadline, authors } = manualCommits[i];
const url = `https://github.com/elastic/kibana/pull/${pr.number}/commits/${oid}`;
const res = await inquirer.prompt({
type: 'list',
choices: [
{ name: 'Yes, attempt to cherry-pick', value: 'yes' },
{ name: 'No, I will add it manually (press when finished)', value: 'no' },
],
name: 'cherryPick',
message: `Do you want to cherry pick '${messageHeadline}' (${authors[0].login})?`,
});
if (res.cherryPick === 'yes') {
try {
await safeExec(`git cherry-pick ${oid}`);
log.info(`Commit '${messageHeadline}' (${authors[0].login}) cherry-picked successfully!`);
} catch (error) {
log.info(`Error trying to cherry-pick: ${url}`);
log.error(error.message);
const res2 = await inquirer.prompt({
type: 'list',
choices: [
{ name: 'Abort this cherry-pick', value: 'abort' },
{ name: 'Conflicts solved (git cherry-pick --continue)', value: 'continue' },
{ name: 'I solved the conflicts and commited', value: 'done' },
],
name: 'cherryPickFailed',
message: `Automatic cherry-pick failed, manual intervention required`,
});
if (res2.cherryPickFailed === 'abort') {
try {
await safeExec(`git cherry-pick --abort`);
log.warning(
'Cherry-pick aborted, please review changes in that commit and apply them manually if needed!'
);
} catch (error2) {
log.error(
'Cherry-pick --abort failed, please cleanup your working tree before continuing!'
);
}
} else if (res2.cherryPickFailed === 'continue') {
try {
await safeExec(`git cherry-pick --continue`);
log.info(
`Commit '${messageHeadline}' (${authors[0].login}) cherry-picked successfully!`
);
} catch (error2) {
await inquirer.prompt({
type: 'confirm',
name: 'cherryPickContinueFailed',
message: `Cherry pick --continue failed, please address conflicts AND COMMIT manually. Hit confirm when ready`,
});
}
}
}
}
}
}
};

View file

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { ToolingLog } from '@kbn/tooling-log';
import { appendFileSync, writeFileSync } from 'fs';
import dedent from 'dedent';
import Table from 'cli-table3';
import type { Package } from '../types';
import { calculateModuleTargetFolder } from './relocate';
import {
BASE_FOLDER,
DESCRIPTION,
GLOBAL_DESCRIPTION,
SCRIPT_ERRORS,
UPDATED_REFERENCES,
UPDATED_RELATIVE_PATHS,
} from '../constants';
export const createModuleTable = (entries: string[][]) => {
const table = new Table({
head: ['Id', 'Target folder'],
colAligns: ['left', 'left'],
style: {
'padding-left': 2,
'padding-right': 2,
},
});
table.push(...entries);
return table;
};
export const relocatePlan = (modules: Package[], log: ToolingLog) => {
const plugins = modules.filter((module) => module.manifest.type === 'plugin');
const packages = modules.filter((module) => module.manifest.type !== 'plugin');
const target = (module: Package) => calculateModuleTargetFolder(module).replace(BASE_FOLDER, '');
writeFileSync(DESCRIPTION, GLOBAL_DESCRIPTION);
if (plugins.length) {
const pluginList = dedent`
\n\n#### ${plugins.length} plugin(s) are going to be relocated:\n
| Id | Target folder |
| -- | ------------- |
${plugins.map((plg) => `| \`${plg.id}\` | \`${target(plg)}\` |`).join('\n')}
\n\n`;
appendFileSync(DESCRIPTION, pluginList);
const plgTable = createModuleTable(plugins.map((plg) => [plg.id, target(plg)]));
log.info(`${plugins.length} plugin(s) are going to be relocated:\n${plgTable.toString()}`);
}
if (packages.length) {
const packageList = dedent`
\n\n#### ${packages.length} packages(s) are going to be relocated:\n
| Id | Target folder |
| -- | ------------- |
${packages.map((pkg) => `| \`${pkg.id}\` | \`${target(pkg)}\` |`).join('\n')}
\n\n`;
appendFileSync(DESCRIPTION, packageList);
const pkgTable = createModuleTable(packages.map((pkg) => [pkg.id, target(pkg)]));
log.info(`${packages.length} packages(s) are going to be relocated:\n${pkgTable.toString()}`);
}
};
export const appendCollapsible = (
fileName: string,
title: string,
contents: string,
open = false
) => {
appendFileSync(
fileName,
dedent`
<details ${open ? 'open' : ''}>
<summary>${title}</summary>
\`\`\`
${contents}
\`\`\`
</details>`
);
};
export const relocateSummary = (log: ToolingLog) => {
if (SCRIPT_ERRORS.length > 0) {
const contents = SCRIPT_ERRORS.sort().join('\n');
appendCollapsible(DESCRIPTION, 'Script errors', contents, true);
log.warning(`Please address the following errors:\n${contents}`);
}
if (UPDATED_REFERENCES.size > 0) {
const contents = Array.from(UPDATED_REFERENCES).sort().join('\n');
appendCollapsible(DESCRIPTION, 'Updated references', contents);
log.info(
`The following files have been updated to replace references to modules:\n${contents}`
);
}
if (UPDATED_RELATIVE_PATHS.size > 0) {
const contents = Array.from(UPDATED_RELATIVE_PATHS)
.sort()
.map((ref) => ref.replace(BASE_FOLDER, ''))
.join('\n');
appendCollapsible(DESCRIPTION, 'Updated relative paths', contents);
log.info(`The following files contain relative paths that have been updated:\n${contents}`);
}
};

View file

@ -0,0 +1,238 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { join } from 'path';
import type { ToolingLog } from '@kbn/tooling-log';
import { orderBy } from 'lodash';
import type { Package } from '../types';
import { applyTransforms } from './transforms';
import {
BASE_FOLDER,
BASE_FOLDER_DEPTH,
EXTENSIONS,
KIBANA_FOLDER,
NO_GREP,
SCRIPT_ERRORS,
TARGET_FOLDERS,
UPDATED_REFERENCES,
UPDATED_RELATIVE_PATHS,
} from '../constants';
import { quietExec, safeExec } from './exec';
export const belongsTo = (module: Package, owner: string): boolean => {
return Array.from(module.manifest.owner)[0] === owner;
};
export const stripFirstChunk = (path: string): string => {
const chunks = path.split('/');
chunks.shift();
return chunks.join('/');
};
export const calculateModuleTargetFolder = (module: Package): string => {
const group = module.manifest.group!;
const isPlugin = module.manifest.type === 'plugin';
const fullPath = join(BASE_FOLDER, module.directory);
let moduleDelimiter = isPlugin ? '/plugins/' : '/packages/';
// for platform modules that are in a sustainable folder, strip the /private/ or /shared/ part too
if (module.directory.includes(`${moduleDelimiter}private/`)) {
moduleDelimiter += 'private/';
} else if (module.directory.includes(`${moduleDelimiter}shared/`)) {
moduleDelimiter += 'shared/';
}
const chunks = fullPath.split(moduleDelimiter);
chunks.shift(); // remove the base path up to '/packages/' or '/plugins/'
const moduleFolder = chunks.join(moduleDelimiter); // in case there's an extra /packages/ or /plugins/ folder
let path: string;
if (group === 'platform') {
if (fullPath.includes(`/${KIBANA_FOLDER}/packages/core/`)) {
// packages/core/* => src/core/packages/*
path = join(BASE_FOLDER, 'src', 'core', 'packages', moduleFolder);
} else {
const isXpack = fullPath.includes(`/${KIBANA_FOLDER}/x-pack/`);
const visibility = module.manifest.visibility!;
path = join(
BASE_FOLDER,
isXpack ? 'x-pack' : 'src',
group,
isPlugin ? 'plugins' : 'packages',
visibility,
moduleFolder
);
}
} else {
path = join(
BASE_FOLDER,
'x-pack', // all solution modules are 'x-pack'
'solutions',
group,
isPlugin ? 'plugins' : 'packages',
moduleFolder
);
}
// after-creation transforms
return applyTransforms(module, path);
};
export const isInTargetFolder = (module: Package, log: ToolingLog): boolean => {
if (!module.group || !module.visibility) {
log.warning(`The module '${module.id}' is missing the group/visibility information`);
return true;
}
const baseTargetFolders = TARGET_FOLDERS[`${module.group}:${module.visibility}`];
const baseTargetFolder = baseTargetFolders.find((candidate) => {
return module.directory.includes(candidate);
});
if (baseTargetFolder) {
log.info(
`The module ${module.id} is already in the correct folder: '${baseTargetFolder}'. Skipping`
);
return true;
}
return false;
};
export const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => {
const dir = module.directory;
const source =
dir.startsWith(KIBANA_FOLDER) || dir.startsWith(`/${KIBANA_FOLDER}`)
? join(BASE_FOLDER, dir)
: dir;
const relativeSource = source.replace(BASE_FOLDER, '');
const relativeDestination = destination.replace(BASE_FOLDER, '');
if (
(relativeSource.startsWith('src') && relativeDestination.startsWith('src')) ||
(relativeSource.startsWith('x-pack') && relativeDestination.startsWith('x-pack'))
) {
await replaceReferencesInternal(
stripFirstChunk(relativeSource),
stripFirstChunk(relativeDestination),
log
);
} else {
await replaceReferencesInternal(relativeSource, relativeDestination, log);
}
};
const replaceReferencesInternal = async (
relativeSource: string,
relativeDestination: string,
log: ToolingLog
) => {
log.info(`Finding and replacing "${relativeSource}" by "${relativeDestination}"`);
const src = relativeSource.replaceAll('/', '\\/');
const dst = relativeDestination.replaceAll('/', '\\/');
const result = await safeExec(
`grep -I -s -R -l ${EXTENSIONS.map((ext) => `--include="*.${ext}"`).join(' ')} \
${NO_GREP} "${relativeSource}"`,
false
);
const matchingFiles = result.stdout.split('\n').filter(Boolean);
matchingFiles.push('.github/CODEOWNERS'); // to update references in the manual section, thanks pgayvallet!
for (let i = 0; i < matchingFiles.length; ++i) {
const file = matchingFiles[i];
if (file.includes('/target/types/') || file.includes('/target/public/')) {
continue;
}
const md5Before = (await quietExec(`md5 ${file} --quiet`)).stdout.trim();
// if we are updating packages/cloud references, we must pay attention to not update packages/cloud_defend too
await safeExec(`sed -i '' -E "/${src}[\-_a-zA-Z0-9]/! s/${src}/${dst}/g" ${file}`, false);
const md5After = (await quietExec(`md5 ${file} --quiet`)).stdout.trim();
if (md5Before !== md5After) {
UPDATED_REFERENCES.add(file);
}
}
// plugins\/pluginName special treatment (.buildkite/scripts/pipelines/pull_request/pipeline.ts)
const backFwdSrc = relativeSource.replaceAll('/', `\\\\\\/`);
const backFwdDst = relativeDestination.replaceAll('/', `\\\\\\/`);
await safeExec(
`sed -i '' -E '/${src}[\-_a-zA-Z0-9]/! s/${backFwdSrc}/${backFwdDst}/g' .buildkite/scripts/pipelines/pull_request/pipeline.ts`,
false
);
};
const getRelativeDepth = (directory: string): number => {
const fullPath = directory.startsWith(BASE_FOLDER) ? directory : join(BASE_FOLDER, directory);
return fullPath.split('/').length - BASE_FOLDER_DEPTH;
};
export const replaceRelativePaths = async (
module: Package,
destination: string,
log: ToolingLog
) => {
log.info('Updating relative paths at fault');
const relativeDepthBefore = getRelativeDepth(module.directory);
const relativeDepthAfter = getRelativeDepth(destination);
const relativeDepthDiff = relativeDepthAfter - relativeDepthBefore;
const result = await safeExec(
`grep -I -s -R -n -o ${NO_GREP} -E "\\.\\.(/\\.\\.)+/?" ${destination}`,
false
);
const matches = result.stdout.split('\n').filter(Boolean);
const brokenReferences = orderBy(
matches
.map((line) => line.split(':'))
.map(([path, line, match]) => {
if (match.endsWith('/')) {
match = match.substring(0, match.length - 1);
}
let moduleRelativePath = path.replace(destination, '');
if (moduleRelativePath.startsWith('/')) {
moduleRelativePath = moduleRelativePath.substring(1);
}
const moduleRelativeDepth = moduleRelativePath.split('/').length - 1; // do not count filename
const matchDepth = match.split('/').length;
return { path, line, moduleRelativeDepth, match, matchDepth };
})
.filter(({ matchDepth, moduleRelativeDepth }) => matchDepth > moduleRelativeDepth),
'matchDepth',
'desc'
);
for (let i = 0; i < brokenReferences.length; ++i) {
const { path, line, match, matchDepth } = brokenReferences[i];
if (path.includes('/target/types/') || path.includes('/target/public/')) {
continue;
}
const pathLine = `${path}:${line}`;
if (UPDATED_RELATIVE_PATHS.has(pathLine)) {
const message = `Cannot replace multiple occurrences of "${match}" in the same line, please fix manually:\t${pathLine}`;
SCRIPT_ERRORS.push(message);
} else {
const escapedMatch = match.replaceAll('/', '\\/').replaceAll('.', '\\.'); // escape '.' too (regexp any char)
const escapedReplacement = new Array(matchDepth + relativeDepthDiff).fill('..').join('\\/');
await safeExec(`sed -i '' "${line}s/${escapedMatch}/${escapedReplacement}/" ${path}`, false);
UPDATED_RELATIVE_PATHS.add(pathLine);
}
}
};

View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { Package } from '../types';
type TransformFunction = (param: string) => string;
const TRANSFORMS: Record<string, string | TransformFunction> = {
'x-pack/solutions/security/packages/security-solution/': 'x-pack/solutions/security/packages/',
'x-pack/solutions/observability/plugins/observability_solution/':
'x-pack/solutions/observability/plugins/',
'x-pack/solutions/observability/packages/observability/':
'x-pack/solutions/observability/packages/',
'src/core/packages/core/': (path: string) => {
const relativePath = path.split('src/core/packages/')[1];
const relativeChunks = relativePath.split('/');
const packageName = relativeChunks.pop();
const unneededPrefix = relativeChunks.join('-') + '-';
// strip the spare /core/ folder
path = path.replace('src/core/packages/core/', 'src/core/packages/');
if (packageName?.startsWith(unneededPrefix)) {
return path.replace(unneededPrefix, '');
} else {
return path;
}
},
};
export const applyTransforms = (module: Package, path: string): string => {
const transform = Object.entries(TRANSFORMS).find(([what]) => path.includes(what));
if (!transform) {
return path;
} else {
const [what, by] = transform;
if (typeof by === 'function') {
return by(path);
} else if (typeof by === 'string') {
return path.replace(what, by);
} else {
throw new Error('Invalid transform function', by);
}
}
};

11
scripts/relocate.js Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
require('../src/setup_node_env');
require('@kbn/relocate').runKbnRelocateCli();

View file

@ -1436,6 +1436,8 @@
"@kbn/react-mute-legacy-root-warning/*": ["packages/kbn-react-mute-legacy-root-warning/*"],
"@kbn/recently-accessed": ["packages/kbn-recently-accessed"],
"@kbn/recently-accessed/*": ["packages/kbn-recently-accessed/*"],
"@kbn/relocate": ["packages/kbn-relocate"],
"@kbn/relocate/*": ["packages/kbn-relocate/*"],
"@kbn/remote-clusters-plugin": ["x-pack/platform/plugins/private/remote_clusters"],
"@kbn/remote-clusters-plugin/*": ["x-pack/platform/plugins/private/remote_clusters/*"],
"@kbn/rendering-plugin": ["test/plugin_functional/plugins/rendering_plugin"],

View file

@ -6679,6 +6679,10 @@
version "0.0.0"
uid ""
"@kbn/relocate@link:packages/kbn-relocate":
version "0.0.0"
uid ""
"@kbn/remote-clusters-plugin@link:x-pack/platform/plugins/private/remote_clusters":
version "0.0.0"
uid ""