mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
SKA: Relocate Script v7 (#205732)
## Summary Addresses the following: * Simplify `isInTargetFolder`, and leverage existing `calculateModuleTargetFolder`. It solves a bug with "current" location being incorrectly determined, as we were using the `group` and `visibility` inferred from path, rather than those in the manifest. * Move the `pre-relocation` hook to BEFORE calculating the list of modules. This allows to update a manifest to re-relocate a module (e.g. when changing its group or visibility). * Fix a bug that caused modules under `src/core/packages` to not be considered in the "correct location". * Fix a bug in the replace logic specific to `pipeline.ts`. We were updating paths that we shouldn't have updated.
This commit is contained in:
Gerard Soldevila
GitHub
parent
08535f54a0
commit
ca42d93bd4
6 changed files with 226 additions and 76 deletions
|
|
@ -13,26 +13,7 @@ export const BASE_FOLDER = process.cwd() + '/';
|
|||
export const BASE_FOLDER_DEPTH = process.cwd().split('/').length;
|
||||
export const KIBANA_FOLDER = process.cwd().split('/').pop()!;
|
||||
export const EXCLUDED_MODULES = ['@kbn/core'];
|
||||
export const TARGET_FOLDERS: Record<string, string[]> = {
|
||||
'platform:private': [
|
||||
'src/platform/packages/private/',
|
||||
'src/platform/plugins/private/',
|
||||
'x-pack/platform/packages/private/',
|
||||
'x-pack/platform/plugins/private/',
|
||||
],
|
||||
'platform:shared': [
|
||||
'src/platform/packages/shared/',
|
||||
'src/platform/plugins/shared/',
|
||||
'x-pack/platform/packages/shared/',
|
||||
'x-pack/platform/plugins/shared/',
|
||||
],
|
||||
'observability:private': [
|
||||
'x-pack/solutions/observability/packages/',
|
||||
'x-pack/solutions/observability/plugins/',
|
||||
],
|
||||
'search:private': ['x-pack/solutions/search/packages/', 'x-pack/solutions/search/plugins/'],
|
||||
'security:private': ['x-pack/solutions/security/packages/', 'x-pack/solutions/security/plugins/'],
|
||||
};
|
||||
|
||||
export const EXTENSIONS = [
|
||||
'eslintignore',
|
||||
'gitignore',
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
import { run } from '@kbn/dev-cli-runner';
|
||||
import { findAndRelocateModules, findAndMoveModule } from './relocate';
|
||||
import { listModules } from './list';
|
||||
|
||||
const toStringArray = (flag: string | boolean | string[] | undefined): string[] => {
|
||||
if (typeof flag === 'string') {
|
||||
|
|
@ -44,6 +45,8 @@ export const runKbnRelocateCli = () => {
|
|||
if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) {
|
||||
log.info('When using --moveOnly flag, the rest of flags are ignored.');
|
||||
await findAndMoveModule(flags.moveOnly, log);
|
||||
} else if (typeof flags.list === 'string' && flags.list.length > 0) {
|
||||
await listModules(flags.list, log);
|
||||
} else {
|
||||
const { pr, team, path, include, exclude, baseBranch } = flags;
|
||||
await findAndRelocateModules(
|
||||
|
|
@ -64,7 +67,7 @@ export const runKbnRelocateCli = () => {
|
|||
defaultLevel: 'info',
|
||||
},
|
||||
flags: {
|
||||
string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly'],
|
||||
string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly', 'list'],
|
||||
help: `
|
||||
Usage: node scripts/relocate [options]
|
||||
|
||||
|
|
@ -75,6 +78,9 @@ export const runKbnRelocateCli = () => {
|
|||
--include <id> Include the specified module in the relocation (can specify multiple modules)
|
||||
--exclude <id> Exclude the specified module from the relocation (can use multiple times)
|
||||
--baseBranch <name> Use a branch different than 'main' (e.g. "8.x")
|
||||
--list "all" List all Kibana modules
|
||||
--list "uncategorised" List Kibana modules that are lacking 'group' or 'visibility' information
|
||||
--list "incorrect" List Kibana modules that are not in the correct folder (aka folder does not match group/visibility in the manifest)
|
||||
|
||||
E.g. relocate all modules owned by Core team and also modules owned by Operations team, excluding 'foo-module-id'. Force push into PR 239847:
|
||||
node scripts/relocate --pr 239847 --team @elastic/kibana-core --team @elastic/kibana-operations --exclude @kbn/foo-module-id
|
||||
|
|
|
|||
140
packages/kbn-relocate/list.ts
Normal file
140
packages/kbn-relocate/list.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { sortBy } from 'lodash';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import { getPackages } from '@kbn/repo-packages';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import type { Package } from './types';
|
||||
import { BASE_FOLDER, EXCLUDED_MODULES, KIBANA_FOLDER } from './constants';
|
||||
import { calculateModuleTargetFolder, isInTargetFolder } from './utils/relocate';
|
||||
import { createModuleTable } from './utils/logging';
|
||||
|
||||
export const listModules = async (listFlag: string, log: ToolingLog) => {
|
||||
// get all modules
|
||||
const modules = getPackages(REPO_ROOT);
|
||||
const devOnly: Package[] = [];
|
||||
const test: Package[] = [];
|
||||
const examples: Package[] = [];
|
||||
const uncategorised: Package[] = [];
|
||||
const incorrect: Package[] = [];
|
||||
const correct: Package[] = [];
|
||||
|
||||
// find modules selected by user filters
|
||||
sortBy(modules, 'directory')
|
||||
// explicit exclusions
|
||||
.filter(({ id }) => !EXCLUDED_MODULES.includes(id))
|
||||
.forEach((module) => {
|
||||
if (module.isDevOnly()) {
|
||||
devOnly.push(module);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
module.directory.includes(`/${KIBANA_FOLDER}/test/`) ||
|
||||
module.directory.includes(`/${KIBANA_FOLDER}/x-pack/test/`)
|
||||
) {
|
||||
test.push(module);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
module.directory.includes(`/${KIBANA_FOLDER}/examples/`) ||
|
||||
module.directory.includes(`/${KIBANA_FOLDER}/x-pack/examples/`)
|
||||
) {
|
||||
examples.push(module);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!module.group || module.group === 'common' || !module.visibility) {
|
||||
// log.warning(`The module ${module.id} does not specify 'group' or 'visibility'. Skipping`);
|
||||
uncategorised.push(module);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isInTargetFolder(module)) {
|
||||
incorrect.push(module);
|
||||
// log.warning(dedent`The module ${module.id} is not in the correct folder:
|
||||
// - ${module.directory}
|
||||
// - ${calculateModuleTargetFolder(module)}`);
|
||||
|
||||
return;
|
||||
}
|
||||
correct.push(module);
|
||||
});
|
||||
|
||||
if (listFlag === 'all') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
[
|
||||
[`${correct.length} modules are placed in a 'sustainable' folder`],
|
||||
[`${devOnly.length} modules are devOnly: true (use --list devOnly)`],
|
||||
[`${test.length} modules are in /test/ and /x-pack/test/ folders (use --list test)`],
|
||||
[
|
||||
`${examples.length} modules are in /examples/ and /x-pack/examples/ folders (use --list examples)`,
|
||||
],
|
||||
[`${incorrect.length} modules are not in the correct folder (use --list incorrect)`],
|
||||
[`${uncategorised.length} modules are not categorised (use --list uncategorised)`],
|
||||
],
|
||||
['Summary']
|
||||
).toString()
|
||||
);
|
||||
} else if (listFlag === 'devOnly') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
devOnly.map((module) => [module.id, module.directory.replace(BASE_FOLDER, '')]),
|
||||
['Id', 'Current folder']
|
||||
).toString()
|
||||
);
|
||||
log.info(`TOTAL: ${devOnly.length} modules`);
|
||||
} else if (listFlag === 'test') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
test.map((module) => [module.id, module.directory.replace(BASE_FOLDER, '')]),
|
||||
['Id', 'Current folder']
|
||||
).toString()
|
||||
);
|
||||
log.info(`TOTAL: ${test.length} modules`);
|
||||
} else if (listFlag === 'examples') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
examples.map((module) => [module.id, module.directory.replace(BASE_FOLDER, '')]),
|
||||
['Id', 'Current folder']
|
||||
).toString()
|
||||
);
|
||||
log.info(`TOTAL: ${examples.length} modules`);
|
||||
} else if (listFlag === 'incorrect') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
sortBy(
|
||||
incorrect.map((module) => [
|
||||
module.id,
|
||||
module.manifest.owner.join(', '),
|
||||
module.directory.replace(BASE_FOLDER, ''),
|
||||
calculateModuleTargetFolder(module).replace(BASE_FOLDER, ''),
|
||||
]),
|
||||
['1', '0']
|
||||
),
|
||||
['Id', 'Team', 'Current folder', 'Target folder']
|
||||
).toString()
|
||||
);
|
||||
log.info(`TOTAL: ${incorrect.length} modules`);
|
||||
} else if (listFlag === 'uncategorised') {
|
||||
log.info(
|
||||
createModuleTable(
|
||||
uncategorised.map((module) => [
|
||||
module.id,
|
||||
`${module.directory.replace(BASE_FOLDER, '')}/kibana.jsonc`,
|
||||
]),
|
||||
['Id', 'Manifest']
|
||||
).toString()
|
||||
);
|
||||
log.info(`TOTAL: ${uncategorised.length} modules`);
|
||||
}
|
||||
};
|
||||
|
|
@ -11,7 +11,7 @@ import { join } from 'path';
|
|||
import { existsSync } from 'fs';
|
||||
import { rename, mkdir, rm } from 'fs/promises';
|
||||
import inquirer from 'inquirer';
|
||||
import { orderBy } from 'lodash';
|
||||
import { sortBy } from 'lodash';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import { getPackages } from '@kbn/repo-packages';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
|
|
@ -104,8 +104,8 @@ const findModules = ({ teams, paths, included, excluded }: FindModulesParams, lo
|
|||
const modules = getPackages(REPO_ROOT);
|
||||
|
||||
// find modules selected by user filters
|
||||
return orderBy(
|
||||
modules
|
||||
return (
|
||||
sortBy(modules, ['directory'])
|
||||
// exclude devOnly modules (they will remain in /packages)
|
||||
.filter(({ manifest }) => !manifest.devOnly)
|
||||
// explicit exclusions
|
||||
|
|
@ -127,8 +127,28 @@ const findModules = ({ teams, paths, included, excluded }: FindModulesParams, lo
|
|||
)
|
||||
// the module is not explicitly excluded
|
||||
.filter(({ id }) => !excluded.includes(id))
|
||||
// exclude modules that don't define a group/visibility
|
||||
.filter((module) => {
|
||||
if (!module.group || module.group === 'common' || !module.visibility) {
|
||||
log.info(`The module ${module.id} does not specify 'group' or 'visibility'. Skipping`);
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
})
|
||||
// exclude modules that are in the correct folder
|
||||
.filter((module) => !isInTargetFolder(module, log))
|
||||
.filter((module) => {
|
||||
if (isInTargetFolder(module)) {
|
||||
log.info(
|
||||
`The module ${
|
||||
module.id
|
||||
} is already in the correct folder: '${calculateModuleTargetFolder(module)}'. Skipping`
|
||||
);
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
|
|
@ -159,27 +179,6 @@ export const findAndRelocateModules = async (params: RelocateModulesParams, log:
|
|||
return;
|
||||
}
|
||||
|
||||
const toMove = findModules(findParams, log);
|
||||
if (!toMove.length) {
|
||||
log.info(
|
||||
`No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
relocatePlan(toMove, log);
|
||||
|
||||
const resConfirmPlan = await inquirer.prompt({
|
||||
type: 'confirm',
|
||||
name: 'confirmPlan',
|
||||
message: `The script will RESET CHANGES in this repository, relocate the modules above and update references. Proceed?`,
|
||||
});
|
||||
|
||||
if (!resConfirmPlan.confirmPlan) {
|
||||
log.info('Aborting');
|
||||
return;
|
||||
}
|
||||
|
||||
if (prNumber) {
|
||||
pr = await findPr(prNumber);
|
||||
|
||||
|
|
@ -187,14 +186,27 @@ export const findAndRelocateModules = async (params: RelocateModulesParams, log:
|
|||
const resOverride = await inquirer.prompt({
|
||||
type: 'confirm',
|
||||
name: 'overrideManualCommits',
|
||||
message: 'Detected manual commits in the PR, do you want to override them?',
|
||||
message:
|
||||
'Manual commits detected in the PR. The script will try to cherry-pick them, but it might require manual intervention to resolve conflicts. Continue?',
|
||||
});
|
||||
if (!resOverride.overrideManualCommits) {
|
||||
log.info('Aborting');
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const resConfirmReset = await inquirer.prompt({
|
||||
type: 'confirm',
|
||||
name: 'confirmReset',
|
||||
message: `The script will RESET CHANGES in this repository. Proceed?`,
|
||||
});
|
||||
|
||||
if (!resConfirmReset.confirmReset) {
|
||||
log.info('Aborting');
|
||||
return;
|
||||
}
|
||||
|
||||
// start with a clean repo
|
||||
await safeExec(`git restore --staged .`);
|
||||
await safeExec(`git restore .`);
|
||||
|
|
@ -215,20 +227,40 @@ export const findAndRelocateModules = async (params: RelocateModulesParams, log:
|
|||
await checkoutBranch(NEW_BRANCH);
|
||||
}
|
||||
|
||||
// push changes in the branch
|
||||
await safeExec(`yarn kbn bootstrap`);
|
||||
await inquirer.prompt({
|
||||
type: 'confirm',
|
||||
name: 'readyRelocate',
|
||||
message: `Ready to relocate! You can commit changes previous to the relocation at this point. Confirm to proceed with the relocation`,
|
||||
});
|
||||
|
||||
const toMove = findModules(findParams, log);
|
||||
if (!toMove.length) {
|
||||
log.info(
|
||||
`No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
relocatePlan(toMove, log);
|
||||
|
||||
const resConfirmPlan = await inquirer.prompt({
|
||||
type: 'confirm',
|
||||
name: 'confirmPlan',
|
||||
message: `The script will relocate the modules above and update references. Proceed?`,
|
||||
});
|
||||
|
||||
if (!resConfirmPlan.confirmPlan) {
|
||||
log.info('Aborting');
|
||||
return;
|
||||
}
|
||||
|
||||
// relocate modules
|
||||
await safeExec(`yarn kbn bootstrap`);
|
||||
const movedCount = await relocateModules(toMove, log);
|
||||
|
||||
if (movedCount === 0) {
|
||||
log.warning(
|
||||
'No modules were relocated, aborting operation to prevent force-pushing empty changes (this would close the existing PR!)'
|
||||
'No modules were relocated, aborting operation to prevent force-pushing empty changes'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ export const createModuleTable = (
|
|||
) => {
|
||||
const table = new Table({
|
||||
head,
|
||||
colAligns: ['left', 'left'],
|
||||
colAligns: head.map(() => 'left'),
|
||||
style: {
|
||||
compact: true,
|
||||
'padding-left': 2,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import {
|
|||
KIBANA_FOLDER,
|
||||
NO_GREP,
|
||||
SCRIPT_ERRORS,
|
||||
TARGET_FOLDERS,
|
||||
UPDATED_REFERENCES,
|
||||
UPDATED_RELATIVE_PATHS,
|
||||
} from '../constants';
|
||||
|
|
@ -38,13 +37,20 @@ export const stripFirstChunk = (path: string): string => {
|
|||
export const calculateModuleTargetFolder = (module: Package): string => {
|
||||
const group = module.manifest.group!;
|
||||
const isPlugin = module.manifest.type === 'plugin';
|
||||
const fullPath = join(BASE_FOLDER, module.directory);
|
||||
const fullPath = module.directory.startsWith(BASE_FOLDER)
|
||||
? module.directory
|
||||
: join(BASE_FOLDER, module.directory);
|
||||
|
||||
let moduleDelimiter: string;
|
||||
if (!fullPath.includes('/plugins/') && !fullPath.includes('/packages/')) {
|
||||
throw new Error(
|
||||
`The module ${module.id} is not located under a '*/plugins/*' or '*/packages/*' folder`
|
||||
);
|
||||
} else if (fullPath.includes('/plugins/') && fullPath.includes('/packages/')) {
|
||||
moduleDelimiter = isPlugin ? '/plugins/' : '/packages/';
|
||||
} else {
|
||||
moduleDelimiter = fullPath.includes('/plugins/') ? '/plugins/' : '/packages/';
|
||||
}
|
||||
let moduleDelimiter = fullPath.includes('/plugins/') ? '/plugins/' : '/packages/';
|
||||
|
||||
// for platform modules that are in a sustainable folder, strip the /private/ or /shared/ part too
|
||||
if (module.directory.includes(`${moduleDelimiter}private/`)) {
|
||||
|
|
@ -60,7 +66,10 @@ export const calculateModuleTargetFolder = (module: Package): string => {
|
|||
let path: string;
|
||||
|
||||
if (group === 'platform') {
|
||||
if (fullPath.includes(`/${KIBANA_FOLDER}/packages/core/`)) {
|
||||
if (
|
||||
fullPath.includes(`/${KIBANA_FOLDER}/packages/core/`) ||
|
||||
fullPath.includes(`/${KIBANA_FOLDER}/src/core/packages`)
|
||||
) {
|
||||
// packages/core/* => src/core/packages/*
|
||||
path = join(BASE_FOLDER, 'src', 'core', 'packages', moduleFolder);
|
||||
} else {
|
||||
|
|
@ -91,26 +100,8 @@ export const calculateModuleTargetFolder = (module: Package): string => {
|
|||
return applyTransforms(module, path);
|
||||
};
|
||||
|
||||
export const isInTargetFolder = (module: Package, log: ToolingLog): boolean => {
|
||||
const { group, visibility } = module.manifest;
|
||||
|
||||
if (!group || group === 'common' || !visibility) {
|
||||
log.warning(`The module '${module.id}' is missing the group/visibility information`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const baseTargetFolders = TARGET_FOLDERS[`${group}:${visibility}`];
|
||||
const baseTargetFolder = baseTargetFolders.find((candidate) => {
|
||||
return module.directory.includes(candidate);
|
||||
});
|
||||
if (baseTargetFolder) {
|
||||
log.info(
|
||||
`The module ${module.id} is already in the correct folder: '${baseTargetFolder}'. Skipping`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
export const isInTargetFolder = (module: Package): boolean => {
|
||||
return module.directory.startsWith(calculateModuleTargetFolder(module));
|
||||
};
|
||||
|
||||
export const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => {
|
||||
|
|
@ -186,7 +177,7 @@ const replaceReferencesInternal = async (
|
|||
const backFwdSrc = relativeSource.replaceAll('/', `\\\\\\/`);
|
||||
const backFwdDst = relativeDestination.replaceAll('/', `\\\\\\/`);
|
||||
await safeExec(
|
||||
`sed -i '' -E '/${src}[\-_a-zA-Z0-9]/! s/${backFwdSrc}/${backFwdDst}/g' .buildkite/scripts/pipelines/pull_request/pipeline.ts`,
|
||||
`sed -i '' -E '/${backFwdSrc}[\-_a-zA-Z0-9]/! s/${backFwdSrc}/${backFwdDst}/g' .buildkite/scripts/pipelines/pull_request/pipeline.ts`,
|
||||
false
|
||||
);
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue