github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[kbn/pm] Remove legacy build code (#107217)

Browse source
This commit is contained in:
Jonathan Budzenski 2021-08-02 13:53:27 -05:00 • committed by GitHub
parent f3a17a80b4
commit 91bb2c5b57
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 5632 additions and 6309 deletions
Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -58,7 +58,6 @@
"makelogs": "node scripts/makelogs",
"build:types": "rm -rf ./target/types && tsc --p tsconfig.types.json",
"docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept",
"kbn:bootstrap": "node scripts/build_ts_refs --ignore-type-failures",
"spec_to_console": "node scripts/spec_to_console",
"backport-skip-ci": "backport --prDescription \"[skip-ci]\"",
"storybook": "node scripts/storybook",

11572
packages/kbn-pm/dist/index.js vendored
View file

File diff suppressed because one or more lines are too long

56
packages/kbn-pm/src/commands/bootstrap.ts
View file

@ -7,14 +7,10 @@
*/
import { resolve, sep } from 'path';
import { spawnStreaming } from '../utils/child_process';
import { linkProjectExecutables } from '../utils/link_project_executables';
import { log } from '../utils/log';
import { parallelizeBatches } from '../utils/parallelize';
import { getNonBazelProjectsOnly, topologicallyBatchProjects } from '../utils/projects';
import { Project } from '../utils/project';
import { ICommand } from './';
import { getAllChecksums } from '../utils/project_checksums';
import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
import { readYarnLock } from '../utils/yarn_lock';
import { validateDependencies } from '../utils/validate_dependencies';
import {
@ -106,45 +102,15 @@ export const BootstrapCommand: ICommand = {
// NOTE: We don't probably need this anymore, is actually not being used
await linkProjectExecutables(projects, projectGraph);
// Bootstrap process for non Bazel packages
/**
* At the end of the bootstrapping process we call all `kbn:bootstrap` scripts
* in the list of non Bazel projects. We do this because some projects need to be
* transpiled before they can be used. Ideally we shouldn't do this unless we
* have to, as it will slow down the bootstrapping process.
*/
const checksums = await getAllChecksums(kbn, log, yarnLock);
const caches = new Map<Project, { file: BootstrapCacheFile; valid: boolean }>();
let cachedProjectCount = 0;
for (const project of nonBazelProjectsOnly.values()) {
if (project.hasScript('kbn:bootstrap') && !project.isBazelPackage()) {
const file = new BootstrapCacheFile(kbn, project, checksums);
const valid = options.cache && file.isValid();
if (valid) {
log.debug(`[${project.name}] cache up to date`);
cachedProjectCount += 1;
}
caches.set(project, { file, valid });
}
}
if (cachedProjectCount > 0) {
log.success(`${cachedProjectCount} bootstrap builds are cached`);
}
await parallelizeBatches(batchedNonBazelProjects, async (project) => {
const cache = caches.get(project);
if (cache && !cache.valid) {
log.info(`[${project.name}] running [kbn:bootstrap] script`);
cache.file.delete();
await project.runScriptStreaming('kbn:bootstrap');
cache.file.write();
log.success(`[${project.name}] bootstrap complete`);
}
});
// Build typescript references
await spawnStreaming(
'node',
['scripts/build_ts_refs', '--ignore-type-failures', '--info'],
{
cwd: kbn.getAbsolute(),
env: process.env,
},
{ prefix: '[ts refs]', debug: false }
);
},
};

81
packages/kbn-pm/src/utils/bootstrap_cache_file.ts
View file

@ -1,81 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Fs from 'fs';
import Path from 'path';
import { ChecksumMap } from './project_checksums';
import { Project } from '../utils/project';
import { Kibana } from '../utils/kibana';
export class BootstrapCacheFile {
private readonly path: string;
private readonly expectedValue: string | undefined;
constructor(kbn: Kibana, project: Project, checksums: ChecksumMap | false) {
this.path = Path.resolve(project.targetLocation, '.bootstrap-cache');
if (!checksums) {
return;
}
const projectAndDepCacheKeys = Array.from(kbn.getProjectAndDeps(project.name).values())
// sort deps by name so that the key is stable
.sort((a, b) => a.name.localeCompare(b.name))
// get the cacheKey for each project, return undefined if the cache key couldn't be determined
.map((p) => {
const cacheKey = checksums.get(p.name);
if (cacheKey) {
return `${p.name}:${cacheKey}`;
}
});
// if any of the relevant cache keys are undefined then the projectCacheKey must be too
this.expectedValue = projectAndDepCacheKeys.some((k) => !k)
? undefined
: [
`# this is only human readable for debugging, please don't try to parse this`,
...projectAndDepCacheKeys,
].join('\n');
}
isValid() {
if (!this.expectedValue) {
return false;
}
try {
return Fs.readFileSync(this.path, 'utf8') === this.expectedValue;
} catch (error) {
if (error.code === 'ENOENT') {
return false;
}
throw error;
}
}
delete() {
try {
Fs.unlinkSync(this.path);
} catch (error) {
if (error.code !== 'ENOENT') {
throw error;
}
}
}
write() {
if (!this.expectedValue) {
return;
}
Fs.mkdirSync(Path.dirname(this.path), { recursive: true });
Fs.writeFileSync(this.path, this.expectedValue);
}
}

231
packages/kbn-pm/src/utils/project_checksums.ts
View file

@ -1,231 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Fs from 'fs';
import Crypto from 'crypto';
import { promisify } from 'util';
import execa from 'execa';
import { YarnLock, resolveDepsForProject } from './yarn_lock';
import { ProjectMap } from '../utils/projects';
import { Project } from '../utils/project';
import { Kibana } from '../utils/kibana';
import { Log } from '../utils/log';
export type ChecksumMap = Map<string, string | undefined>;
/** map of [repo relative path to changed file, type of change] */
type Changes = Map<string, 'modified' | 'deleted' | 'invalid' | 'untracked'>;
const statAsync = promisify(Fs.stat);
const projectBySpecificitySorter = (a: Project, b: Project) => b.path.length - a.path.length;
/** Get the changed files for a set of projects */
async function getChangesForProjects(projects: ProjectMap, kbn: Kibana, log: Log) {
log.verbose('getting changed files');
const { stdout } = await execa(
'git',
[
'ls-files',
'-dmto',
'--exclude-standard',
'--',
...Array.from(projects.values())
.filter((p) => kbn.isPartOfRepo(p))
.map((p) => p.path),
],
{
cwd: kbn.getAbsolute(),
}
);
const output = stdout.trim();
const unassignedChanges: Changes = new Map();
if (output) {
for (const line of output.split('\n')) {
const [tag, ...pathParts] = line.trim().split(' ');
const path = pathParts.join(' ');
switch (tag) {
case 'M':
case 'C':
// for some reason ls-files returns deleted files as both deleted
// and modified, so make sure not to overwrite changes already
// tracked as "deleted"
if (unassignedChanges.get(path) !== 'deleted') {
unassignedChanges.set(path, 'modified');
}
break;
case 'R':
unassignedChanges.set(path, 'deleted');
break;
case '?':
unassignedChanges.set(path, 'untracked');
break;
case 'H':
case 'S':
case 'K':
default:
log.warning(`unexpected modification status "${tag}" for ${path}, please report this!`);
unassignedChanges.set(path, 'invalid');
break;
}
}
}
const sortedRelevantProjects = Array.from(projects.values()).sort(projectBySpecificitySorter);
const changesByProject = new Map<Project, Changes | undefined>();
for (const project of sortedRelevantProjects) {
if (kbn.isOutsideRepo(project)) {
changesByProject.set(project, undefined);
continue;
}
const ownChanges: Changes = new Map();
const prefix = kbn.getRelative(project.path);
for (const [path, type] of unassignedChanges) {
if (path.startsWith(prefix)) {
ownChanges.set(path, type);
unassignedChanges.delete(path);
}
}
log.verbose(`[${project.name}] found ${ownChanges.size} changes`);
changesByProject.set(project, ownChanges);
}
if (unassignedChanges.size) {
throw new Error(
`unable to assign all change paths to a project: ${JSON.stringify(
Array.from(unassignedChanges.entries())
)}`
);
}
return changesByProject;
}
/** Get the latest commit sha for a project */
async function getLatestSha(project: Project, kbn: Kibana) {
if (kbn.isOutsideRepo(project)) {
return;
}
const { stdout } = await execa(
'git',
['log', '-n', '1', '--pretty=format:%H', '--', project.path],
{
cwd: kbn.getAbsolute(),
}
);
return stdout.trim() || undefined;
}
/**
* Get the checksum for a specific project in the workspace
*/
async function getChecksum(
project: Project,
changes: Changes | undefined,
yarnLock: YarnLock,
kbn: Kibana,
log: Log
) {
const sha = await getLatestSha(project, kbn);
if (sha) {
log.verbose(`[${project.name}] local sha:`, sha);
}
if (!changes || Array.from(changes.values()).includes('invalid')) {
log.warning(`[${project.name}] unable to determine local changes, caching disabled`);
return;
}
const changesSummary = await Promise.all(
Array.from(changes)
.sort((a, b) => a[0].localeCompare(b[0]))
.map(async ([path, type]) => {
if (type === 'deleted') {
return `${path}:deleted`;
}
const stats = await statAsync(kbn.getAbsolute(path));
log.verbose(`[${project.name}] modified time ${stats.mtimeMs} for ${path}`);
return `${path}:${stats.mtimeMs}`;
})
);
const depMap = resolveDepsForProject({
project,
yarnLock,
kbn,
log,
includeDependentProject: false,
productionDepsOnly: false,
});
if (!depMap) {
return;
}
const deps = Array.from(depMap.values())
.map(({ name, version }) => `${name}@${version}`)
.sort((a, b) => a.localeCompare(b));
log.verbose(`[${project.name}] resolved %d deps`, deps.length);
const checksum = JSON.stringify(
{
sha,
changes: changesSummary,
deps,
},
null,
2
);
if (process.env.BOOTSTRAP_CACHE_DEBUG_CHECKSUM) {
return checksum;
}
const hash = Crypto.createHash('sha1');
hash.update(checksum);
return hash.digest('hex');
}
/**
* Calculate checksums for all projects in the workspace based on
* - last git commit to project directory
* - un-committed changes
* - resolved dependencies from yarn.lock referenced by project package.json
*/
export async function getAllChecksums(kbn: Kibana, log: Log, yarnLock: YarnLock) {
const projects = kbn.getAllProjects();
const changesByProject = await getChangesForProjects(projects, kbn, log);
/** map of [project.name, cacheKey] */
const cacheKeys: ChecksumMap = new Map();
await Promise.all(
Array.from(projects.values()).map(async (project) => {
cacheKeys.set(
project.name,
await getChecksum(project, changesByProject.get(project), yarnLock, kbn, log)
);
})
);
return cacheKeys;
}