github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[dev/build] scan node_modules rather than lots of deleteAll() calls (#24692) (#24774)

Browse source
This commit is contained in:
Spencer 2018-10-29 18:18:05 -07:00 • committed by GitHub
parent 2cffddc237
commit ecc24b27f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 272 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -249,6 +249,7 @@
"@types/classnames": "^2.2.3",
"@types/d3": "^5.0.0",
"@types/dedent": "^0.7.0",
"@types/del": "^3.0.1",
"@types/elasticsearch": "^5.0.26",
"@types/enzyme": "^3.1.12",
"@types/eslint": "^4.16.2",

2
src/dev/build/lib/fs.js
View file

@ -39,7 +39,7 @@ const readFileAsync = promisify(fs.readFile);
const readdirAsync = promisify(fs.readdir);
const utimesAsync = promisify(fs.utimes);
function assertAbsolute(path) {
export function assertAbsolute(path) {
if (!isAbsolute(path)) {
throw new TypeError(
'Please use absolute paths to keep things explicit. You probably want to use `build.resolvePath()` or `config.resolveFromRepo()`.'

1
src/dev/build/lib/index.js
View file

@ -31,3 +31,4 @@ export {
untar,
deleteAll,
} from './fs';
export { scanDelete } from './scan_delete';

64
src/dev/build/lib/scan_delete.test.ts Normal file
View file

@ -0,0 +1,64 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { readdirSync } from 'fs';
import { relative, resolve } from 'path';
import del from 'del';
// @ts-ignore
import { mkdirp, write } from './fs';
import { scanDelete } from './scan_delete';
const TMP = resolve(__dirname, '__tests__/__tmp__');
// clean and recreate TMP directory
beforeEach(async () => {
await del(TMP);
await mkdirp(resolve(TMP, 'foo/bar/baz'));
await mkdirp(resolve(TMP, 'foo/bar/box'));
await mkdirp(resolve(TMP, 'a/b/c/d/e'));
await write(resolve(TMP, 'a/bar'), 'foo');
});
// cleanup TMP directory
afterAll(async () => {
await del(TMP);
});
it('requires an absolute directory', async () => {
await expect(
scanDelete({
directory: relative(process.cwd(), TMP),
regularExpressions: [],
})
).rejects.toMatchInlineSnapshot(
`[TypeError: Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`.]`
);
});
it('deletes files/folders matching regular expression', async () => {
await scanDelete({
directory: TMP,
regularExpressions: [/^.*[\/\\](bar|c)([\/\\]|$)/],
});
expect(readdirSync(resolve(TMP, 'foo'))).toEqual([]);
expect(readdirSync(resolve(TMP, 'a'))).toEqual(['b']);
expect(readdirSync(resolve(TMP, 'a/b'))).toEqual([]);
});

80
src/dev/build/lib/scan_delete.ts Normal file
View file

@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Fs from 'fs';
import del from 'del';
import { join } from 'path';
import * as Rx from 'rxjs';
import { count, map, mergeAll, mergeMap } from 'rxjs/operators';
// @ts-ignore
import { assertAbsolute } from './fs';
const getStat$ = Rx.bindNodeCallback(Fs.stat);
const getReadDir$ = Rx.bindNodeCallback(Fs.readdir);
interface Options {
directory: string;
regularExpressions: RegExp[];
concurrency?: 20;
}
/**
* Scan the files in a directory and delete the directories/files that
* are matched by an array of regular expressions.
*
* @param options.directory the directory to scan, all files including dot files will be checked
* @param options.regularExpressions an array of regular expressions, if any matches the file/directory will be deleted
* @param options.concurrency optional concurrency to run deletes, defaults to 20
*/
export async function scanDelete(options: Options) {
const { directory, regularExpressions, concurrency = 20 } = options;
assertAbsolute(directory);
// get an observable of absolute paths within a directory
const getChildPath$ = (path: string) =>
getReadDir$(path).pipe(
mergeAll(),
map(name => join(path, name))
);
// get an observable of all paths to be deleted, by starting with the arg
// and recursively iterating through all children, unless a child matches
// one of the supplied regular expressions
const getPathsToDelete$ = (path: string): Rx.Observable<string> => {
if (regularExpressions.some(re => re.test(path))) {
return Rx.of(path);
}
return getStat$(path).pipe(
mergeMap(stat => (stat.isDirectory() ? getChildPath$(path) : Rx.EMPTY)),
mergeMap(getPathsToDelete$)
);
};
return await Rx.of(directory)
.pipe(
mergeMap(getPathsToDelete$),
mergeMap(async path => await del(path), concurrency),
count()
)
.toPromise();
}

199
src/dev/build/tasks/clean_tasks.js
View file

@ -17,7 +17,9 @@
* under the License.
*/
import { deleteAll } from '../lib';
import minimatch from 'minimatch';
import { deleteAll, scanDelete } from '../lib';
export const CleanTask = {
global: true,
@ -49,10 +51,13 @@ export const CleanTypescriptTask = {
'Cleaning typescript source files that have been transpiled to JS',
async run(config, log, build) {
await deleteAll(log, [
build.resolvePath('**/*.{ts,tsx,d.ts}'),
build.resolvePath('**/tsconfig*.json'),
]);
log.info('Deleted %d files', await scanDelete({
directory: build.resolvePath(),
regularExpressions: [
/\.(ts|tsx|d\.ts)$/,
/tsconfig.*\.json$/
]
}));
},
};
@ -60,94 +65,108 @@ export const CleanExtraFilesFromModulesTask = {
description: 'Cleaning tests, examples, docs, etc. from node_modules',
async run(config, log, build) {
const deleteFromNodeModules = globs => {
return deleteAll(
log,
globs.map(p => build.resolvePath(`node_modules/**/${p}`))
const makeRegexps = patterns =>
patterns.map(pattern =>
minimatch.makeRe(pattern, { nocase: true })
);
};
const tests = [
'test',
'tests',
'__tests__',
'mocha.opts',
'*.test.js',
'*.snap',
'coverage',
];
const docs = [
'doc',
'docs',
'CONTRIBUTING.md',
'Contributing.md',
'contributing.md',
'History.md',
'HISTORY.md',
'history.md',
'CHANGELOG.md',
'Changelog.md',
'changelog.md',
];
const examples = ['example', 'examples', 'demo', 'samples'];
const bins = ['.bin'];
const linters = [
'.eslintrc',
'.eslintrc.js',
'.eslintrc.yml',
'.prettierrc',
'.jshintrc',
'.babelrc',
'.jscs.json',
'.lint',
];
const hints = ['*.flow', '*.webidl', '*.map', '@types'];
const scripts = [
'*.sh',
'*.bat',
'*.exe',
'Gruntfile.js',
'gulpfile.js',
'Makefile',
];
const untranspiledSources = ['*.coffee', '*.scss', '*.sass', '.ts', '.tsx'];
const editors = ['.editorconfig', '.vscode'];
const git = [
'.gitattributes',
'.gitkeep',
'.gitempty',
'.gitmodules',
'.keep',
'.empty',
];
const ci = [
'.travis.yml',
'.coveralls.yml',
'.instanbul.yml',
'appveyor.yml',
'.zuul.yml',
];
const meta = [
'package-lock.json',
'component.json',
'bower.json',
'yarn.lock',
];
const misc = ['.*ignore', '.DS_Store', 'Dockerfile', 'docker-compose.yml'];
log.info('Deleted %d files', await scanDelete({
directory: build.resolvePath('node_modules'),
regularExpressions: makeRegexps([
// tests
'**/test',
'**/tests',
'**/__tests__',
'**/mocha.opts',
'**/*.test.js',
'**/*.snap',
'**/coverage',
await deleteFromNodeModules(tests);
await deleteFromNodeModules(docs);
await deleteFromNodeModules(examples);
await deleteFromNodeModules(bins);
await deleteFromNodeModules(linters);
await deleteFromNodeModules(hints);
await deleteFromNodeModules(scripts);
await deleteFromNodeModules(untranspiledSources);
await deleteFromNodeModules(editors);
await deleteFromNodeModules(git);
await deleteFromNodeModules(ci);
await deleteFromNodeModules(meta);
await deleteFromNodeModules(misc);
// docs
'**/doc',
'**/docs',
'**/CONTRIBUTING.md',
'**/Contributing.md',
'**/contributing.md',
'**/History.md',
'**/HISTORY.md',
'**/history.md',
'**/CHANGELOG.md',
'**/Changelog.md',
'**/changelog.md',
// examples
'**/example',
'**/examples',
'**/demo',
'**/samples',
// bins
'**/.bin',
// linters
'**/.eslintrc',
'**/.eslintrc.js',
'**/.eslintrc.yml',
'**/.prettierrc',
'**/.jshintrc',
'**/.babelrc',
'**/.jscs.json',
'**/.lint',
// hints
'**/*.flow',
'**/*.webidl',
'**/*.map',
'**/@types',
// scripts
'**/*.sh',
'**/*.bat',
'**/*.exe',
'**/Gruntfile.js',
'**/gulpfile.js',
'**/Makefile',
// untranspiled sources
'**/*.coffee',
'**/*.scss',
'**/*.sass',
'**/.ts',
'**/.tsx',
// editors
'**/.editorconfig',
'**/.vscode',
// git
'**/.gitattributes',
'**/.gitkeep',
'**/.gitempty',
'**/.gitmodules',
'**/.keep',
'**/.empty',
// ci
'**/.travis.yml',
'**/.coveralls.yml',
'**/.instanbul.yml',
'**/appveyor.yml',
'**/.zuul.yml',
// metadata
'**/package-lock.json',
'**/component.json',
'**/bower.json',
'**/yarn.lock',
// misc
'**/.*ignore',
'**/.DS_Store',
'**/Dockerfile',
'**/docker-compose.yml'
])
}));
},
};

16
yarn.lock
View file

@ -651,6 +651,13 @@
resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050"
integrity sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A==
"@types/del@^3.0.1":
version "3.0.1"
resolved "https://registry.yarnpkg.com/@types/del/-/del-3.0.1.tgz#4712da8c119873cbbf533ad8dbf1baac5940ac5d"
integrity sha512-y6qRq6raBuu965clKgx6FHuiPu3oHdtmzMPXi8Uahsjdq1L6DL5fS/aY5/s71YwM7k6K1QIWvem5vNwlnNGIkQ==
dependencies:
"@types/glob" "*"
"@types/delay@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@types/delay/-/delay-2.0.1.tgz#61bcf318a74b61e79d1658fbf054f984c90ef901"
@ -716,6 +723,15 @@
resolved "https://registry.yarnpkg.com/@types/getopts/-/getopts-2.0.0.tgz#8a603370cb367d3192bd8012ad39ab2320b5b476"
integrity sha512-/WJ73/6+Ffulo6LDm0P11Y0uGDaitJBJyVhXr4Eg+/Bqi0epRLOnGDNOgplhMBFy7NLBMlZ5UQcukSABqaV5Kg==
"@types/glob@*":
version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
dependencies:
"@types/events" "*"
"@types/minimatch" "*"
"@types/node" "*"
"@types/glob@^5.0.35":
version "5.0.35"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-5.0.35.tgz#1ae151c802cece940443b5ac246925c85189f32a"