mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[kbn/optimizer/node] properly separate lmdb databases, log better (#83849)
Co-authored-by: spalger <spalger@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Spencer
GitHub
parent
39291e16b5
commit
0db0d95142
5 changed files with 218 additions and 93 deletions
|
|
@ -723,7 +723,7 @@
|
|||
"less": "npm:@elastic/less@2.7.3-kibana",
|
||||
"license-checker": "^16.0.0",
|
||||
"listr": "^0.14.1",
|
||||
"lmdb-store": "^0.6.10",
|
||||
"lmdb-store": "^0.8.15",
|
||||
"load-grunt-config": "^3.0.1",
|
||||
"loader-utils": "^1.2.3",
|
||||
"log-symbols": "^2.2.0",
|
||||
|
|
|
|||
|
|
@ -17,91 +17,67 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import Fs from 'fs';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
// @ts-expect-error no types available
|
||||
import chalk from 'chalk';
|
||||
import * as LmdbStore from 'lmdb-store';
|
||||
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
|
||||
|
||||
const LMDB_PKG = JSON.parse(
|
||||
Fs.readFileSync(Path.resolve(REPO_ROOT, 'node_modules/lmdb-store/package.json'), 'utf8')
|
||||
);
|
||||
const CACHE_DIR = Path.resolve(
|
||||
REPO_ROOT,
|
||||
`data/node_auto_transpilation_cache/lmdb-${LMDB_PKG.version}/${UPSTREAM_BRANCH}`
|
||||
);
|
||||
|
||||
const reportError = () => {
|
||||
// right now I'm not sure we need to worry about errors, the cache isn't actually
|
||||
// necessary, and if the cache is broken it should just rebuild on the next restart
|
||||
// of the process. We don't know how often errors occur though and what types of
|
||||
// things might fail on different machines so we probably want some way to signal
|
||||
// to users that something is wrong
|
||||
};
|
||||
|
||||
const GLOBAL_ATIME = `${Date.now()}`;
|
||||
const MINUTE = 1000 * 60;
|
||||
const HOUR = MINUTE * 60;
|
||||
const DAY = HOUR * 24;
|
||||
|
||||
interface Lmdb<T> {
|
||||
name: string;
|
||||
get(key: string): T | undefined;
|
||||
put(key: string, value: T, version?: number, ifVersion?: number): Promise<boolean>;
|
||||
remove(key: string, ifVersion?: number): Promise<boolean>;
|
||||
removeSync(key: string): void;
|
||||
openDB<T2>(options: {
|
||||
name: string;
|
||||
encoding: 'msgpack' | 'string' | 'json' | 'binary';
|
||||
}): Lmdb<T2>;
|
||||
getRange(options?: {
|
||||
start?: T;
|
||||
end?: T;
|
||||
reverse?: boolean;
|
||||
limit?: number;
|
||||
versions?: boolean;
|
||||
}): Iterable<{ key: string; value: T }>;
|
||||
}
|
||||
const dbName = (db: LmdbStore.Database) =>
|
||||
// @ts-expect-error db.name is not a documented/typed property
|
||||
db.name;
|
||||
|
||||
export class Cache {
|
||||
private readonly codes: Lmdb<string>;
|
||||
private readonly atimes: Lmdb<string>;
|
||||
private readonly mtimes: Lmdb<string>;
|
||||
private readonly sourceMaps: Lmdb<any>;
|
||||
private readonly codes: LmdbStore.RootDatabase<string, string>;
|
||||
private readonly atimes: LmdbStore.Database<string, string>;
|
||||
private readonly mtimes: LmdbStore.Database<string, string>;
|
||||
private readonly sourceMaps: LmdbStore.Database<string, string>;
|
||||
private readonly prefix: string;
|
||||
private readonly log?: Writable;
|
||||
private readonly timer: NodeJS.Timer;
|
||||
|
||||
constructor(config: { prefix: string }) {
|
||||
constructor(config: { dir: string; prefix: string; log?: Writable }) {
|
||||
this.prefix = config.prefix;
|
||||
this.log = config.log;
|
||||
|
||||
this.codes = LmdbStore.open({
|
||||
this.codes = LmdbStore.open(config.dir, {
|
||||
name: 'codes',
|
||||
path: CACHE_DIR,
|
||||
encoding: 'string',
|
||||
maxReaders: 500,
|
||||
});
|
||||
|
||||
this.atimes = this.codes.openDB({
|
||||
// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
|
||||
this.atimes = this.codes.openDB('atimes', {
|
||||
name: 'atimes',
|
||||
encoding: 'string',
|
||||
});
|
||||
|
||||
this.mtimes = this.codes.openDB({
|
||||
this.mtimes = this.codes.openDB('mtimes', {
|
||||
name: 'mtimes',
|
||||
encoding: 'string',
|
||||
});
|
||||
|
||||
this.sourceMaps = this.codes.openDB({
|
||||
this.sourceMaps = this.codes.openDB('sourceMaps', {
|
||||
name: 'sourceMaps',
|
||||
encoding: 'msgpack',
|
||||
encoding: 'string',
|
||||
});
|
||||
|
||||
// after the process has been running for 30 minutes prune the
|
||||
// keys which haven't been used in 30 days. We use `unref()` to
|
||||
// make sure this timer doesn't hold other processes open
|
||||
// unexpectedly
|
||||
setTimeout(() => {
|
||||
this.timer = setTimeout(() => {
|
||||
this.pruneOldKeys();
|
||||
}, 30 * MINUTE).unref();
|
||||
}, 30 * MINUTE);
|
||||
|
||||
// timer.unref is not defined in jest which emulates the dom by default
|
||||
if (typeof this.timer.unref === 'function') {
|
||||
this.timer.unref();
|
||||
}
|
||||
}
|
||||
|
||||
getMtime(path: string) {
|
||||
|
|
@ -110,45 +86,78 @@ export class Cache {
|
|||
|
||||
getCode(path: string) {
|
||||
const key = this.getKey(path);
|
||||
const code = this.safeGet(this.codes, key);
|
||||
|
||||
// when we use a file from the cache set the "atime" of that cache entry
|
||||
// so that we know which cache items we use and which haven't been
|
||||
// touched in a long time (currently 30 days)
|
||||
this.atimes.put(key, GLOBAL_ATIME).catch(reportError);
|
||||
if (code !== undefined) {
|
||||
// when we use a file from the cache set the "atime" of that cache entry
|
||||
// so that we know which cache items we use and which haven't been
|
||||
// touched in a long time (currently 30 days)
|
||||
this.safePut(this.atimes, key, GLOBAL_ATIME);
|
||||
}
|
||||
|
||||
return this.safeGet(this.codes, key);
|
||||
return code;
|
||||
}
|
||||
|
||||
getSourceMap(path: string) {
|
||||
return this.safeGet(this.sourceMaps, this.getKey(path));
|
||||
const map = this.safeGet(this.sourceMaps, this.getKey(path));
|
||||
if (typeof map === 'string') {
|
||||
return JSON.parse(map);
|
||||
}
|
||||
}
|
||||
|
||||
update(path: string, file: { mtime: string; code: string; map: any }) {
|
||||
async update(path: string, file: { mtime: string; code: string; map: any }) {
|
||||
const key = this.getKey(path);
|
||||
|
||||
Promise.all([
|
||||
this.atimes.put(key, GLOBAL_ATIME),
|
||||
this.mtimes.put(key, file.mtime),
|
||||
this.codes.put(key, file.code),
|
||||
this.sourceMaps.put(key, file.map),
|
||||
]).catch(reportError);
|
||||
await Promise.all([
|
||||
this.safePut(this.atimes, key, GLOBAL_ATIME),
|
||||
this.safePut(this.mtimes, key, file.mtime),
|
||||
this.safePut(this.codes, key, file.code),
|
||||
this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
|
||||
]);
|
||||
}
|
||||
|
||||
close() {
|
||||
clearTimeout(this.timer);
|
||||
}
|
||||
|
||||
private getKey(path: string) {
|
||||
return `${this.prefix}${path}`;
|
||||
}
|
||||
|
||||
private safeGet<V>(db: Lmdb<V>, key: string) {
|
||||
private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {
|
||||
try {
|
||||
return db.get(key);
|
||||
const value = db.get(key);
|
||||
this.debug(value === undefined ? 'MISS' : 'HIT', db, key);
|
||||
return value;
|
||||
} catch (error) {
|
||||
process.stderr.write(
|
||||
`failed to read node transpilation [${db.name}] cache for [${key}]: ${error.stack}\n`
|
||||
);
|
||||
db.removeSync(key);
|
||||
this.logError('GET', db, key, error);
|
||||
}
|
||||
}
|
||||
|
||||
private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
|
||||
try {
|
||||
await db.put(key, value);
|
||||
this.debug('PUT', db, key);
|
||||
} catch (error) {
|
||||
this.logError('PUT', db, key, error);
|
||||
}
|
||||
}
|
||||
|
||||
private debug(type: string, db: LmdbStore.Database, key: LmdbStore.Key) {
|
||||
if (this.log) {
|
||||
this.log.write(`${type} [${dbName(db)}] ${String(key)}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
private logError(type: 'GET' | 'PUT', db: LmdbStore.Database, key: LmdbStore.Key, error: Error) {
|
||||
this.debug(`ERROR/${type}`, db, `${String(key)}: ${error.stack}`);
|
||||
process.stderr.write(
|
||||
chalk.red(
|
||||
`[@kbn/optimizer/node] ${type} error [${dbName(db)}/${String(key)}]: ${error.stack}\n`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private async pruneOldKeys() {
|
||||
try {
|
||||
const ATIME_LIMIT = Date.now() - 30 * DAY;
|
||||
|
|
@ -157,9 +166,10 @@ export class Cache {
|
|||
const validKeys: string[] = [];
|
||||
const invalidKeys: string[] = [];
|
||||
|
||||
// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
|
||||
for (const { key, value } of this.atimes.getRange()) {
|
||||
const atime = parseInt(value, 10);
|
||||
if (atime < ATIME_LIMIT) {
|
||||
const atime = parseInt(`${value}`, 10);
|
||||
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
|
||||
invalidKeys.push(key);
|
||||
} else {
|
||||
validKeys.push(key);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
import del from 'del';
|
||||
|
||||
import { Cache } from '../cache';
|
||||
|
||||
const DIR = Path.resolve(__dirname, '../__tmp__/cache');
|
||||
|
||||
const makeTestLog = () => {
|
||||
const log = Object.assign(
|
||||
new Writable({
|
||||
write(chunk, enc, cb) {
|
||||
log.output += chunk;
|
||||
cb();
|
||||
},
|
||||
}),
|
||||
{
|
||||
output: '',
|
||||
}
|
||||
);
|
||||
|
||||
return log;
|
||||
};
|
||||
|
||||
const instances: Cache[] = [];
|
||||
const makeCache = (...options: ConstructorParameters<typeof Cache>) => {
|
||||
const instance = new Cache(...options);
|
||||
instances.push(instance);
|
||||
return instance;
|
||||
};
|
||||
|
||||
beforeEach(async () => await del(DIR));
|
||||
afterEach(async () => {
|
||||
await del(DIR);
|
||||
for (const instance of instances) {
|
||||
instance.close();
|
||||
}
|
||||
instances.length = 0;
|
||||
});
|
||||
|
||||
it('returns undefined until values are set', async () => {
|
||||
const path = '/foo/bar.js';
|
||||
const mtime = new Date().toJSON();
|
||||
const log = makeTestLog();
|
||||
const cache = makeCache({
|
||||
dir: DIR,
|
||||
prefix: 'foo',
|
||||
log,
|
||||
});
|
||||
|
||||
expect(cache.getMtime(path)).toBe(undefined);
|
||||
expect(cache.getCode(path)).toBe(undefined);
|
||||
expect(cache.getSourceMap(path)).toBe(undefined);
|
||||
|
||||
await cache.update(path, {
|
||||
mtime,
|
||||
code: 'var x = 1',
|
||||
map: { foo: 'bar' },
|
||||
});
|
||||
|
||||
expect(cache.getMtime(path)).toBe(mtime);
|
||||
expect(cache.getCode(path)).toBe('var x = 1');
|
||||
expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' });
|
||||
expect(log.output).toMatchInlineSnapshot(`
|
||||
"MISS [mtimes] foo/foo/bar.js
|
||||
MISS [codes] foo/foo/bar.js
|
||||
MISS [sourceMaps] foo/foo/bar.js
|
||||
PUT [atimes] foo/foo/bar.js
|
||||
PUT [mtimes] foo/foo/bar.js
|
||||
PUT [codes] foo/foo/bar.js
|
||||
PUT [sourceMaps] foo/foo/bar.js
|
||||
HIT [mtimes] foo/foo/bar.js
|
||||
HIT [codes] foo/foo/bar.js
|
||||
HIT [sourceMaps] foo/foo/bar.js
|
||||
"
|
||||
`);
|
||||
});
|
||||
|
|
@ -39,7 +39,7 @@ import Crypto from 'crypto';
|
|||
|
||||
import * as babel from '@babel/core';
|
||||
import { addHook } from 'pirates';
|
||||
import { REPO_ROOT } from '@kbn/dev-utils';
|
||||
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
|
||||
import sourceMapSupport from 'source-map-support';
|
||||
|
||||
import { Cache } from './cache';
|
||||
|
|
@ -134,7 +134,13 @@ export function registerNodeAutoTranspilation() {
|
|||
installed = true;
|
||||
|
||||
const cache = new Cache({
|
||||
dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache', UPSTREAM_BRANCH),
|
||||
prefix: determineCachePrefix(),
|
||||
log: process.env.DEBUG_NODE_TRANSPILER_CACHE
|
||||
? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), {
|
||||
flags: 'a',
|
||||
})
|
||||
: undefined,
|
||||
});
|
||||
|
||||
sourceMapSupport.install({
|
||||
|
|
|
|||
50
yarn.lock
50
yarn.lock
|
|
@ -18805,16 +18805,28 @@ livereload-js@^2.3.0:
|
|||
resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.4.0.tgz#447c31cf1ea9ab52fc20db615c5ddf678f78009c"
|
||||
integrity sha512-XPQH8Z2GDP/Hwz2PCDrh2mth4yFejwA1OZ/81Ti3LgKyhDcEjsSsqFWZojHG0va/duGd+WyosY7eXLDoOyqcPw==
|
||||
|
||||
lmdb-store@^0.6.10:
|
||||
version "0.6.10"
|
||||
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.6.10.tgz#db8efde6e052aabd17ebc63c8a913e1f31694129"
|
||||
integrity sha512-ZLvp3qbBQ5VlBmaWa4EUAPyYEZ8qdUHsW69HmxkDi84pFQ37WMxYhFaF/7PQkdtxS/vyiKkZigd9TFgHjek1Nw==
|
||||
lmdb-store-0.9@0.7.3:
|
||||
version "0.7.3"
|
||||
resolved "https://registry.yarnpkg.com/lmdb-store-0.9/-/lmdb-store-0.9-0.7.3.tgz#c2cb27dfa916ab966cceed692c67e4236813104a"
|
||||
integrity sha512-t8iCnN6T3NZPFelPmjYIjCg+nhGbOdc0xcHCG40v01AWRTN49OINSt2k/u+16/2/HrI+b6Ssb8WByXUhbyHz6w==
|
||||
dependencies:
|
||||
fs-extra "^9.0.1"
|
||||
msgpackr "^0.5.0"
|
||||
msgpackr "^0.5.3"
|
||||
nan "^2.14.1"
|
||||
node-gyp-build "^4.2.3"
|
||||
weak-lru-cache "^0.2.0"
|
||||
weak-lru-cache "^0.3.9"
|
||||
|
||||
lmdb-store@^0.8.15:
|
||||
version "0.8.15"
|
||||
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.8.15.tgz#4efb0341c2df505dd6f3a7f26f834f0a142a80a2"
|
||||
integrity sha512-4Q0WZh2FmcJC6esZRUWMfkCmNiz0WU9cOgrxt97ZMTnVfHyOdZhtrt0oOF5EQPfetxxJf/BorKY28aX92R6G6g==
|
||||
dependencies:
|
||||
fs-extra "^9.0.1"
|
||||
lmdb-store-0.9 "0.7.3"
|
||||
msgpackr "^0.5.4"
|
||||
nan "^2.14.1"
|
||||
node-gyp-build "^4.2.3"
|
||||
weak-lru-cache "^0.3.9"
|
||||
|
||||
load-bmfont@^1.3.1, load-bmfont@^1.4.0:
|
||||
version "1.4.0"
|
||||
|
|
@ -20330,20 +20342,20 @@ ms@2.1.1, ms@^2.0.0, ms@^2.1.1:
|
|||
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
|
||||
integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
|
||||
|
||||
msgpackr-extract@^0.3.4:
|
||||
version "0.3.4"
|
||||
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.4.tgz#8ee5e73d1135340e564c498e8c593134365eb060"
|
||||
integrity sha512-d3+qwTJzgqqsq2L2sQuH0SoO4StvpUhMqMAKy6tMimn7XdBaRtDlquFzRJsp0iMGt2hnU4UOqD8Tz9mb0KglTA==
|
||||
msgpackr-extract@^0.3.5:
|
||||
version "0.3.5"
|
||||
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.5.tgz#0f206da058bd3dad0f8605d324de001a8f4de967"
|
||||
integrity sha512-zHhstybu+m/j3H6CVBMcILVIzATK6dWRGtlePJjsnSAj8kLT5joMa9i0v21Uc80BPNDcwFsnG/dz2318tfI81w==
|
||||
dependencies:
|
||||
nan "^2.14.1"
|
||||
node-gyp-build "^4.2.3"
|
||||
|
||||
msgpackr@^0.5.0:
|
||||
version "0.5.1"
|
||||
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.1.tgz#7eecbf342645b7718dd2e3386894368d06732b3f"
|
||||
integrity sha512-nK2uJl67Q5KU3MWkYBUlYynqKS1UUzJ5M1h6TQejuJtJzD3hW2Suv2T1pf01E9lUEr93xaLokf/xC+jwBShMPQ==
|
||||
msgpackr@^0.5.3, msgpackr@^0.5.4:
|
||||
version "0.5.4"
|
||||
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.4.tgz#c21c03d5e132d2e54d0b9ced02a75b1f48413380"
|
||||
integrity sha512-ILEWtIWwd5ESWHKoVjJ4GP7JWkpuAUJ20qi2j2qEC6twecBmK4E6YG3QW847OpmvdAhMJGq2LoDJRn/kNERTeQ==
|
||||
optionalDependencies:
|
||||
msgpackr-extract "^0.3.4"
|
||||
msgpackr-extract "^0.3.5"
|
||||
|
||||
multicast-dns-service-types@^1.1.0:
|
||||
version "1.1.0"
|
||||
|
|
@ -29142,10 +29154,10 @@ wcwidth@^1.0.1:
|
|||
dependencies:
|
||||
defaults "^1.0.3"
|
||||
|
||||
weak-lru-cache@^0.2.0:
|
||||
version "0.2.0"
|
||||
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.2.0.tgz#447379ccff6dfda1b7a9566c9ef168260be859d1"
|
||||
integrity sha512-M1l5CzKvM7maa7tCbtL0NW6sOnp8gqup853+9Aq7GL0XNWKNnFOkeE3v3Z5X2IeMzedPwQyPbi4RlFvD6rxs7A==
|
||||
weak-lru-cache@^0.3.9:
|
||||
version "0.3.9"
|
||||
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.3.9.tgz#9e56920d4115e8542625d8ef8cc278cbd97f7624"
|
||||
integrity sha512-WqAu3wzbHQvjSi/vgYhidZkf2p7L3Z8iDEIHnqvE31EQQa7Vh7PDOphrRJ1oxlW8JIjgr2HvMcRe9Q1GhW2NPw==
|
||||
|
||||
web-namespaces@^1.0.0:
|
||||
version "1.1.4"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue