[pkgs/peggy] automatically transform peggy files with babel-register and webpack (#145615)

In order to get us closer to the developer experience we want for
packages, we are trying to move package builds out of bazel and instead
we want to build files on demand. In the case of .peggy files this means
importing them directly and teaching babel/jest/webpack how to handle
these imports by automatically transpiling and caching the results.

This change does just that, adding a `@kbn/peggy` package which wraps
peggy for types, and also adds support for defining peggy config
adjacent to a peggy grammar file in a `${basename}.config.json` file.
This file will be parsed and used to configure things like
`allowedStartRules` as described in [the peggy
docs](https://peggyjs.org/documentation.html#generating-a-parser-javascript-api).

This PR also implements `@kbn/peggy-loader` which uses `@kbn/peggy` to
transpile peggy files in webpack, and a peggy transform for both Jest
and our custom babel register hook.

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Spencer 2022-11-22 11:25:50 -07:00 committed by GitHub
parent cffaa60bd1
commit cfdb8553ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
70 changed files with 1245 additions and 282 deletions

3
.github/CODEOWNERS vendored
View file

@ -887,6 +887,7 @@ packages/home/sample_data_tab @elastic/kibana-global-experience
packages/home/sample_data_types @elastic/kibana-global-experience
packages/kbn-ace @elastic/platform-deployment-management
packages/kbn-alerts @elastic/security-solution
packages/kbn-ambient-common-types @elastic/kibana-operations
packages/kbn-ambient-storybook-types @elastic/kibana-operations
packages/kbn-ambient-ui-types @elastic/kibana-operations
packages/kbn-analytics @elastic/kibana-core
@ -958,6 +959,8 @@ packages/kbn-monaco @elastic/kibana-app-services
packages/kbn-optimizer @elastic/kibana-operations
packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations
packages/kbn-osquery-io-ts-types @elastic/security-asset-management
packages/kbn-peggy @elastic/kibana-operations
packages/kbn-peggy-loader @elastic/kibana-operations
packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing
packages/kbn-plugin-discovery @elastic/kibana-operations
packages/kbn-plugin-generator @elastic/kibana-operations

View file

@ -64,6 +64,7 @@ layout: landing
{ pageId: "kibDevDocsOpsExpect" },
{ pageId: "kibDevDocsOpsAmbientStorybookTypes" },
{ pageId: "kibDevDocsOpsAmbientUiTypes" },
{ pageId: "kibDevDocsOpsAmbientCommonTypes" },
{ pageId: "kibDevDocsOpsTestSubjSelector" },
{ pageId: "kibDevDocsOpsBazelRunner" },
{ pageId: "kibDevDocsOpsCliDevMode" },
@ -77,5 +78,7 @@ layout: landing
{ pageId: "kibDevDocsOpsManagedVscodeConfigCli" },
{ pageId: "kibDevDocsOpsTest" },
{ pageId: "kibDevDocsOpsEsArchiver" },
{ pageId: "kibDevDocsOpsPeggy" },
{ pageId: "kibDevDocsOpsPeggyLoader" },
]}
/>

View file

@ -718,7 +718,9 @@
"@istanbuljs/schema": "^0.1.2",
"@jest/console": "^29.3.1",
"@jest/reporters": "^29.3.1",
"@jest/transform": "^29.3.1",
"@jest/types": "^29.3.1",
"@kbn/ambient-common-types": "link:bazel-bin/packages/kbn-ambient-common-types",
"@kbn/ambient-storybook-types": "link:bazel-bin/packages/kbn-ambient-storybook-types",
"@kbn/ambient-ui-types": "link:bazel-bin/packages/kbn-ambient-ui-types",
"@kbn/apm-synthtrace": "link:bazel-bin/packages/kbn-apm-synthtrace",
@ -757,6 +759,8 @@
"@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli",
"@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer",
"@kbn/optimizer-webpack-helpers": "link:bazel-bin/packages/kbn-optimizer-webpack-helpers",
"@kbn/peggy": "link:bazel-bin/packages/kbn-peggy",
"@kbn/peggy-loader": "link:bazel-bin/packages/kbn-peggy-loader",
"@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor",
"@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator",
"@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers",
@ -1060,7 +1064,7 @@
"jsondiffpatch": "0.4.1",
"license-checker": "^25.0.1",
"listr": "^0.14.1",
"lmdb-store": "^1.6.11",
"lmdb-store": "^1",
"loader-utils": "^2.0.4",
"marge": "^1.0.1",
"micromatch": "^4.0.5",

View file

@ -194,6 +194,7 @@ filegroup(
"//packages/home/sample_data_types:build",
"//packages/kbn-ace:build",
"//packages/kbn-alerts:build",
"//packages/kbn-ambient-common-types:build",
"//packages/kbn-ambient-storybook-types:build",
"//packages/kbn-ambient-ui-types:build",
"//packages/kbn-analytics:build",
@ -265,6 +266,8 @@ filegroup(
"//packages/kbn-optimizer:build",
"//packages/kbn-optimizer-webpack-helpers:build",
"//packages/kbn-osquery-io-ts-types:build",
"//packages/kbn-peggy:build",
"//packages/kbn-peggy-loader:build",
"//packages/kbn-performance-testing-dataset-extractor:build",
"//packages/kbn-plugin-discovery:build",
"//packages/kbn-plugin-generator:build",
@ -617,6 +620,8 @@ filegroup(
"//packages/kbn-optimizer:build_types",
"//packages/kbn-optimizer-webpack-helpers:build_types",
"//packages/kbn-osquery-io-ts-types:build_types",
"//packages/kbn-peggy:build_types",
"//packages/kbn-peggy-loader:build_types",
"//packages/kbn-performance-testing-dataset-extractor:build_types",
"//packages/kbn-plugin-discovery:build_types",
"//packages/kbn-plugin-generator:build_types",

View file

@ -0,0 +1,58 @@
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "pkg_npm")
PKG_DIRNAME = "kbn-ambient-common-types"
PKG_REQUIRE_NAME = "@kbn/ambient-common-types"
SRCS = glob(
[
"*.d.ts",
]
)
filegroup(
name = "srcs",
srcs = SRCS,
)
NPM_MODULE_EXTRA_FILES = [
"package.json",
]
# In this array place runtime dependencies, including other packages and NPM packages
# which must be available for this code to run.
#
# To reference other packages use:
# "//repo/relative/path/to/package"
# eg. "//packages/kbn-utils"
#
# To reference a NPM package use:
# "@npm//name-of-package"
# eg. "@npm//lodash"
RUNTIME_DEPS = [
]
js_library(
name = PKG_DIRNAME,
srcs = SRCS + NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS,
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)
pkg_npm(
name = "npm_module",
deps = [":" + PKG_DIRNAME],
)
filegroup(
name = "build",
srcs = [":npm_module"],
visibility = ["//visibility:public"],
)
alias(
name = "npm_module_types",
actual = ":" + PKG_DIRNAME,
visibility = ["//visibility:public"],
)

View file

@ -0,0 +1,20 @@
---
id: kibDevDocsOpsAmbientCommonTypes
slug: /kibana-dev-docs/ops/ambient-common-types
title: "@kbn/ambient-common-types"
description: A package holding ambient type definitions for files that are expected to run on the server and the browser
date: 2022-05-18
tags: ['kibana', 'dev', 'contributor', 'operations', 'ambient', 'ui', 'common', 'server', 'types']
---
This package holds ambient typescript definitions which should be included in projects which are expected to run on the server and the browser.
## Plugins
These types will automatically be included for plugins.
## Packages
To include these types in a package:
- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file.
- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file.

View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/**
* These ambient types are used to define default types for anything which is
* supported in both server/browser environments.
*/
/**
* peggy grammars are built automatically on import in both browser/server
*/
declare module '*.peggy' {
export interface ParserOptions {
[key: string]: any;
/**
* Object that will be attached to the each `LocationRange` object created by
* the parser. For example, this can be path to the parsed file or even the
* File object.
*/
grammarSource?: any;
startRule?: string;
tracer?: ParserTracer;
}
/**
* parse `input` using the peggy grammer
* @param input code to parse
* @param options parse options
*/
export function parse(input: string, options?: ParserOptions): any;
}

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
module.exports = {
preset: '@kbn/test/jest_node',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-ambient-common-types'],
};

View file

@ -0,0 +1,8 @@
{
"type": "shared-common",
"id": "@kbn/ambient-common-types",
"owner": "@elastic/kibana-operations",
"devOnly": true,
"runtimeDeps": [],
"typeDeps": [],
}

View file

@ -0,0 +1,8 @@
{
"name": "@kbn/ambient-common-types",
"private": true,
"version": "1.0.0",
"main": "./target_node/index.js",
"types": "./target_types/index.d.ts",
"license": "SSPL-1.0 OR Elastic License 2.0"
}

View file

@ -0,0 +1,15 @@
{
"extends": "../../tsconfig.bazel.json",
"compilerOptions": {
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "target_types",
"types": [
"jest",
"node"
]
},
"include": [
"**/*.ts",
]
}

View file

@ -1,5 +1,4 @@
load("@npm//@bazel/typescript:index.bzl", "ts_config")
load("@npm//peggy:index.bzl", "peggy")
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
@ -9,6 +8,8 @@ PKG_REQUIRE_NAME = "@kbn/es-query"
SOURCE_FILES = glob(
[
"**/*.ts",
"**/grammar.peggy.config.json",
"**/grammar.peggy",
],
exclude = [
"**/*.config.js",
@ -51,6 +52,7 @@ RUNTIME_DEPS = [
TYPES_DEPS = [
"//packages/kbn-utility-types:npm_module_types",
"//packages/kbn-i18n:npm_module_types",
"//packages/kbn-ambient-common-types:npm_module_types",
"@npm//@elastic/elasticsearch",
"@npm//tslib",
"@npm//@types/jest",
@ -59,26 +61,14 @@ TYPES_DEPS = [
"@npm//@types/node",
]
peggy(
name = "grammar",
data = [
":grammar/grammar.peggy",
":grammar/grammar.config.json"
],
output_dir = True,
args = [
"--extra-options-file",
"./%s/grammar/grammar.config.json" % package_name(),
"-o",
"$(@D)/built_grammar.js",
"./%s/grammar/grammar.peggy" % package_name()
],
)
jsts_transpiler(
name = "target_node",
srcs = SRCS,
build_pkg_name = package_name(),
additional_args = [
"--copy-files",
"--quiet"
],
)
jsts_transpiler(
@ -86,6 +76,10 @@ jsts_transpiler(
srcs = SRCS,
build_pkg_name = package_name(),
web = True,
additional_args = [
"--copy-files",
"--quiet"
],
)
ts_config(
@ -110,7 +104,7 @@ ts_project(
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":target_web"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
@ -118,7 +112,7 @@ js_library(
js_library(
name = "npm_module_types",
srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],

View file

@ -6,4 +6,4 @@
* Side Public License, v 1.
*/
export { parse } from '../../../../grammar/built_grammar.js';
export { parse } from './grammar.peggy';

View file

@ -6,7 +6,8 @@
"outDir": "target_types",
"types": [
"jest",
"node"
"node",
"@kbn/ambient-common-types"
]
},
"include": [

View file

@ -125,14 +125,10 @@ export class ImportResolver {
return true;
}
// ignore requests to grammar/built_grammar.js files or bazel target dirs, these files are only
// available in the build output and will never resolve in dev. We will validate that people don't
// import these files from outside the package in another rule
if (
req.endsWith('grammar/built_grammar.js') ||
req.includes('/target_workers/') ||
req.includes('/target_node/')
) {
// ignore requests to bazel target dirs, these files are only available in the build output
// and will never resolve in dev. We will validate that people don't import these files from
// outside the package in another rule
if (req.includes('/target_workers/') || req.includes('/target_node/')) {
return true;
}

View file

@ -77,12 +77,6 @@ describe('#resolve()', () => {
});
it('returns ignore results for known unresolvable but okay import statements', () => {
expect(resolver.resolve('../../grammar/built_grammar.js', FIXTURES_DIR)).toMatchInlineSnapshot(`
Object {
"type": "ignore",
}
`);
expect(resolver.resolve('kibana-buildkite-library', FIXTURES_DIR)).toMatchInlineSnapshot(`
Object {
"type": "ignore",

View file

@ -1,5 +1,4 @@
load("@npm//@bazel/typescript:index.bzl", "ts_config")
load("@npm//peggy:index.bzl", "peggy")
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
@ -10,6 +9,8 @@ SOURCE_FILES = glob(
[
"**/*.ts",
"**/*.js",
"**/grammar.peggy.config.json",
"**/grammar.peggy",
],
exclude = [
"**/*.config.js",
@ -48,12 +49,17 @@ TYPES_DEPS = [
"@npm//@types/jest",
"@npm//@types/lodash",
"@npm//@types/node",
"//packages/kbn-ambient-common-types:npm_module_types"
]
jsts_transpiler(
name = "target_node",
srcs = SRCS,
build_pkg_name = package_name(),
additional_args = [
"--copy-files",
"--quiet"
],
)
jsts_transpiler(
@ -61,21 +67,9 @@ jsts_transpiler(
srcs = SRCS,
build_pkg_name = package_name(),
web = True,
)
peggy(
name = "grammar",
data = [
":grammar/grammar.config.json",
":grammar/grammar.peggy"
],
output_dir = True,
args = [
"--extra-options-file",
"./%s/grammar/grammar.config.json" % package_name(),
"-o",
"$(@D)/built_grammar.js",
"./%s/grammar/grammar.peggy" % package_name()
additional_args = [
"--copy-files",
"--quiet"
],
)
@ -102,7 +96,7 @@ ts_project(
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":target_web"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
@ -110,7 +104,7 @@ js_library(
js_library(
name = "npm_module_types",
srcs = NPM_MODULE_EXTRA_FILES + [":grammar"],
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],

View file

@ -7,7 +7,7 @@
*/
import type { Ast, AstWithMeta } from './ast';
import { parse } from '../../../../grammar/built_grammar.js';
import { parse } from './grammar.peggy';
interface Options {
startRule?: string;

View file

@ -6,7 +6,8 @@
"outDir": "target_types",
"types": [
"jest",
"node"
"node",
"@kbn/ambient-common-types"
]
},
"include": [

View file

@ -47,6 +47,7 @@ RUNTIME_DEPS = [
"//packages/kbn-ui-shared-deps-src",
"//packages/kbn-utils",
"//packages/kbn-synthetic-package-map",
"//packages/kbn-peggy",
"@npm//@babel/core",
"@npm//chalk",
"@npm//clean-webpack-plugin",
@ -81,6 +82,7 @@ TYPES_DEPS = [
"//packages/kbn-utils:npm_module_types",
"//packages/kbn-tooling-log:npm_module_types",
"//packages/kbn-synthetic-package-map:npm_module_types",
"//packages/kbn-peggy:npm_module_types",
"@npm//chalk",
"@npm//clean-webpack-plugin",
"@npm//cpy",

View file

@ -101,28 +101,27 @@ export class Cache {
}
}
async update(path: string, file: { mtime: string; code: string; map: any }) {
const key = this.getKey(path);
close() {
clearTimeout(this.timer);
}
async update(path: string, file: { mtime: string; code: string; map?: any }) {
const key = this.getKey(path);
await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
file.map != null ? this.safePut(this.sourceMaps, key, JSON.stringify(file.map)) : null,
]);
}
close() {
clearTimeout(this.timer);
}
private getKey(path: string) {
const normalizedPath =
Path.sep !== '/'
? Path.relative(this.pathRoot, path).split(Path.sep).join('/')
: Path.relative(this.pathRoot, path);
return `${this.prefix}${normalizedPath}`;
return `${this.prefix}:${normalizedPath}`;
}
private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {

View file

@ -53,7 +53,7 @@ it('returns undefined until values are set', async () => {
const log = makeTestLog();
const cache = makeCache({
dir: DIR,
prefix: 'prefix:',
prefix: 'prefix',
log,
pathRoot: '/foo/',
});

View file

@ -37,15 +37,17 @@ import Fs from 'fs';
import Path from 'path';
import Crypto from 'crypto';
import * as babel from '@babel/core';
import { version as babelVersion } from '@babel/core';
import { VERSION as peggyVersion } from '@kbn/peggy';
import { addHook } from 'pirates';
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/utils';
import sourceMapSupport from 'source-map-support';
import { readHashOfPackageMap } from '@kbn/synthetic-package-map';
import { Cache } from './cache';
import { TRANSFORMS } from './transforms';
import { getBabelOptions } from './transforms/babel';
const cwd = process.cwd();
import { Cache } from './cache';
const IGNORE_PATTERNS = [
/[\/\\]kbn-pm[\/\\]dist[\/\\]/,
@ -63,70 +65,6 @@ const IGNORE_PATTERNS = [
/[\/\\]packages[\/\\](eslint-|kbn-)[^\/\\]+[\/\\](?!src[\/\\].*|(.+[\/\\])?(test|__tests__)[\/\\].+|.+\.test\.(js|ts|tsx)$)(.+$)/,
];
function getBabelOptions(path: string) {
return babel.loadOptions({
cwd,
sourceRoot: Path.dirname(path) + Path.sep,
filename: path,
babelrc: false,
presets: [require.resolve('@kbn/babel-preset/node_preset')],
sourceMaps: 'both',
ast: false,
})!;
}
/**
* @babel/register uses a JSON encoded copy of the config + babel.version
* as the cache key for files, so we do something similar but we don't need
* a unique cache key for every file as our config isn't different for
* different files (by design). Instead we determine a unique prefix and
* automatically prepend all paths with the prefix to create cache keys
*/
function determineCachePrefix() {
const json = JSON.stringify({
synthPkgMapHash: readHashOfPackageMap(),
babelVersion: babel.version,
// get a config for a fake js, ts, and tsx file to make sure we
// capture conditional config portions based on the file extension
js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')),
ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')),
tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')),
});
const checksum = Crypto.createHash('sha256').update(json).digest('hex').slice(0, 8);
return `${checksum}:`;
}
function compile(cache: Cache, source: string, path: string) {
try {
const mtime = `${Fs.statSync(path).mtimeMs}`;
if (cache.getMtime(path) === mtime) {
const code = cache.getCode(path);
if (code) {
// code *should* always be defined, but if it isn't for some reason rebuild it
return code;
}
}
const options = getBabelOptions(path);
const result = babel.transform(source, options);
if (!result || !result.code || !result.map) {
throw new Error(`babel failed to transpile [${path}]`);
}
cache.update(path, {
mtime,
map: result.map,
code: result.code,
});
return result.code;
} catch (error) {
throw error;
}
}
let installed = false;
export function registerNodeAutoTranspilation() {
@ -135,16 +73,45 @@ export function registerNodeAutoTranspilation() {
}
installed = true;
const cacheLog = process.env.DEBUG_NODE_TRANSPILER_CACHE
? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'))
: undefined;
const cacheDir = Path.resolve(
REPO_ROOT,
'data/node_auto_transpilation_cache_v5',
UPSTREAM_BRANCH
);
/**
* @babel/register uses a JSON encoded copy of the config + babel.version
* as the cache key for files, so we do something similar but we don't need
* a unique cache key for every file as our config isn't different for
* different files (by design). Instead we determine a unique prefix and
* automatically prepend all paths with the prefix to create cache keys
*/
const cache = new Cache({
dir: cacheDir,
log: cacheLog,
pathRoot: REPO_ROOT,
dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache_v4', UPSTREAM_BRANCH),
prefix: determineCachePrefix(),
log: process.env.DEBUG_NODE_TRANSPILER_CACHE
? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), {
flags: 'a',
prefix: Crypto.createHash('sha256')
.update(
JSON.stringify({
synthPkgMapHash: readHashOfPackageMap(),
babelVersion,
peggyVersion,
// get a config for a fake js, ts, and tsx file to make sure we
// capture conditional config portions based on the file extension
js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')),
ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')),
tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')),
})
: undefined,
)
.digest('hex')
.slice(0, 8),
});
cacheLog?.write(`cache initialized\n`);
sourceMapSupport.install({
handleUncaughtExceptions: false,
@ -152,39 +119,36 @@ export function registerNodeAutoTranspilation() {
// @ts-expect-error bad source-map-support types
retrieveSourceMap(path: string) {
const map = cache.getSourceMap(path);
if (map) {
return {
url: null,
map,
};
} else {
return null;
}
return map ? { map, url: null } : null;
},
});
let compiling = false;
let transformInProgress = false;
addHook(
(code, path) => {
if (compiling) {
if (transformInProgress) {
return code;
}
if (IGNORE_PATTERNS.some((re) => re.test(path))) {
const ext = Path.extname(path);
if (ext !== '.peggy' && IGNORE_PATTERNS.some((re) => re.test(path))) {
return code;
}
try {
compiling = true;
return compile(cache, code, path);
transformInProgress = true;
const transform = Object.hasOwn(TRANSFORMS, ext)
? TRANSFORMS[ext as keyof typeof TRANSFORMS]
: TRANSFORMS.default;
return transform(path, code, cache);
} finally {
compiling = false;
transformInProgress = false;
}
},
{
exts: ['.js', '.ts', '.tsx'],
exts: ['.js', '.ts', '.tsx', '.peggy'],
ignoreNodeModules: false,
}
);

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Path from 'path';
import Fs from 'fs';
import * as babel from '@babel/core';
import { Transform } from './transform';
export function getBabelOptions(path: string) {
return babel.loadOptions({
cwd: process.cwd(),
sourceRoot: Path.dirname(path) + Path.sep,
filename: path,
babelrc: false,
presets: [require.resolve('@kbn/babel-preset/node_preset')],
sourceMaps: 'both',
ast: false,
})!;
}
export const babelTransform: Transform = (path, source, cache) => {
const mtime = `${Fs.statSync(path).mtimeMs}`;
if (cache.getMtime(path) === mtime) {
const code = cache.getCode(path);
if (code) {
return code;
}
}
const options = getBabelOptions(path);
const result = babel.transform(source, options);
if (!result || !result.code || !result.map) {
throw new Error(`babel failed to transpile [${path}]`);
}
cache.update(path, {
mtime,
code: result.code,
map: result.map,
});
return result.code;
};

View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { peggyTransform } from './peggy';
import { babelTransform } from './babel';
export const TRANSFORMS = {
'.peggy': peggyTransform,
default: babelTransform,
};

View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Fs from 'fs';
import Crypto from 'crypto';
import * as Peggy from '@kbn/peggy';
import { Transform } from './transform';
export const peggyTransform: Transform = (path, source, cache) => {
const config = Peggy.findConfigFile(path);
const mtime = `${Fs.statSync(path).mtimeMs}`;
const key = !config
? path
: `${path}.config.${Crypto.createHash('sha256')
.update(config.source)
.digest('hex')
.slice(0, 8)}`;
if (cache.getMtime(key) === mtime) {
const code = cache.getCode(key);
if (code) {
return code;
}
}
const code = Peggy.getJsSourceSync({
content: source,
path,
format: 'commonjs',
optimize: 'speed',
config,
skipConfigSearch: true,
}).source;
cache.update(key, {
code,
mtime,
});
return code;
};

View file

@ -6,6 +6,6 @@
* Side Public License, v 1.
*/
declare module '*/grammar/built_grammar.js' {
export const parse: import('./parse').Parse;
}
import { Cache } from '../cache';
export type Transform = (path: string, source: string, cache: Cache) => string;

View file

@ -239,6 +239,10 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
loader: 'raw-loader',
},
},
{
test: /\.peggy$/,
loader: '@kbn/peggy-loader',
},
],
},

View file

@ -0,0 +1,128 @@
load("@npm//@bazel/typescript:index.bzl", "ts_config")
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
PKG_DIRNAME = "kbn-peggy-loader"
PKG_REQUIRE_NAME = "@kbn/peggy-loader"
SOURCE_FILES = glob(
[
"**/*.ts",
],
exclude = [
"**/*.config.js",
"**/*.mock.*",
"**/*.test.*",
"**/*.stories.*",
"**/__snapshots__/**",
"**/integration_tests/**",
"**/mocks/**",
"**/scripts/**",
"**/storybook/**",
"**/test_fixtures/**",
"**/test_helpers/**",
],
)
SRCS = SOURCE_FILES
filegroup(
name = "srcs",
srcs = SRCS,
)
NPM_MODULE_EXTRA_FILES = [
"package.json",
]
# In this array place runtime dependencies, including other packages and NPM packages
# which must be available for this code to run.
#
# To reference other packages use:
# "//repo/relative/path/to/package"
# eg. "//packages/kbn-utils"
#
# To reference a NPM package use:
# "@npm//name-of-package"
# eg. "@npm//lodash"
RUNTIME_DEPS = [
"//packages/kbn-peggy",
"@npm//peggy",
"@npm//webpack",
]
# In this array place dependencies necessary to build the types, which will include the
# :npm_module_types target of other packages and packages from NPM, including @types/*
# packages.
#
# To reference the types for another package use:
# "//repo/relative/path/to/package:npm_module_types"
# eg. "//packages/kbn-utils:npm_module_types"
#
# References to NPM packages work the same as RUNTIME_DEPS
TYPES_DEPS = [
"//packages/kbn-peggy:npm_module_types",
"@npm//@types/node",
"@npm//@types/jest",
"@npm//@types/webpack",
"@npm//peggy",
]
jsts_transpiler(
name = "target_node",
srcs = SRCS,
build_pkg_name = package_name(),
)
ts_config(
name = "tsconfig",
src = "tsconfig.json",
deps = [
"//:tsconfig.base.json",
"//:tsconfig.bazel.json",
],
)
ts_project(
name = "tsc_types",
args = ['--pretty'],
srcs = SRCS,
deps = TYPES_DEPS,
declaration = True,
emit_declaration_only = True,
out_dir = "target_types",
tsconfig = ":tsconfig",
)
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)
js_library(
name = "npm_module_types",
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)
pkg_npm(
name = "npm_module",
deps = [":" + PKG_DIRNAME],
)
filegroup(
name = "build",
srcs = [":npm_module"],
visibility = ["//visibility:public"],
)
filegroup(
name = "build_types",
srcs = [":npm_module_types"],
visibility = ["//visibility:public"],
)

View file

@ -0,0 +1,10 @@
---
id: kibDevDocsOpsPeggyLoader
slug: /kibana-dev-docs/ops/peggy-loader
title: "@kbn/peggy"
description: A package which wraps @kbn/peggy for use in Webpack
date: 2022-05-18
tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy', 'loader']
---
This package wraps the <DocLink id="kibDevDocsOpsPeggy" /> package so that webpack can consume it via its loader interface. This loader, like the `@kbn/peggy` package, loads config files next to grammar files for configuring the peggy parser-generator.

View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { getJsSource } from '@kbn/peggy';
import webpack from 'webpack';
// eslint-disable-next-line import/no-default-export
export default function (this: webpack.loader.LoaderContext) {
this.cacheable(true);
const callback = this.async();
if (!callback) {
throw new Error('loader requires async support');
}
getJsSource({
path: this.resourcePath,
format: 'esm',
optimize: 'size',
}).then((result) => {
if (result.config) {
this.addDependency(result.config.path);
}
callback(null, result.source);
}, callback);
}

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
module.exports = {
preset: '@kbn/test/jest_node',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-peggy-loader'],
};

View file

@ -0,0 +1,8 @@
{
"type": "shared-common",
"id": "@kbn/peggy-loader",
"owner": "@elastic/kibana-operations",
"devOnly": true,
"runtimeDeps": [],
"typeDeps": [],
}

View file

@ -0,0 +1,8 @@
{
"name": "@kbn/peggy-loader",
"private": true,
"version": "1.0.0",
"main": "./target_node/index.js",
"types": "./target_types/index.d.ts",
"license": "SSPL-1.0 OR Elastic License 2.0"
}

View file

@ -0,0 +1,15 @@
{
"extends": "../../tsconfig.bazel.json",
"compilerOptions": {
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "target_types",
"types": [
"jest",
"node"
]
},
"include": [
"**/*.ts",
]
}

View file

@ -0,0 +1,124 @@
load("@npm//@bazel/typescript:index.bzl", "ts_config")
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
PKG_DIRNAME = "kbn-peggy"
PKG_REQUIRE_NAME = "@kbn/peggy"
SOURCE_FILES = glob(
[
"**/*.ts",
],
exclude = [
"**/*.config.js",
"**/*.mock.*",
"**/*.test.*",
"**/*.stories.*",
"**/__snapshots__/**",
"**/integration_tests/**",
"**/mocks/**",
"**/scripts/**",
"**/storybook/**",
"**/test_fixtures/**",
"**/test_helpers/**",
],
)
SRCS = SOURCE_FILES
filegroup(
name = "srcs",
srcs = SRCS,
)
NPM_MODULE_EXTRA_FILES = [
"package.json",
]
# In this array place runtime dependencies, including other packages and NPM packages
# which must be available for this code to run.
#
# To reference other packages use:
# "//repo/relative/path/to/package"
# eg. "//packages/kbn-utils"
#
# To reference a NPM package use:
# "@npm//name-of-package"
# eg. "@npm//lodash"
RUNTIME_DEPS = [
"@npm//peggy",
]
# In this array place dependencies necessary to build the types, which will include the
# :npm_module_types target of other packages and packages from NPM, including @types/*
# packages.
#
# To reference the types for another package use:
# "//repo/relative/path/to/package:npm_module_types"
# eg. "//packages/kbn-utils:npm_module_types"
#
# References to NPM packages work the same as RUNTIME_DEPS
TYPES_DEPS = [
"@npm//@types/node",
"@npm//@types/jest",
"@npm//peggy",
]
jsts_transpiler(
name = "target_node",
srcs = SRCS,
build_pkg_name = package_name(),
)
ts_config(
name = "tsconfig",
src = "tsconfig.json",
deps = [
"//:tsconfig.base.json",
"//:tsconfig.bazel.json",
],
)
ts_project(
name = "tsc_types",
args = ['--pretty'],
srcs = SRCS,
deps = TYPES_DEPS,
declaration = True,
emit_declaration_only = True,
out_dir = "target_types",
tsconfig = ":tsconfig",
)
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)
js_library(
name = "npm_module_types",
srcs = NPM_MODULE_EXTRA_FILES,
deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)
pkg_npm(
name = "npm_module",
deps = [":" + PKG_DIRNAME],
)
filegroup(
name = "build",
srcs = [":npm_module"],
visibility = ["//visibility:public"],
)
filegroup(
name = "build_types",
srcs = [":npm_module_types"],
visibility = ["//visibility:public"],
)

View file

@ -0,0 +1,23 @@
---
id: kibDevDocsOpsPeggy
slug: /kibana-dev-docs/ops/peggy
title: "@kbn/peggy"
description: A package which wraps the peggy library for use in Kibana
date: 2022-05-18
tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy']
---
This package wraps the peggy package, exposing a synchronous and async version of the generator which includes two modifications:
1. When a `path` is provided a `${basename}.config.json` file will be loaded if it exists and is expected to include peggy config options as defined in [the peggy docs](https://peggyjs.org/documentation.html#generating-a-parser-javascript-api). This config will be used when compiling this file
## Plugins
These types will automatically be included for plugins.
## Packages
To include these types in a package:
- add `"//packages/kbn-ambient-ui-types"` to the `RUNTIME_DEPS` portion of the `BUILD.bazel` file.
- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file.
- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file.

133
packages/kbn-peggy/index.ts Normal file
View file

@ -0,0 +1,133 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Path from 'path';
import Fs from 'fs';
import Fsp from 'fs/promises';
import Peggy from 'peggy';
export interface Options {
/**
* The path to the peggy content. If this is not defined then
* config files can not be found and `content` must be passed.
*/
path?: string;
/**
* Prevent loading the content from disk by specifying it here
*/
content?: string;
/**
* Prevent loading the config from disk by specifying it here
*/
config?: Config;
/**
* What type of module format should the generated code use. Defaults to
* commonjs for broadest compatibility
*/
format?: 'esm' | 'commonjs';
/**
* Should the parser optimize for execution speed or size of the code
*/
optimize?: 'size' | 'speed';
/**
* Disable checking for a config file a `{basename}.config.json` in
* the same directory as the grammar file.
*/
skipConfigSearch?: boolean;
}
export interface Config {
/** the path of the discovered config file */
path: string;
/** the content of the config file as a string (primarily for hashing) */
source: string;
/** the parsed content of the config file */
parsed: any;
}
export interface Result {
/**
* The source code of the module which parses expressions in the format
* defined by the peggy grammar file
*/
config: Config | null;
/**
* The loaded config if it was found
*/
source: string;
}
export function findConfigFile(grammarPath: string): Config | undefined {
const path = Path.resolve(Path.dirname(grammarPath), `${Path.basename(grammarPath)}.config.json`);
let source;
let parsed;
try {
source = Fs.readFileSync(path, 'utf8');
parsed = JSON.parse(source);
} catch (error) {
if (error.code === 'ENOENT') {
return undefined;
}
throw error;
}
return { path, source, parsed };
}
export async function getJsSource(options: Options): Promise<Result> {
let source;
if (options.content) {
source = options.content;
} else if (options.path) {
source = await Fsp.readFile(options.path, 'utf8');
} else {
throw new Error('you must either specify the path of the grammar file, or the content');
}
return getJsSourceSync({
content: source,
...options,
});
}
export function getJsSourceSync(
options: Options & {
/** The content of the grammar file to parse */
content: string;
}
): Result {
const config =
options.config ??
(options.path && options.skipConfigSearch !== true ? findConfigFile(options.path) : null);
const result = Peggy.generate(options.content, {
...config?.parsed,
format: options.format === 'esm' ? 'es' : 'commonjs',
optimize: options.optimize,
output: 'source',
});
return {
/**
* The source code of the module which parses expressions in the format
* defined by the peggy grammar file
*/
source: result as unknown as string,
/**
* The loaded config if it was found
*/
config: config ?? null,
};
}
export const VERSION = Peggy.VERSION;

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
module.exports = {
preset: '@kbn/test/jest_node',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-peggy'],
};

View file

@ -0,0 +1,8 @@
{
"type": "shared-common",
"id": "@kbn/peggy",
"owner": "@elastic/kibana-operations",
"devOnly": true,
"runtimeDeps": [],
"typeDeps": [],
}

View file

@ -0,0 +1,8 @@
{
"name": "@kbn/peggy",
"private": true,
"version": "1.0.0",
"main": "./target_node/index.js",
"types": "./target_types/index.d.ts",
"license": "SSPL-1.0 OR Elastic License 2.0"
}

View file

@ -0,0 +1,15 @@
{
"extends": "../../tsconfig.bazel.json",
"compilerOptions": {
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "target_types",
"types": [
"jest",
"node"
]
},
"include": [
"**/*.ts",
]
}

View file

@ -82,6 +82,12 @@ export default ({ config: storybookConfig }: { config: Configuration }) => {
loader: 'raw-loader',
},
},
{
test: /\.peggy$/,
use: {
loader: '@kbn/peggy-loader',
},
},
{
test: /\.scss$/,
exclude: /\.module.(s(a|c)ss)$/,

View file

@ -100,6 +100,7 @@ TYPES_DEPS = [
"//packages/kbn-bazel-packages:npm_module_types",
"//packages/kbn-get-repo-files:npm_module_types",
"//packages/kbn-ftr-screenshot-filename:npm_module_types",
"//packages/kbn-peggy:npm_module_types",
"@npm//@elastic/elasticsearch",
"@npm//@jest/console",
"@npm//@jest/reporters",
@ -119,6 +120,7 @@ TYPES_DEPS = [
"@npm//rxjs",
"@npm//playwright",
"@npm//xmlbuilder",
"@npm//@jest/transform",
"@npm//@types/archiver",
"@npm//@types/chance",
"@npm//@types/dedent",

View file

@ -120,9 +120,9 @@ module.exports = {
// A map from regular expressions to paths to transformers
transform: {
'^.+\\.(js|tsx?)$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/babel_transform.js',
'^.+\\.txt?$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/raw_transform.js',
'^.+\\.html?$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/raw_transform.js',
'^.+\\.(js|tsx?)$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/transforms/babel.js',
'^.+\\.(txt|html)?$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/transforms/raw.js',
'^.+\\.peggy?$': '<rootDir>/node_modules/@kbn/test/target_node/src/jest/transforms/peggy.js',
},
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation

View file

@ -38,6 +38,10 @@ export class EsVersion {
this.parsed = parsed;
}
toJSON() {
return this.toString();
}
toString() {
return this.parsed.version;
}

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
const Peggy = require('@kbn/peggy');
const Crypto = require('crypto');
/** @type {import('@jest/transform').AsyncTransformer} */
module.exports = {
canInstrument: false,
getCacheKey(sourceText, sourcePath) {
const config = Peggy.findConfigFile(sourcePath);
return (
Crypto.createHash('sha256').update(sourceText).digest('hex') +
(!config ? '' : `-${Crypto.createHash('sha256').update(config.source).digest('hex')}`)
);
},
process(sourceText, sourcePath) {
return {
code: Peggy.getJsSourceSync({
content: sourceText,
path: sourcePath,
format: 'commonjs',
optimize: 'speed',
}).source,
};
},
};

View file

@ -1,32 +1,18 @@
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("@npm//peggy:index.bzl", "peggy")
load("//src/dev/bazel:index.bzl", "pkg_npm")
PKG_DIRNAME = "kbn-timelion-grammar"
PKG_REQUIRE_NAME = "@kbn/timelion-grammar"
NPM_MODULE_EXTRA_FILES = [
"index.js",
"chain.peggy",
"package.json",
]
peggy(
name = "grammar",
data = [
":grammar/chain.peggy"
],
output_dir = True,
args = [
"-o",
"$(@D)/built_grammar.js",
"./%s/grammar/chain.peggy" % package_name()
],
)
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES + [
":grammar"
],
srcs = NPM_MODULE_EXTRA_FILES,
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],
)

View file

@ -6,6 +6,4 @@
* Side Public License, v 1.
*/
declare module '*/grammar/built_grammar.js' {
export const parse: any;
}
module.exports = require('./chain.peggy');

View file

@ -2,6 +2,5 @@
"name": "@kbn/timelion-grammar",
"version": "1.0.0",
"license": "SSPL-1.0 OR Elastic License 2.0",
"private": true,
"main": "grammar/built_grammar.js"
"private": true
}

View file

@ -1,5 +1,4 @@
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
load("@npm//peggy:index.bzl", "peggy")
load("//src/dev/bazel:index.bzl", "pkg_npm")
PKG_DIRNAME = "kbn-tinymath"
@ -31,25 +30,9 @@ RUNTIME_DEPS = [
"@npm//lodash",
]
peggy(
name = "grammar",
data = [
":grammar/grammar.peggy"
],
output_dir = True,
args = [
"-o",
"$(@D)/built_grammar.js",
"./%s/grammar/grammar.peggy" % package_name()
],
)
js_library(
name = PKG_DIRNAME,
srcs = NPM_MODULE_EXTRA_FILES + [
":srcs",
":grammar"
],
srcs = NPM_MODULE_EXTRA_FILES + [":srcs"],
deps = RUNTIME_DEPS,
package_name = PKG_REQUIRE_NAME,
visibility = ["//visibility:public"],

View file

@ -9,7 +9,7 @@
const { get } = require('lodash');
const memoizeOne = require('memoize-one');
const { functions: includedFunctions } = require('./functions');
const { parse: parseFn } = require('../grammar/built_grammar.js');
const { parse: parseFn } = require('./grammar.peggy');
function parse(input, options) {
if (input == null) {

View file

@ -50,6 +50,7 @@ RUNTIME_DEPS = [
"//packages/kbn-std",
"//packages/kbn-ui-shared-deps-npm",
"//packages/kbn-ui-theme",
"//packages/kbn-peggy-loader",
]
TYPES_DEPS = [

View file

@ -56,6 +56,10 @@ module.exports = {
},
],
},
{
test: /\.peggy$/,
use: ['@kbn/peggy-loader'],
},
{
test: /\.css$/,
use: [MiniCssExtractPlugin.loader, 'css-loader'],

View file

@ -89,7 +89,7 @@ it('applies filter function specified', async () => {
await scanCopy({
source: FIXTURES,
destination,
filter: (record) => !record.name.includes('bar'),
filter: (record) => !record.source.name.includes('bar'),
});
expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([

View file

@ -9,11 +9,16 @@
import Fs from 'fs';
import Fsp from 'fs/promises';
import Path from 'path';
import { asyncMap, asyncForEach } from '@kbn/std';
import * as Rx from 'rxjs';
import { assertAbsolute, mkdirp } from './fs';
const fsReadDir$ = Rx.bindNodeCallback(
(path: string, cb: (err: Error | null, ents: Fs.Dirent[]) => void) => {
Fs.readdir(path, { withFileTypes: true }, cb);
}
);
interface Options {
/**
* directory to copy from
@ -26,75 +31,158 @@ interface Options {
/**
* function that is called with each Record
*/
filter?: (record: Record) => boolean;
filter?: (record: Readonly<Record>) => boolean;
/**
* define permissions for reach item copied
*/
permissions?: (record: Record) => number | undefined;
permissions?: (record: Readonly<Record>) => number | undefined;
/**
* Date to use for atime/mtime
*/
time?: Date;
/**
*
*/
map?: (record: Readonly<FileRecord>) => Promise<undefined | FileRecord>;
}
class Record {
export class SomePath {
static fromAbs(path: string) {
return new SomePath(Path.dirname(path), Path.basename(path));
}
constructor(
public isDirectory: boolean,
public name: string,
public absolute: string,
public absoluteDest: string
/** The directory of the item at this path */
public readonly dir: string,
/** The name of the item at this path */
public readonly name: string
) {}
private _abs: string | null = null;
/** The absolute path of the file */
public get abs() {
if (this._abs === null) {
this._abs = Path.resolve(this.dir, this.name);
}
return this._abs;
}
private _ext: string | null = null;
/** The extension of the filename, starts with a . like the Path.extname API */
public get ext() {
if (this._ext === null) {
this._ext = Path.extname(this.name);
}
return this._ext;
}
/** return a file path with the file name changed to `name` */
withName(name: string) {
return new SomePath(this.dir, name);
}
/** return a file path with the file extension changed to `extension` */
withExt(extension: string) {
return new SomePath(this.dir, Path.basename(this.name, this.ext) + extension);
}
child(childName: string) {
return new SomePath(this.abs, childName);
}
}
interface DirRecord {
type: 'dir';
source: SomePath;
dest: SomePath;
}
interface FileRecord {
type: 'file';
source: SomePath;
dest: SomePath;
content?: string;
}
type Record = FileRecord | DirRecord;
/**
* Copy all of the files from one directory to another, optionally filtered with a
* function or modifying mtime/atime for each file.
*/
export async function scanCopy(options: Options) {
const { source, destination, filter, time, permissions } = options;
const { source, destination, filter, time, permissions, map } = options;
assertAbsolute(source);
assertAbsolute(destination);
// create or copy each child of a directory
const copyChildren = async (parent: Record) => {
const names = await Fsp.readdir(parent.absolute);
/**
* recursively fetch all the file records within a directory, starting with the
* files in the passed directory, then the files in all the child directories in
* no particular order
*/
const readDir$ = (dir: DirRecord): Rx.Observable<Record> =>
fsReadDir$(dir.source.abs).pipe(
Rx.mergeAll(),
Rx.mergeMap((ent) => {
const rec: Record = {
type: ent.isDirectory() ? 'dir' : 'file',
source: dir.source.child(ent.name),
dest: dir.dest.child(ent.name),
};
const records = await asyncMap(names, async (name) => {
const absolute = Path.join(parent.absolute, name);
const stat = await Fsp.stat(absolute);
return new Record(stat.isDirectory(), name, absolute, Path.join(parent.absoluteDest, name));
});
await asyncForEach(records, async (rec) => {
if (filter && !filter(rec)) {
return;
}
if (rec.isDirectory) {
await Fsp.mkdir(rec.absoluteDest, {
mode: permissions ? permissions(rec) : undefined,
});
} else {
await Fsp.copyFile(rec.absolute, rec.absoluteDest, Fs.constants.COPYFILE_EXCL);
if (permissions) {
const perm = permissions(rec);
if (perm !== undefined) {
await Fsp.chmod(rec.absoluteDest, perm);
}
if (filter && !filter(rec)) {
return Rx.EMPTY;
}
}
if (time) {
await Fsp.utimes(rec.absoluteDest, time, time);
}
return Rx.of(rec);
})
);
if (rec.isDirectory) {
await copyChildren(rec);
const handleGenericRec = async (rec: Record) => {
if (permissions) {
const perm = permissions(rec);
if (perm !== undefined) {
await Fsp.chmod(rec.dest.abs, perm);
}
});
}
if (time) {
await Fsp.utimes(rec.dest.abs, time, time);
}
};
await mkdirp(destination);
await copyChildren(new Record(true, Path.basename(source), source, destination));
const handleDir$ = (rec: DirRecord): Rx.Observable<unknown> =>
Rx.defer(async () => {
await mkdirp(rec.dest.abs);
await handleGenericRec(rec);
}).pipe(
Rx.mergeMap(() => readDir$(rec)),
Rx.mergeMap((ent) => (ent.type === 'dir' ? handleDir$(ent) : handleFile$(ent)))
);
const handleFile$ = (srcRec: FileRecord): Rx.Observable<unknown> =>
Rx.defer(async () => {
const rec = (map && (await map(srcRec))) ?? srcRec;
if (rec.content) {
await Fsp.writeFile(rec.dest.abs, rec.content, {
flag: 'wx',
});
} else {
await Fsp.copyFile(rec.source.abs, rec.dest.abs, Fs.constants.COPYFILE_EXCL);
}
await handleGenericRec(rec);
});
await Rx.lastValueFrom(
handleDir$({
type: 'dir',
source: SomePath.fromAbs(source),
dest: SomePath.fromAbs(destination),
})
);
}

View file

@ -11,8 +11,9 @@ import Path from 'path';
import { REPO_ROOT } from '@kbn/utils';
import { discoverBazelPackages } from '@kbn/bazel-packages';
import { runBazel } from '@kbn/bazel-runner';
import * as Peggy from '@kbn/peggy';
import { Task, scanCopy, write } from '../lib';
import { Task, scanCopy, write, deleteAll } from '../lib';
export const BuildBazelPackages: Task = {
description: 'Building distributable versions of Bazel packages',
@ -26,16 +27,54 @@ export const BuildBazelPackages: Task = {
log.info(`Copying build of`, pkg.manifest.id, 'into build');
const pkgDirInBuild = build.resolvePath(pkg.normalizedRepoRelativeDir);
const peggyConfigOutputPaths = new Set<string>();
const pkgBuildDir = config.resolveFromRepo(
'bazel-bin',
pkg.normalizedRepoRelativeDir,
'npm_module'
);
// copy the built npm_module target dir into the build, package.json is updated to copy
// the sources we actually end up using into the node_modules directory when we run
// yarn install
await scanCopy({
source: config.resolveFromRepo('bazel-bin', pkg.normalizedRepoRelativeDir, 'npm_module'),
source: pkgBuildDir,
destination: pkgDirInBuild,
permissions: (rec) => (rec.isDirectory ? 0o755 : 0o644),
permissions: (rec) => (rec.type === 'file' ? 0o644 : 0o755),
filter: (rec) => !(rec.type === 'dir' && rec.source.name === 'target_web'),
async map(rec) {
const extname = Path.extname(rec.source.name);
if (extname !== '.peggy') {
return undefined;
}
const result = await Peggy.getJsSource({
path: rec.source.abs,
format: 'commonjs',
optimize: 'speed',
});
if (result.config) {
// if there was a config file for this peggy grammar, capture its output path and
// delete it after the copy is complete
peggyConfigOutputPaths.add(
Path.resolve(pkgDirInBuild, Path.relative(pkgBuildDir, result.config.path))
);
}
return {
...rec,
dest: rec.dest.withName(rec.dest.name + '.js'),
content: result.source,
};
},
});
// cleanup any peggy config files
if (peggyConfigOutputPaths.size) {
await deleteAll(Array.from(peggyConfigOutputPaths), log);
}
await write(
Path.resolve(pkgDirInBuild, 'kibana.jsonc'),
JSON.stringify(pkg.manifest, null, 2)

View file

@ -376,6 +376,8 @@
"@kbn/ace/*": ["packages/kbn-ace/*"],
"@kbn/alerts": ["packages/kbn-alerts"],
"@kbn/alerts/*": ["packages/kbn-alerts/*"],
"@kbn/ambient-common-types": ["packages/kbn-ambient-common-types"],
"@kbn/ambient-common-types/*": ["packages/kbn-ambient-common-types/*"],
"@kbn/ambient-storybook-types": ["packages/kbn-ambient-storybook-types"],
"@kbn/ambient-storybook-types/*": ["packages/kbn-ambient-storybook-types/*"],
"@kbn/ambient-ui-types": ["packages/kbn-ambient-ui-types"],
@ -518,6 +520,10 @@
"@kbn/optimizer-webpack-helpers/*": ["packages/kbn-optimizer-webpack-helpers/*"],
"@kbn/osquery-io-ts-types": ["packages/kbn-osquery-io-ts-types"],
"@kbn/osquery-io-ts-types/*": ["packages/kbn-osquery-io-ts-types/*"],
"@kbn/peggy": ["packages/kbn-peggy"],
"@kbn/peggy/*": ["packages/kbn-peggy/*"],
"@kbn/peggy-loader": ["packages/kbn-peggy-loader"],
"@kbn/peggy-loader/*": ["packages/kbn-peggy-loader/*"],
"@kbn/performance-testing-dataset-extractor": ["packages/kbn-performance-testing-dataset-extractor"],
"@kbn/performance-testing-dataset-extractor/*": ["packages/kbn-performance-testing-dataset-extractor/*"],
"@kbn/plugin-discovery": ["packages/kbn-plugin-discovery"],
@ -1276,6 +1282,7 @@
"@testing-library/jest-dom",
"@emotion/react/types/css-prop",
"@kbn/ambient-ui-types",
"@kbn/ambient-common-types",
"@kbn/ambient-storybook-types"
]
},

View file

@ -187,6 +187,10 @@ module.exports = {
],
use: require.resolve('null-loader'),
},
{
test: /\.peggy$/,
use: require.resolve('@kbn/peggy-loader'),
},
],
},
node: {

View file

@ -11,11 +11,8 @@ import Path from 'path';
import webpack from 'webpack';
import { ToolingLog } from '@kbn/tooling-log';
import { REPO_ROOT } from '@kbn/utils';
import normalizePath from 'normalize-path';
import { CiStatsReporter } from '@kbn/ci-stats-reporter';
import { isNormalModule, isConcatenatedModule } from '@kbn/optimizer-webpack-helpers';
import { RUNTIME_SIZE_LIMIT } from './runtime_size_limit';
const IGNORED_EXTNAME = ['.map', '.br', '.gz'];
@ -91,10 +88,6 @@ export class CiStatsPlugin {
group: `canvas shareable runtime`,
id: 'total size',
value: entry.size,
limit: RUNTIME_SIZE_LIMIT,
limitConfigPath: normalizePath(
Path.relative(REPO_ROOT, require.resolve('./runtime_size_limit'))
),
},
{
group: `canvas shareable runtime`,

View file

@ -1,8 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const RUNTIME_SIZE_LIMIT = 8_200_000;

View file

@ -2705,6 +2705,10 @@
version "0.0.0"
uid ""
"@kbn/ambient-common-types@link:bazel-bin/packages/kbn-ambient-common-types":
version "0.0.0"
uid ""
"@kbn/ambient-storybook-types@link:bazel-bin/packages/kbn-ambient-storybook-types":
version "0.0.0"
uid ""
@ -3741,6 +3745,14 @@
version "0.0.0"
uid ""
"@kbn/peggy-loader@link:bazel-bin/packages/kbn-peggy-loader":
version "0.0.0"
uid ""
"@kbn/peggy@link:bazel-bin/packages/kbn-peggy":
version "0.0.0"
uid ""
"@kbn/performance-testing-dataset-extractor@link:bazel-bin/packages/kbn-performance-testing-dataset-extractor":
version "0.0.0"
uid ""
@ -4399,6 +4411,36 @@
call-me-maybe "^1.0.1"
glob-to-regexp "^0.3.0"
"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.2.0.tgz#901c5937e1441572ea23e631fe6deca68482fe76"
integrity sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==
"@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.2.0.tgz#fb877fe6bae3c4d3cea29786737840e2ae689066"
integrity sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==
"@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.2.0.tgz#986179c38b10ac41fbdaf7d036c825cbc72855d9"
integrity sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==
"@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.2.0.tgz#15f2c6fe9e0adc06c21af7e95f484ff4880d79ce"
integrity sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==
"@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.2.0.tgz#30cae5c9a202f3e1fa1deb3191b18ffcb2f239a2"
integrity sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==
"@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0":
version "2.2.0"
resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz#016d855b6bc459fd908095811f6826e45dd4ba64"
integrity sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==
"@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents.3":
version "2.1.8-no-fsevents.3"
resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz#323d72dd25103d0c4fbdce89dadf574a787b1f9b"
@ -18425,7 +18467,7 @@ listr@^0.14.1:
p-map "^2.0.0"
rxjs "^6.3.3"
lmdb-store@^1.6.11:
lmdb-store@^1:
version "1.6.11"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d"
integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q==
@ -19781,20 +19823,26 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
msgpackr-extract@^1.0.14:
version "1.0.14"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.14.tgz#87d3fe825d226e7f3d9fe136375091137f958561"
integrity sha512-t8neMf53jNZRF+f0H9VvEUVvtjGZ21odSBRmFfjZiyxr9lKYY0mpY3kSWZAIc7YWXtCZGOvDQVx2oqcgGiRBrw==
msgpackr-extract@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262"
integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==
dependencies:
nan "^2.14.2"
node-gyp-build "^4.2.3"
node-gyp-build-optional-packages "5.0.3"
optionalDependencies:
"@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-darwin-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-arm" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-arm64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0"
msgpackr@^1.4.7:
version "1.4.7"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748"
integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w==
version "1.8.0"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.0.tgz#6cf213e88f04c5a358c61085a42a4dbe5542de44"
integrity sha512-1Cos3r86XACdjLVY4CN8r72Cgs5lUzxSON6yb81sNZP9vC9nnBrEbu1/ldBhuR9BKejtoYV5C9UhmYUvZFJSNQ==
optionalDependencies:
msgpackr-extract "^1.0.14"
msgpackr-extract "^2.2.0"
multicast-dns@^7.2.5:
version "7.2.5"
@ -20670,9 +20718,9 @@ ora@^5.4.1:
wcwidth "^1.0.1"
ordered-binary@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.0.0.tgz#4f7485186b12aa42b99011aeb7aa272991d6a487"
integrity sha512-0RMlzqix3YAOZKMoXv97OIvHlqJxnmIzihjShVkYNV3JuzHbqeBOOP7wpz6yo4af1ZFnOHGsh8RK77ZmaBY3Lg==
version "1.4.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389"
integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ==
ordered-read-streams@^1.0.0:
version "1.0.1"
@ -27418,9 +27466,9 @@ wcwidth@^1.0.1:
defaults "^1.0.3"
weak-lru-cache@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.0.0.tgz#f1394721169883488c554703704fbd91cda05ddf"
integrity sha512-135bPugHHIJLNx20guHgk4etZAbd7nou34NQfdKkJPgMuC3Oqn4cT6f7ORVvnud9oEyXJVJXPcTFsUvttGm5xg==
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==
web-namespaces@^1.0.0:
version "1.1.4"