Using re2 for Timelion regular expressions (#67416) (#68795)

* Revert "Revert "Using re2 for Timelion regular expressions (#55208)""

This reverts commit c90293d03f.

* Updating re2 to 1.14.0. Still need to update build patching

* Extract the gzip to the destination, supporting multiple extract methods

* Adding 'node' to jest's moduleFileExtensions

'node' is in the defaults, not sure why we aren't using the defaults...
https://jestjs.io/docs/en/configuration#modulefileextensions-arraystring

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Brandon Kobel 2020-06-18 06:29:15 -07:00 committed by GitHub
parent 293d6de10c
commit 0d7471f741
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 253 additions and 11 deletions

1
.gitignore vendored
View file

@ -4,6 +4,7 @@
/.es
.DS_Store
.node_binaries
.native_modules
node_modules
!/src/dev/npm/__tests__/fixtures/fixture1/node_modules
!/src/dev/notice/__fixtures__/node_modules

View file

@ -213,6 +213,7 @@
"pug": "^2.0.3",
"querystring-browser": "1.0.4",
"raw-loader": "0.5.1",
"re2": "1.14.0",
"react": "^16.6.0",
"react-addons-shallow-compare": "15.6.2",
"react-anything-sortable": "^1.7.4",

View file

@ -44,6 +44,7 @@ import {
ExtractNodeBuildsTask,
InstallDependenciesTask,
OptimizeBuildTask,
PatchNativeModulesTask,
RemovePackageJsonDepsTask,
RemoveWorkspacesTask,
TranspileBabelTask,
@ -127,6 +128,7 @@ export async function buildDistributables(options) {
* directories and perform platform-specific steps
*/
await run(CreateArchivesSourcesTask);
await run(PatchNativeModulesTask);
await run(CleanExtraBinScriptsTask);
await run(CleanExtraBrowsersTask);
await run(CleanNodeBuildsTask);

Binary file not shown.

View file

@ -23,11 +23,12 @@ import { chmodSync, statSync } from 'fs';
import del from 'del';
import expect from 'expect.js';
import { mkdirp, write, read, getChildPaths, copy, copyAll, getFileHash, untar } from '../fs';
import { mkdirp, write, read, getChildPaths, copy, copyAll, getFileHash, untar, gunzip } from '../fs';
const TMP = resolve(__dirname, '__tmp__');
const FIXTURES = resolve(__dirname, 'fixtures');
const FOO_TAR_PATH = resolve(FIXTURES, 'foo_dir.tar.gz');
const FOO_GZIP_PATH = resolve(FIXTURES, 'foo.txt.gz');
const BAR_TXT_PATH = resolve(FIXTURES, 'foo_dir/bar.txt');
const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable');
@ -362,4 +363,39 @@ describe('dev/build/lib/fs', () => {
expect(await read(resolve(destination, 'foo/foo.txt'))).to.be('foo\n');
});
});
describe('gunzip()', () => {
it('rejects if source path is not absolute', async () => {
try {
await gunzip('foo/bar', '**/*', __dirname);
throw new Error('Expected gunzip() to reject');
} catch (error) {
assertNonAbsoluteError(error);
}
});
it('rejects if destination path is not absolute', async () => {
try {
await gunzip(__dirname, '**/*', 'foo/bar');
throw new Error('Expected gunzip() to reject');
} catch (error) {
assertNonAbsoluteError(error);
}
});
it('rejects if neither path is not absolute', async () => {
try {
await gunzip('foo/bar', '**/*', 'foo/bar');
throw new Error('Expected gunzip() to reject');
} catch (error) {
assertNonAbsoluteError(error);
}
});
it('extracts gzip from source into destination, creating destination if necessary', async () => {
const destination = resolve(TMP, 'z/y/x/v/u/t/foo.txt');
await gunzip(FOO_GZIP_PATH, destination);
expect(await read(resolve(destination))).to.be('foo\n');
});
});
});

View file

@ -205,6 +205,19 @@ export async function untar(source, destination, extractOptions = {}) {
]);
}
export async function gunzip(source, destination) {
assertAbsolute(source);
assertAbsolute(destination);
await mkdirp(dirname(destination));
await createPromiseFromStreams([
fs.createReadStream(source),
createGunzip(),
fs.createWriteStream(destination),
]);
}
export async function compress(type, options = {}, source, destination) {
const output = fs.createWriteStream(destination);
const archive = archiver(type, options.archiverOptions);

View file

@ -29,9 +29,11 @@ export {
copyAll,
getFileHash,
untar,
gunzip,
deleteAll,
deleteEmptyFolders,
compress,
} from './fs';
export { download } from './download';
export { scanDelete } from './scan_delete';
export { scanCopy } from './scan_copy';

View file

@ -32,6 +32,7 @@ export * from './nodejs_modules';
export * from './notice_file_task';
export * from './optimize_task';
export * from './os_packages';
export * from './patch_native_modules_task';
export * from './transpile_babel_task';
export * from './transpile_typescript_task';
export * from './transpile_scss_task';

View file

@ -22,7 +22,7 @@ import expect from 'expect.js';
import * as NodeShasumsNS from '../node_shasums';
import * as NodeDownloadInfoNS from '../node_download_info';
import * as DownloadNS from '../download';
import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
import { DownloadNodeBuildsTask } from '../download_node_builds_task';
describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

View file

@ -17,7 +17,7 @@
* under the License.
*/
import { download } from './download';
import { download } from '../../lib';
import { getNodeShasums } from './node_shasums';
import { getNodeDownloadInfo } from './node_download_info';

View file

@ -0,0 +1,103 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import fs from 'fs';
import path from 'path';
import util from 'util';
import { deleteAll, download, gunzip, untar } from '../lib';
const DOWNLOAD_DIRECTORY = '.native_modules';
const packages = [
{
name: 're2',
version: '1.14.0',
destinationPath: 'node_modules/re2/build/Release/re2.node',
extractMethod: 'gunzip',
archives: {
darwin: {
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/darwin-x64-64.gz',
sha256: '54c8386cb7cd53895cf379522114bfe82378e300e127e58d392ddd40a77e396f',
},
linux: {
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/linux-x64-64.gz',
sha256: 'f54f059035e71a7ccb3fa201080e260c41d228d13a8247974b4bb157691b6757',
},
windows: {
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/win32-x64-64.gz',
sha256: 'de708446a8b802f4634c2cfef097c2625a2811fdcd8133dfd7b7c485f966caa9',
},
},
},
];
async function getInstalledVersion(config, packageName) {
const packageJSONPath = config.resolveFromRepo(
path.join('node_modules', packageName, 'package.json')
);
const buffer = await util.promisify(fs.readFile)(packageJSONPath);
const packageJSON = JSON.parse(buffer);
return packageJSON.version;
}
async function patchModule(config, log, build, platform, pkg) {
const installedVersion = await getInstalledVersion(config, pkg.name);
if (installedVersion !== pkg.version) {
throw new Error(
`Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
);
}
const platformName = platform.getName();
const archive = pkg.archives[platformName];
const archiveName = path.basename(archive.url);
const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName);
const extractPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`);
await deleteAll([extractPath], log);
await download({
log,
url: archive.url,
destination: downloadPath,
sha256: archive.sha256,
retries: 3,
});
switch (pkg.extractMethod) {
case 'gunzip':
await gunzip(downloadPath, extractPath);
break;
case 'untar':
await untar(downloadPath, extractPath);
break;
default:
throw new Error(`Extract method of ${pkg.extractMethod} is not supported`);
}
}
export const PatchNativeModulesTask = {
description: 'Patching platform-specific native modules',
async run(config, log, build) {
for (const pkg of packages) {
await Promise.all(
config.getTargetPlatforms().map(async (platform) => {
await patchModule(config, log, build, platform, pkg);
})
);
}
},
};

View file

@ -68,6 +68,7 @@ export default {
'json',
'ts',
'tsx',
'node',
],
modulePathIgnorePatterns: [
'__fixtures__/',

View file

@ -25,7 +25,7 @@ export default new Chainable('label', {
args: [
{
name: 'inputSeries',
types: ['seriesList']
types: ['seriesList'],
},
{
name: 'label',
@ -42,7 +42,7 @@ export default new Chainable('label', {
help: i18n.translate('timelion.help.functions.label.args.regexHelpText', {
defaultMessage: 'A regex with capture group support',
}),
}
},
],
help: i18n.translate('timelion.help.functions.labelHelpText', {
defaultMessage: 'Change the label of the series. Use %s to reference the existing label',
@ -51,12 +51,15 @@ export default new Chainable('label', {
const config = args.byName;
return alter(args, function (eachSeries) {
if (config.regex) {
eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
// not using a standard `import` so that if there's an issue with the re2 native module
// that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
const RE2 = require('re2');
eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
} else {
eachSeries.label = config.label;
}
return eachSeries;
});
}
},
});

View file

@ -19,6 +19,7 @@ export function createJestConfig({
"json",
"ts",
"tsx",
"node",
],
moduleNameMapper: {
"^ui/(.*)": `${kibanaDirectory}/src/ui/public/$1`,

View file

@ -42,6 +42,7 @@ export default {
'json',
'ts',
'tsx',
'node',
],
modulePathIgnorePatterns: [
'__fixtures__/',

View file

@ -7969,6 +7969,11 @@ env-paths@^1.0.0:
resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-1.0.0.tgz#4168133b42bb05c38a35b1ae4397c8298ab369e0"
integrity sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA=
env-paths@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43"
integrity sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA==
env-variable@0.0.x:
version "0.0.5"
resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
@ -10179,6 +10184,18 @@ glob@^7.1.3:
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.1.4:
version "7.1.6"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@~3.1.21:
version "3.1.21"
resolved "https://registry.yarnpkg.com/glob/-/glob-3.1.21.tgz#d29e0a055dea5138f4d07ed40e8982e83c2066cd"
@ -15191,7 +15208,7 @@ minipass@^2.2.1:
safe-buffer "^5.1.1"
yallist "^3.0.0"
minipass@^2.2.4, minipass@^2.9.0:
minipass@^2.2.4, minipass@^2.8.6, minipass@^2.9.0:
version "2.9.0"
resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6"
integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==
@ -15207,7 +15224,7 @@ minipass@^2.3.4:
safe-buffer "^5.1.2"
yallist "^3.0.0"
minizlib@^1.1.0:
minizlib@^1.1.0, minizlib@^1.2.1:
version "1.3.3"
resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d"
integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==
@ -15477,6 +15494,11 @@ nan@^2.10.0, nan@^2.9.2:
resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f"
integrity sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==
nan@^2.14.1:
version "2.14.1"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==
nanomatch@^1.2.5:
version "1.2.7"
resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.7.tgz#53cd4aa109ff68b7f869591fdc9d10daeeea3e79"
@ -15687,6 +15709,23 @@ node-gyp@^3.8.0:
tar "^2.0.0"
which "1"
node-gyp@^6.1.0:
version "6.1.0"
resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-6.1.0.tgz#64e31c61a4695ad304c1d5b82cf6b7c79cc79f3f"
integrity sha512-h4A2zDlOujeeaaTx06r4Vy+8MZ1679lU+wbCKDS4ZtvY2A37DESo37oejIw0mtmR3+rvNwts5B6Kpt1KrNYdNw==
dependencies:
env-paths "^2.2.0"
glob "^7.1.4"
graceful-fs "^4.2.2"
mkdirp "^0.5.1"
nopt "^4.0.1"
npmlog "^4.1.2"
request "^2.88.0"
rimraf "^2.6.3"
semver "^5.7.1"
tar "^4.4.12"
which "^1.3.1"
node-int64@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
@ -15910,7 +15949,7 @@ npm-run-path@^2.0.0:
dependencies:
path-key "^2.0.0"
"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0, npmlog@^4.0.2:
"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0, npmlog@^4.0.2, npmlog@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b"
integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==
@ -17743,6 +17782,14 @@ rc@^1.0.1, rc@^1.1.6, rc@^1.2.7:
minimist "^1.2.0"
strip-json-comments "~2.0.1"
re2@1.14.0:
version "1.14.0"
resolved "https://registry.yarnpkg.com/re2/-/re2-1.14.0.tgz#727076590acfe868cf04e115a3a3f6c373ddd63b"
integrity sha512-TYogJmzni8zNVaw4gNOVORRTUaggLZwnMhJoTD0POKeACEoCxTWa9BAYehRnh3S1JUXIMEfcEUa7piiGEn71Zg==
dependencies:
nan "^2.14.1"
node-gyp "^6.1.0"
react-ace@^5.5.0:
version "5.10.0"
resolved "https://registry.yarnpkg.com/react-ace/-/react-ace-5.10.0.tgz#e328b37ac52759f700be5afdb86ada2f5ec84c5e"
@ -19140,6 +19187,13 @@ rimraf@2.4.3:
dependencies:
glob "^5.0.14"
rimraf@^2.6.3:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
dependencies:
glob "^7.1.3"
rimraf@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@ -19580,6 +19634,11 @@ semver@^5.5.1:
resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.1.tgz#7dfdd8814bdb7cabc7be0fb1d734cfb66c940477"
integrity sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw==
semver@^5.7.1:
version "5.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
semver@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
@ -20985,6 +21044,19 @@ tar@^4:
safe-buffer "^5.1.2"
yallist "^3.0.2"
tar@^4.4.12:
version "4.4.13"
resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525"
integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==
dependencies:
chownr "^1.1.1"
fs-minipass "^1.2.5"
minipass "^2.8.6"
minizlib "^1.2.1"
mkdirp "^0.5.0"
safe-buffer "^5.1.2"
yallist "^3.0.3"
tcp-port-used@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/tcp-port-used/-/tcp-port-used-1.0.1.tgz#46061078e2d38c73979a2c2c12b5a674e6689d70"
@ -23459,7 +23531,7 @@ which-module@^2.0.0:
resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=
which@1:
which@1, which@^1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
@ -23828,6 +23900,11 @@ yallist@^3.0.0, yallist@^3.0.2:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9"
integrity sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=
yallist@^3.0.3:
version "3.1.1"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
yargs-parser@^10.1.0:
version "10.1.0"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"