Using re2 for Timelion regular expressions (#55208)

* Using re2 for Timelion's regexs

* Patching native modules

* Restructuring to be more generic

* Fixing download_node_builds_task tests

* Updating linux sha after properly gzipping the archive

* Using a Centos7 machine with devtoolset-6. That's what node does

* Using new archives which Travis built for us

* Not using a standard import to prevent Kibana from not starting up

If the "portable" version of RE2 for some reason isn't truly portable
and can't load, we don't want to prevent the rest of Kibana from working
properly. This will only prevent Timelion from working, which isn't
great, but is less worse

* Isolating the require even further

* Detecting the package version mismatches, thanks Larry!

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.gitignore

@@ -4,6 +4,7 @@
 /.es
 .DS_Store
 .node_binaries
+.native_modules
 node_modules
 !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules
 !/src/dev/notice/__fixtures__/node_modules

package.json

@@ -227,6 +227,7 @@
     "pug": "^2.0.4",
     "query-string": "6.10.1",
     "raw-loader": "3.1.0",
+    "re2": "1.10.5",
     "react": "^16.12.0",
     "react-color": "^2.13.8",
     "react-dom": "^16.12.0",

src/dev/build/build_distributables.js

@@ -45,6 +45,7 @@ import {
   InstallDependenciesTask,
   BuildKibanaPlatformPluginsTask,
   OptimizeBuildTask,
+  PatchNativeModulesTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
@@ -132,6 +133,7 @@ export async function buildDistributables(options) {
    * directories and perform platform-specific steps
    */
   await run(CreateArchivesSourcesTask);
+  await run(PatchNativeModulesTask);
   await run(CleanExtraBinScriptsTask);
   await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);

src/dev/build/lib/download.js

@@ -24,7 +24,7 @@ import chalk from 'chalk';
 import { createHash } from 'crypto';
 import Axios from 'axios';
 
-import { mkdirp } from '../../lib';
+import { mkdirp } from './fs';
 
 function tryUnlink(path) {
   try {

src/dev/build/lib/index.js

@@ -33,5 +33,6 @@ export {
   compress,
   isFileAccessible,
 } from './fs';
+export { download } from './download';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/index.js

@@ -32,6 +32,7 @@ export * from './nodejs_modules';
 export * from './notice_file_task';
 export * from './optimize_task';
 export * from './os_packages';
+export * from './patch_native_modules_task';
 export * from './transpile_babel_task';
 export * from './transpile_scss_task';
 export * from './verify_env_task';

src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js

@@ -22,7 +22,7 @@ import expect from '@kbn/expect';
 
 import * as NodeShasumsNS from '../node_shasums';
 import * as NodeDownloadInfoNS from '../node_download_info';
-import * as DownloadNS from '../download';
+import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
 import { DownloadNodeBuildsTask } from '../download_node_builds_task';
 
 describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

src/dev/build/tasks/nodejs/download_node_builds_task.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { download } from './download';
+import { download } from '../../lib';
 import { getNodeShasums } from './node_shasums';
 import { getNodeDownloadInfo } from './node_download_info';
 

src/dev/build/tasks/patch_native_modules_task.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import path from 'path';
+import util from 'util';
+import { deleteAll, download, untar } from '../lib';
+
+const BASE_URL = `https://storage.googleapis.com/native-modules`;
+const DOWNLOAD_DIRECTORY = '.native_modules';
+
+const packages = [
+  {
+    name: 're2',
+    version: '1.10.5',
+    destinationPath: 'node_modules/re2/build/Release/',
+    shas: {
+      darwin: '066533b592094f91e00412499e44c338ce2466d63c9eaf0dc32be8214bde2099',
+      linux: '0322cac3c2e106129b650a8eac509f598ed283791d6116984fec4c151b24e574',
+      windows: '65b5bef7de2352f4787224c2c76a619b6683a868c8d4d71e0fdd500786fc422b',
+    },
+  },
+];
+
+async function getInstalledVersion(config, packageName) {
+  const packageJSONPath = config.resolveFromRepo(
+    path.join('node_modules', packageName, 'package.json')
+  );
+  const buffer = await util.promisify(fs.readFile)(packageJSONPath);
+  const packageJSON = JSON.parse(buffer);
+  return packageJSON.version;
+}
+
+async function patchModule(config, log, build, platform, pkg) {
+  const installedVersion = await getInstalledVersion(config, pkg.name);
+  if (installedVersion !== pkg.version) {
+    throw new Error(
+      `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
+    );
+  }
+  const platformName = platform.getName();
+  const archiveName = `${pkg.version}-${platformName}.tar.gz`;
+  const downloadUrl = `${BASE_URL}/${pkg.name}/${archiveName}`;
+  const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, archiveName);
+  const extractedPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
+  log.debug(`Patching ${pkg.name} binaries from ${downloadUrl} to ${extractedPath}`);
+
+  await deleteAll([extractedPath], log);
+  await download({
+    log,
+    url: downloadUrl,
+    destination: downloadPath,
+    sha256: pkg.shas[platformName],
+    retries: 3,
+  });
+  await untar(downloadPath, extractedPath);
+}
+
+export const PatchNativeModulesTask = {
+  description: 'Patching platform-specific native modules',
+  async run(config, log, build) {
+    for (const pkg of packages) {
+      await Promise.all(
+        config.getTargetPlatforms().map(async platform => {
+          await patchModule(config, log, build, platform, pkg);
+        })
+      );
+    }
+  },
+};

src/plugins/timelion/server/series_functions/label.js

@@ -51,7 +51,10 @@ export default new Chainable('label', {
     const config = args.byName;
     return alter(args, function(eachSeries) {
       if (config.regex) {
-        eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
+        // not using a standard `import` so that if there's an issue with the re2 native module
+        // that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
+        const RE2 = require('re2');
+        eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
       } else {
         eachSeries.label = config.label;
       }

yarn.lock

@@ -21086,7 +21086,7 @@ mute-stream@0.0.8:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
   integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
 
-nan@^2.13.2, nan@^2.9.2:
+nan@^2.13.2, nan@^2.14.0, nan@^2.9.2:
   version "2.14.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
   integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
@@ -24096,6 +24096,13 @@ re-resizable@^6.1.1:
   dependencies:
     fast-memoize "^2.5.1"
 
+re2@1.10.5:
+  version "1.10.5"
+  resolved "https://registry.yarnpkg.com/re2/-/re2-1.10.5.tgz#b3730438121c6bf59d459aff3471177eef513445"
+  integrity sha512-ssO3AD8/YJzuQUgEasS8PxA8n1yg8JB2VNSJhCebuuHLwaGiufhtFGUypS2bONrCPDbjwlMy7OZD9LkcrQMr1g==
+  dependencies:
+    nan "^2.14.0"
+
 react-ace@^5.5.0:
   version "5.10.0"
   resolved "https://registry.yarnpkg.com/react-ace/-/react-ace-5.10.0.tgz#e328b37ac52759f700be5afdb86ada2f5ec84c5e"