Integration of a static filesystem for the node_modules (#47998) (#65346)

* feat(NA): added new build step to generate a static file system for node_modules and monkey patch the bin start files to include the static file system loader on begining.

* chore(NA): updated lockfile.

* chore(NA): replace import to the correct static-fs files.

* fix(NA): resolve for a correct static fs loader.

* chore(NA): last static-fs working version.

* feat(NA): changed patched entryPoints.

* chore(NA): update to last static fs.

* fix(NA): fix imports on webpack shims. chore(NA): added new static-fs version.

* chore(NA): update to static-fs with stat and statSync.

* fix(NA): vendor chunk filtering in order to include default node modules and static node modules.

* refact(NA): major refact on create static modules fs task.

* fix(NA): remove global from clean empty folders task.

* chore(NA): change args order for deleteAll.

* chore(NA): move to the new static fs

* feat(NA): migrate to last apis of static fs

* fix(NA): patch correct entry file.

* chore(NA): last stable integration with static-fs task

* chore(NA): rollback some unecessary changes

* chore(NA): remove changes on sinon webpackshim

* chore(NA): integrate with official static-fs version

* chore(NA): integrate last thread-loader version

* chore(NA): added last static-fs version

* chore(NA): upgrade to last static-fs version

* chore(NA): upgrade to last static-fs version 1.1.1

* chore(NA): rever static-fs version to 1.1.0

* chore(NA): upgrade static-fs to 1.1.1

* chore(NA): upgrade static-fs for last 1.2.0 version

* chore(NA): update package.json

* chore(NA): run apm after patched environment

* chore(NA): specify rule to disable eslint for

* chore(NA): remove changes in the src/apm file

* chore(NA): change apm order in the dev scripts

* chore(na): update static-fs versin

* chore(NA): bump static fs version

* chore(NA): correctly lint task file

* chore(NA): move away from rimraf to del

* chore(NA): bump static-fs version

* chore(NA): bump static-fs version

* chore(NA): bump static-fs version

* chore(NA): bump static-fs version

* chore(NA): bump to last static-fs version

* chore(NA): apply changed according PR review feedback

* chore(NA): remove changes to base optimizer

* chore(NA): fix discover pattern

* chore(NA): bump static-fs version

* chore(NA): bump static fs to 1.6.3

* chore(NA): bump static-fs to 1.6.4

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs to last version

* chore(NA): bump static-fs for 1.8.3

* chore(NA): bump static-fs to 1.9.0

* chore(NA): bump static-fs to 1.9.1

* chore(NA): update to last static-fs version 1.10.0

* chore(NA): moving to @elastic/static-fs package

* fix(NA): change import to the new package

* chore(NA): bump elastic static-fs to 1.0.1

Co-authored-by: spalger <email@spalger.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: spalger <email@spalger.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

package.json

@@ -293,6 +293,7 @@
     "@elastic/eslint-plugin-eui": "0.0.2",
     "@elastic/github-checks-reporter": "0.0.20b3",
     "@elastic/makelogs": "^5.0.1",
+    "@elastic/static-fs": "1.0.1",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/es": "1.0.0",
     "@kbn/eslint-import-resolver-kibana": "2.0.0",

scripts/kibana.js

@@ -17,6 +17,6 @@
  * under the License.
  */
 
-require('../src/apm')(process.env.ELASTIC_APM_PROXY_SERVICE_NAME || 'kibana-proxy');
 require('../src/setup_node_env');
+require('../src/apm')(process.env.ELASTIC_APM_PROXY_SERVICE_NAME || 'kibana-proxy');
 require('../src/cli/cli');

src/cli/index.js

@@ -17,6 +17,6 @@
  * under the License.
  */
 
-require('../apm')();
 require('../setup_node_env');
+require('../apm')();
 require('./cli');

src/dev/build/build_distributables.js

@@ -40,6 +40,7 @@ import {
   CreatePackageJsonTask,
   CreateReadmeTask,
   CreateRpmPackageTask,
+  CreateStaticFsWithNodeModulesTask,
   DownloadNodeBuildsTask,
   ExtractNodeBuildsTask,
   InstallDependenciesTask,
@@ -126,6 +127,7 @@ export async function buildDistributables(options) {
   await run(CleanTypescriptTask);
   await run(CleanExtraFilesFromModulesTask);
   await run(CleanEmptyFoldersTask);
+  await run(CreateStaticFsWithNodeModulesTask);
 
   /**
    * copy generic build outputs into platform-specific build

src/dev/build/tasks/create_static_fs_with_node_modules_task.js

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import del from 'del';
+import globby from 'globby';
+import { resolve } from 'path';
+import { generateStaticFsVolume } from '@elastic/static-fs';
+
+async function deletePathsList(list) {
+  for (const path of list) {
+    await del(path);
+  }
+}
+
+async function getTopLevelNodeModulesFolders(rootDir) {
+  const nodeModulesFoldersForCwd = await globby(['**/node_modules', '!**/node_modules/**/*'], {
+    cwd: rootDir,
+    onlyDirectories: true,
+  });
+
+  return nodeModulesFoldersForCwd.map(folder => resolve(rootDir, folder));
+}
+
+export const CreateStaticFsWithNodeModulesTask = {
+  description:
+    'Creating static filesystem with node_modules, patching entryPoints and deleting node_modules folder',
+
+  async run(config, log, build) {
+    const rootDir = build.resolvePath('.');
+
+    // Get all the top node_modules folders
+    const nodeModulesFolders = await getTopLevelNodeModulesFolders(rootDir);
+
+    // Define root entry points
+    const rootEntryPoints = [build.resolvePath('src/setup_node_env/index.js')];
+
+    // Creates the static filesystem with
+    // every node_module we have
+    const staticFsAddedPaths = await generateStaticFsVolume(
+      rootDir,
+      nodeModulesFolders,
+      rootEntryPoints
+    );
+
+    // Delete node_modules folder
+    await deletePathsList(staticFsAddedPaths);
+  },
+};

src/dev/build/tasks/index.js

@@ -25,6 +25,7 @@ export * from './create_archives_task';
 export * from './create_empty_dirs_and_files_task';
 export * from './create_package_json_task';
 export * from './create_readme_task';
+export * from './create_static_fs_with_node_modules_task';
 export * from './install_dependencies_task';
 export * from './license_file_task';
 export * from './nodejs';

yarn.lock

@@ -1393,6 +1393,11 @@
     "@types/node-jose" "1.1.0"
     node-jose "1.1.0"
 
+"@elastic/static-fs@1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@elastic/static-fs/-/static-fs-1.0.1.tgz#2e084e9fc321dd4c7fb4579021ca8a6b26f3464e"
+  integrity sha512-Vl40Va/h0P6aDUrzgDeTabGVUb/s/W92le64E1UXTcG5927cZtTnOu0datMjr98xdr9C6RAJ3mr6zgxfox5TNw==
+
 "@elastic/ui-ace@0.2.3":
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/@elastic/ui-ace/-/ui-ace-0.2.3.tgz#5281aed47a79b7216c55542b0675e435692f20cd"