chore(NA): manage npm dependencies within bazel (#92864)

* chore(NA): full WORKSPACE.bazel logic plus manage yarn dependencies with Bazel

* chore(NA): update BUILD.bazel files comments on root and packages

* chore(NA): add workspace file with useful data

* chore(NA): install deps through bazel

* chore(NA): update workspace file

* chore(NA): update into last rules nodejs

* chore(NA): ensure bazel always run yarn install

* chore(NA): support offline mode

* chore(NA): remove elastic-datemath

* chore(NA): restore bazel 4.0.0

* chore(NA): update kbn pm dist

* chore(NA): introduce force-install command

* docs(NA): update docs with new yarn kbn bootstrap flags

* chore(NA): use path.resolve on kbn bootstrap integrity check verification

* chore(NA): update .yarnrc

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): change cli argument typo

* chore(NA): fix spacing on kbn pm cli

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tyler Smalley <tylersmalley@me.com>

.bazelrc

@@ -16,3 +16,12 @@ build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages
 
+###############################
+# Offline Support             #
+# Turn on these settings with #
+#  --config=offline           #
+###############################
+
+## Reset remote cache and backend support
+build:offline --bes_backend="" --remote_cache=""
+run:offline --bes_backend="" --remote_cache=""

.bazelrc.common

@@ -7,7 +7,9 @@
 
 # Local Cache Settings
 ## Avoid cache results from being corrupt when changing source during build
-common --experimental_guard_against_concurrent_changes
+build --experimental_guard_against_concurrent_changes
+run --experimental_guard_against_concurrent_changes
+test --experimental_guard_against_concurrent_changes
 
 ## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
 build --disk_cache=~/.bazel-cache/disk-cache

.yarnrc

@@ -3,3 +3,9 @@ yarn-offline-mirror ".yarn-local-mirror"
 
 # Always look into the cache first before fetching online
 --install.prefer-offline true
+
+# Disable interactive and progress logs as yarn install is now
+# managed by Bazel and we are piping the logs from the underlying
+# process running bazel into the parent one running kbn
+--install.non-interactive true
+--install.no-progress true

BUILD.bazel

@@ -1,3 +1,5 @@
+# Expose those targets as they are required as part of
+# other packages builds and need to be included as inputs
 exports_files(
   [
     "tsconfig.json",

WORKSPACE.bazel

@@ -1,3 +1,68 @@
+# Define the workspace base name and a managed directory by bazel
+# that will hold the node_modules called @npm
 workspace(
   name = "kibana",
+  managed_directories = {"@npm": ["node_modules"]},
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# Fetch Node.js rules
+http_archive(
+  name = "build_bazel_rules_nodejs",
+  sha256 = "bfacf15161d96a6a39510e7b3d3b522cf61cb8b82a31e79400a84c5abcab5347",
+  urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.2.1/rules_nodejs-3.2.1.tar.gz"],
+)
+
+# Now that we have the rules let's import from them to complete the work
+load("@build_bazel_rules_nodejs//:index.bzl", "check_rules_nodejs_version", "node_repositories", "yarn_install")
+
+# Assure we have at least a given rules_nodejs version
+check_rules_nodejs_version(minimum_version_string = "3.2.1")
+
+# Setup the Node.js toolchain for the architectures we want to support
+#
+# NOTE: darwin-arm64 is not being installed because bazel is not yet available on that architecture.
+# The PR for it was merged and should be available in the next release of bazel and bazelisk. As soon as they have it
+# we can update that rule.
+node_repositories(
+  node_repositories = {
+    "14.16.0-darwin_amd64": ("node-v14.16.0-darwin-x64.tar.gz", "node-v14.16.0-darwin-x64", "14ec767e376d1e2e668f997065926c5c0086ec46516d1d45918af8ae05bd4583"),
+    "14.16.0-linux_arm64": ("node-v14.16.0-linux-arm64.tar.xz", "node-v14.16.0-linux-arm64", "440489a08bfd020e814c9e65017f58d692299ac3f150c8e78d01abb1104c878a"),
+    "14.16.0-linux_s390x": ("node-v14.16.0-linux-s390x.tar.xz", "node-v14.16.0-linux-s390x", "335348e46f45284b6356416ef58f85602d2dee99094588b65900f6c8839df77e"),
+    "14.16.0-linux_amd64": ("node-v14.16.0-linux-x64.tar.xz", "node-v14.16.0-linux-x64", "2e079cf638766fedd720d30ec8ffef5d6ceada4e8b441fc2a093cb9a865f4087"),
+    "14.16.0-windows_amd64": ("node-v14.16.0-win-x64.zip", "node-v14.16.0-win-x64", "716045c2f16ea10ca97bd04cf2e5ef865f9c4d6d677a9bc25e2ea522b594af4f"),
+  },
+  node_version = "14.16.0",
+  node_urls = [
+    "https://nodejs.org/dist/v{version}/{filename}",
+  ],
+  yarn_repositories = {
+    "1.21.1": ("yarn-v1.21.1.tar.gz", "yarn-v1.21.1", "d1d9f4a0f16f5ed484e814afeb98f39b82d4728c6c8beaafb5abc99c02db6674"),
+  },
+  yarn_version = "1.21.1",
+  yarn_urls = [
+    "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}",
+  ],
+  package_json = ["//:package.json"],
+)
+
+# Run yarn_install rule to take care of dependencies
+#
+# NOTE: FORCE_COLOR env var forces colors on non tty mode
+yarn_install(
+  name = "npm",
+  environment = {
+    "FORCE_COLOR": "True",
+  },
+  package_json = "//:package.json",
+  yarn_lock = "//:yarn.lock",
+  data = [
+    "//:.yarnrc",
+    "//:preinstall_check.js",
+    "//:node_modules/.yarn-integrity",
+  ],
+  symlink_node_modules = True,
+  quiet = False,
+  frozen_lockfile = False,
 )

docs/developer/getting-started/index.asciidoc

@@ -47,6 +47,23 @@ https://github.com/nodejs/node-gyp#installation[https://github.com/nodejs/node-g
 and follow the guide according your platform.
 ____
 
+In case you don't have an internet connection, the `yarn kbn bootstrap` command will 
+fail. As it is likely you have the required node_modules in the 
+offline mirror, you can try to run the step in offline mode by using:
+
+[source,bash]
+----
+yarn kbn bootstrap --offline
+----
+
+In any other circumstance where you want to force the node_modules install step 
+you can use:
+
+[source,bash]
+----
+yarn kbn bootstrap --force-install
+----
+
 (You can also run `yarn kbn` to see the other available commands. For
 more info about this tool, see
 {kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}/packages/kbn-pm].)

packages/BUILD.bazel

@@ -1,4 +1,5 @@
-# Call each package final target
+# Grouping target to call all underlying packages build
+# targets so we can build them all at once
 filegroup(
   name = "build",
   srcs = [],

packages/kbn-pm/src/cli.ts

@@ -37,6 +37,8 @@ function help() {
         --skip-kibana-plugins   Filter all plugins in ./plugins and ../kibana-extra when running command.
         --no-cache              Disable the kbn packages bootstrap cache
         --no-validate           Disable the bootstrap yarn.lock validation
+        --force-install         Forces yarn install to run on bootstrap
+        --offline               Run in offline mode
         --verbose               Set log level to verbose
         --debug                 Set log level to debug
         --quiet                 Set log level to error
@@ -73,9 +75,11 @@ export async function run(argv: string[]) {
     },
     default: {
       cache: true,
+      'force-install': true,
+      offline: false,
       validate: true,
     },
-    boolean: ['cache', 'validate'],
+    boolean: ['cache', 'force-install', 'offline', 'validate'],
   });
 
   const args = options._;

packages/kbn-pm/src/commands/bootstrap.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { sep } from 'path';
+import { resolve, sep } from 'path';
 import { linkProjectExecutables } from '../utils/link_project_executables';
 import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
@@ -17,7 +17,7 @@ import { getAllChecksums } from '../utils/project_checksums';
 import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
 import { readYarnLock } from '../utils/yarn_lock';
 import { validateDependencies } from '../utils/validate_dependencies';
-import { installBazelTools, runBazel } from '../utils/bazel';
+import { ensureYarnIntegrityFileExists, installBazelTools, runBazel } from '../utils/bazel';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
@@ -26,12 +26,36 @@ export const BootstrapCommand: ICommand = {
   async run(projects, projectGraph, { options, kbn, rootPath }) {
     const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
     const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
-    const kibanaProjectPath = projects.get('kibana')?.path;
+    const kibanaProjectPath = projects.get('kibana')?.path || '';
+    const runOffline = options?.offline === true;
+    const forceInstall = !!options && options['force-install'] === true;
+
+    // Ensure we have a `node_modules/.yarn-integrity` file as we depend on it
+    // for bazel to know it has to re-install the node_modules after a reset or a clean
+    await ensureYarnIntegrityFileExists(resolve(kibanaProjectPath, 'node_modules'));
 
     // Install bazel machinery tools if needed
     await installBazelTools(rootPath);
 
-    // Install monorepo npm dependencies
+    // Bootstrap process for Bazel packages
+    // Bazel is now managing dependencies so yarn install
+    // will happen as part of this
+    //
+    // NOTE: Bazel projects will be introduced incrementally
+    // And should begin from the ones with none dependencies forward.
+    // That way non bazel projects could depend on bazel projects but not the other way around
+    // That is only intended during the migration process while non Bazel projects are not removed at all.
+    //
+    // Until we have our first package build within Bazel we will always need to directly call the yarn rule
+    // otherwise yarn install won't trigger as we don't have any npm dependency within Bazel
+    // TODO: Change CLI default in order to not force install as soon as we have our first Bazel package being built
+    if (forceInstall) {
+      await runBazel(['run', '@nodejs//:yarn'], runOffline);
+    }
+
+    await runBazel(['build', '//packages:build'], runOffline);
+
+    // Install monorepo npm dependencies outside of the Bazel managed ones
     for (const batch of batchedNonBazelProjects) {
       for (const project of batch) {
         const isExternalPlugin = project.path.includes(`${kibanaProjectPath}${sep}plugins`);
@@ -40,12 +64,16 @@ export const BootstrapCommand: ICommand = {
           continue;
         }
 
-        if (project.isSinglePackageJsonProject || isExternalPlugin) {
+        if (isExternalPlugin) {
           await project.installDependencies();
           continue;
         }
 
-        if (!project.isEveryDependencyLocal() && !isExternalPlugin) {
+        if (
+          !project.isSinglePackageJsonProject &&
+          !project.isEveryDependencyLocal() &&
+          !isExternalPlugin
+        ) {
           throw new Error(
             `[${project.name}] is not eligible to hold non local dependencies. Move the non local dependencies into the top level package.json.`
           );
@@ -61,15 +89,9 @@ export const BootstrapCommand: ICommand = {
 
     // Assure all kbn projects with bin defined scripts
     // copy those scripts into the top level node_modules folder
-    await linkProjectExecutables(projects, projectGraph);
-
-    // Bootstrap process for Bazel packages
     //
-    // NOTE: Bazel projects will be introduced incrementally
+    // NOTE: We don't probably need this anymore, is actually not being used
-    // And should begin from the ones with none dependencies forward.
+    await linkProjectExecutables(projects, projectGraph);
-    // That way non bazel projects could depend on bazel projects but not the other way around
-    // That is only intended during the migration process while non Bazel projects are not removed at all.
-    await runBazel(['build', '//packages:build']);
 
     // Bootstrap process for non Bazel packages
     /**

packages/kbn-pm/src/utils/bazel/ensure_yarn_integrity_exists.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { join } from 'path';
+import { writeFile } from '../fs';
+
+export async function ensureYarnIntegrityFileExists(nodeModulesPath: string) {
+  try {
+    await writeFile(join(nodeModulesPath, '.yarn-integrity'), '', { flag: 'wx' });
+  } catch {
+    // no-op
+  }
+}

packages/kbn-pm/src/utils/bazel/index.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+export * from './ensure_yarn_integrity_exists';
 export * from './get_cache_folders';
 export * from './install_tools';
 export * from './run';

packages/kbn-pm/src/utils/bazel/run.ts

@@ -14,13 +14,21 @@ import { observeLines } from '@kbn/dev-utils/stdio';
 import { spawn } from '../child_process';
 import { log } from '../log';
 
-export async function runBazel(bazelArgs: string[], runOpts: execa.Options = {}) {
+export async function runBazel(
+  bazelArgs: string[],
+  offline: boolean = false,
+  runOpts: execa.Options = {}
+) {
   // Force logs to pipe in order to control the output of them
   const bazelOpts: execa.Options = {
     ...runOpts,
     stdio: 'pipe',
   };
 
+  if (offline) {
+    bazelArgs.push('--config=offline');
+  }
+
   const bazelProc = spawn('bazel', bazelArgs, bazelOpts);
 
   const bazelLogs$ = new Rx.Subject<string>();