chore(NA): @kbn/pm new commands to support development on Bazel packages (#96465) (#96911)

* chore(NA): add warnings both to run and watch commands about Bazel built packages

* chore(NA): add new commands to build and watch bazel packages

* docs(NA): add documentation about how to deal with bazel packages

* chore(NA): addressed majority of the feedback received except for improved error logging

* chore(NA): disable ibazel info notification.

* chore(NA): remove iBazel notification

* chore(NA): remove iBazel notification - kbn pm dist

* chore(NA): move show_results option to kbn-pm only

* chore(NA): patch build bazel command to include packages target list

* chore(NA): add pretty logging for elastic-datemath

* chore(NA): remove double error output from commands ran with Bazel

* fix(NA): include simple error message to preserve subprocess failure state

* docs(NA): missing docs about how to independentely watch non bazel packages

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Tiago Costa <tiagoffcc@hotmail.com>

.bazelrc.common

@@ -10,12 +10,13 @@
 build --experimental_guard_against_concurrent_changes
 run --experimental_guard_against_concurrent_changes
 test --experimental_guard_against_concurrent_changes
+query --experimental_guard_against_concurrent_changes
 
 ## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
-build --disk_cache=~/.bazel-cache/disk-cache
+common --disk_cache=~/.bazel-cache/disk-cache
 
 ## Bazel repo cache settings
-build --repository_cache=~/.bazel-cache/repository-cache
+common --repository_cache=~/.bazel-cache/repository-cache
 
 # Bazel will create symlinks from the workspace directory to output artifacts.
 # Build results will be placed in a directory called "bazel-bin"
@@ -35,13 +36,16 @@ build --experimental_inprocess_symlink_creation
 # Incompatible flags to run with
 build --incompatible_no_implicit_file_export
 build --incompatible_restrict_string_escapes
+query --incompatible_no_implicit_file_export
+query --incompatible_restrict_string_escapes
 
 # Log configs
 ## different from default
 common --color=yes
-build --show_task_finish
-build --noshow_progress
+common --noshow_progress
+common --show_task_finish
 build --noshow_loading_progress
+query --noshow_loading_progress
 build --show_result=0
 
 # Specifies desired output mode for running tests.
@@ -82,7 +86,7 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999
 run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk
 # The following option will change the build output of certain rules such as terser and may not be desirable in all cases
 # It will also output both the repo cache and action cache to a folder inside the repo
-build:debug --compilation_mode=dbg --show_result=1
+build:debug --compilation_mode=dbg --show_result=0 --noshow_loading_progress --noshow_progress --show_task_finish
 
 # Turn off legacy external runfiles
 # This prevents accidentally depending on this feature, which Bazel will remove.

WORKSPACE.bazel

@@ -52,9 +52,6 @@ node_repositories(
 # NOTE: FORCE_COLOR env var forces colors on non tty mode
 yarn_install(
   name = "npm",
-  environment = {
-    "FORCE_COLOR": "True",
-  },
   package_json = "//:package.json",
   yarn_lock = "//:yarn.lock",
   data = [

docs/developer/getting-started/index.asciidoc

@@ -66,7 +66,8 @@ yarn kbn bootstrap --force-install
 
 (You can also run `yarn kbn` to see the other available commands. For
 more info about this tool, see
-{kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}/packages/kbn-pm].)
+{kib-repo}tree/{branch}/packages/kbn-pm[{kib-repo}tree/{branch}/packages/kbn-pm]. If you want more 
+information about how to actively develop over packages please read <<monorepo-packages>>)
 
 When switching branches which use different versions of npm packages you
 may need to run:
@@ -169,3 +170,5 @@ include::debugging.asciidoc[leveloffset=+1]
 include::building-kibana.asciidoc[leveloffset=+1]
 
 include::development-plugin-resources.asciidoc[leveloffset=+1]
+
+include::monorepo-packages.asciidoc[leveloffset=+1]

docs/developer/getting-started/monorepo-packages.asciidoc

@@ -0,0 +1,66 @@
+[[monorepo-packages]]
+== {kib} Monorepo Packages
+
+Currently {kib} works as a monorepo composed by a core, plugins and packages.
+The latest are located in a folder called `packages` and are pieces of software that 
+composes a set of features that can be isolated and reused across the entire repository.
+They are also supposed to be able to imported just like any other `node_module`.
+
+Previously we relied solely on `@kbn/pm` to manage the development tools of those packages, but we are 
+now in the middle of migrating those responsibilities into Bazel. Every package already migrated 
+will contain in its root folder a `BUILD.bazel` file and other `build` and `watching` strategies should be used.
+
+Remember that any time you need to make sure the monorepo is ready to be used just run:
+
+[source,bash]
+----
+yarn kbn bootstrap
+----
+
+[discrete]
+=== Building Non Bazel Packages
+
+Non Bazel packages can be built independently with
+
+[source,bash]
+----
+yarn kbn run build -i PACKAGE_NAME
+----
+
+[discrete]
+=== Watching Non Bazel Packages
+
+Non Bazel packages can be watched independently with
+
+[source,bash]
+----
+yarn kbn watch -i PACKAGE_NAME
+----
+
+[discrete]
+=== Building Bazel Packages
+
+Bazel packages are built as a whole for now. You can use:
+
+[source,bash]
+----
+yarn kbn build-bazel
+----
+
+[discrete]
+=== Watching Bazel Packages
+
+Bazel packages are watched as a whole for now. You can use:
+
+[source,bash]
+----
+yarn kbn watch-bazel
+----
+
+
+[discrete]
+=== List of Already Migrated Packages to Bazel
+
+- @elastic/datemath
+
+

packages/elastic-datemath/BUILD.bazel

@@ -40,6 +40,7 @@ ts_config(
 
 ts_project(
   name = "tsc",
+  args = ['--pretty'],
   srcs = SRCS,
   deps = DEPS,
   declaration = True,

packages/kbn-pm/src/commands/bootstrap.ts

@@ -66,7 +66,7 @@ export const BootstrapCommand: ICommand = {
       await runBazel(['run', '@nodejs//:yarn'], runOffline);
     }
 
-    await runBazel(['build', '//packages:build'], runOffline);
+    await runBazel(['build', '//packages:build', '--show_result=1'], runOffline);
 
     // Install monorepo npm dependencies outside of the Bazel managed ones
     for (const batch of batchedNonBazelProjects) {

packages/kbn-pm/src/commands/build_bazel.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { runBazel } from '../utils/bazel';
+import { ICommand } from './';
+
+export const BuildBazelCommand: ICommand = {
+  description: 'Runs a build in the Bazel built packages',
+  name: 'build-bazel',
+
+  async run(projects, projectGraph, { options }) {
+    const runOffline = options?.offline === true;
+
+    // Call bazel with the target to build all available packages
+    await runBazel(['build', '//packages:build', '--show_result=1'], runOffline);
+  },
+};

packages/kbn-pm/src/commands/index.ts

@@ -27,16 +27,20 @@ export interface ICommand {
 }
 
 import { BootstrapCommand } from './bootstrap';
+import { BuildBazelCommand } from './build_bazel';
 import { CleanCommand } from './clean';
 import { ResetCommand } from './reset';
 import { RunCommand } from './run';
 import { WatchCommand } from './watch';
+import { WatchBazelCommand } from './watch_bazel';
 import { Kibana } from '../utils/kibana';
 
 export const commands: { [key: string]: ICommand } = {
   bootstrap: BootstrapCommand,
+  'build-bazel': BuildBazelCommand,
   clean: CleanCommand,
   reset: ResetCommand,
   run: RunCommand,
   watch: WatchCommand,
+  'watch-bazel': WatchBazelCommand,
 };

packages/kbn-pm/src/commands/run.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import dedent from 'dedent';
 import { CliError } from '../utils/errors';
 import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
@@ -13,10 +14,17 @@ import { topologicallyBatchProjects } from '../utils/projects';
 import { ICommand } from './';
 
 export const RunCommand: ICommand = {
-  description: 'Run script defined in package.json in each package that contains that script.',
+  description:
+    'Run script defined in package.json in each package that contains that script (only works on packages not using Bazel yet)',
   name: 'run',
 
   async run(projects, projectGraph, { extraArgs, options }) {
+    log.warning(dedent`
+      We are migrating packages into the Bazel build system and we will no longer support running npm scripts on
+      packages using 'yarn kbn run' on Bazel built packages. If the package you are trying to act on contains a
+      BUILD.bazel file please just use 'yarn kbn build-bazel' to build it or 'yarn kbn watch-bazel' to watch it
+    `);
+
     const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
 
     if (extraArgs.length === 0) {

packages/kbn-pm/src/commands/watch.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import dedent from 'dedent';
 import { CliError } from '../utils/errors';
 import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
@@ -34,10 +35,16 @@ const kibanaProjectName = 'kibana';
  * `webpack` and `tsc` only, for the rest we rely on predefined timeouts.
  */
 export const WatchCommand: ICommand = {
-  description: 'Runs `kbn:watch` script for every project.',
+  description:
+    'Runs `kbn:watch` script for every project (only works on packages not using Bazel yet)',
   name: 'watch',
 
   async run(projects, projectGraph) {
+    log.warning(dedent`
+      We are migrating packages into the Bazel build system. If the package you are trying to watch
+      contains a BUILD.bazel file please just use 'yarn kbn watch-bazel'
+    `);
+
     const projectsToWatch: ProjectMap = new Map();
     for (const project of projects.values()) {
       // We can't watch project that doesn't have `kbn:watch` script.

packages/kbn-pm/src/commands/watch_bazel.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { runIBazel } from '../utils/bazel';
+import { ICommand } from './';
+
+export const WatchBazelCommand: ICommand = {
+  description: 'Runs a build in the Bazel built packages and keeps watching them for changes',
+  name: 'watch-bazel',
+
+  async run(projects, projectGraph, { options }) {
+    const runOffline = options?.offline === true;
+
+    // Call bazel with the target to build all available packages and run it through iBazel to watch it for changes
+    //
+    // Note: --run_output=false arg will disable the iBazel notifications about gazelle and buildozer when running it
+    // Can also be solved by adding a root `.bazel_fix_commands.json` but its not needed at the moment
+    await runIBazel(['--run_output=false', 'build', '//packages:build'], runOffline);
+  },
+};

packages/kbn-pm/src/utils/bazel/run.ts

@@ -13,8 +13,12 @@ import { tap } from 'rxjs/operators';
 import { observeLines } from '@kbn/dev-utils/stdio';
 import { spawn } from '../child_process';
 import { log } from '../log';
+import { CliError } from '../errors';
 
-export async function runBazel(
+type BazelCommandRunner = 'bazel' | 'ibazel';
+
+async function runBazelCommandWithRunner(
+  bazelCommandRunner: BazelCommandRunner,
   bazelArgs: string[],
   offline: boolean = false,
   runOpts: execa.Options = {}
@@ -29,7 +33,7 @@ export async function runBazel(
     bazelArgs.push('--config=offline');
   }
 
-  const bazelProc = spawn('bazel', bazelArgs, bazelOpts);
+  const bazelProc = spawn(bazelCommandRunner, bazelArgs, bazelOpts);
 
   const bazelLogs$ = new Rx.Subject<string>();
 
@@ -37,15 +41,35 @@ export async function runBazel(
   // Therefore we need to get both. In order to get errors we need to parse the actual text line
   const bazelLogSubscription = Rx.merge(
     observeLines(bazelProc.stdout!).pipe(
-      tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
+      tap((line) => log.info(`${chalk.cyan(`[${bazelCommandRunner}]`)} ${line}`))
     ),
     observeLines(bazelProc.stderr!).pipe(
-      tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
+      tap((line) => log.info(`${chalk.cyan(`[${bazelCommandRunner}]`)} ${line}`))
     )
   ).subscribe(bazelLogs$);
 
   // Wait for process and logs to finish, unsubscribing in the end
-  await bazelProc;
+  try {
+    await bazelProc;
+  } catch {
+    throw new CliError(`The bazel command that was running failed to complete.`);
+  }
   await bazelLogs$.toPromise();
   await bazelLogSubscription.unsubscribe();
 }
+
+export async function runBazel(
+  bazelArgs: string[],
+  offline: boolean = false,
+  runOpts: execa.Options = {}
+) {
+  await runBazelCommandWithRunner('bazel', bazelArgs, offline, runOpts);
+}
+
+export async function runIBazel(
+  bazelArgs: string[],
+  offline: boolean = false,
+  runOpts: execa.Options = {}
+) {
+  await runBazelCommandWithRunner('ibazel', bazelArgs, offline, runOpts);
+}