chore(NA): support bazel and kbn packages in parallel on kbn pm and on distributable build scripts (#89961)

* chore(NA): bazel projects support initial flag on kbn pm

* chore(NA): every needed step to build production bazel packages except the final function

* chore(NA): include build bazel production projects on kibana distributable build

* chore(NA): support bazel packages when creating the package.json

* chore(NA): including last changes on kbn pm and build distributable to support both bazel and kbn packages

* chore(NA): missing annotation on build bazel packages

* chore(NA): changed values on bazelrc common

* chore(NA): fix bazel common rc

* chore(NA): auto discovery if a kbn package is a Bazel package

* chore(NA): last details to make bazel packages to built on distributable scripts

* chore(NA): removed wrongly added line to x-pack package.json

* chore(NA): apply correct formating

* chore(NA): move into bazel bin

* chore(NA): merge chnages on kbn pm

* chore(NA): correctly setup ignore files for new bazel aggregated folder

* chore(NA): merge with last master

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

BUILD.bazel

@@ -0,0 +1,7 @@
+exports_files(
+  [
+    "tsconfig.json",
+    "package.json"
+  ],
+  visibility = ["//visibility:public"]
+)

packages/BUILD.bazel

@@ -0,0 +1,5 @@
+# Call each package final target
+filegroup(
+  name = "build",
+  srcs = [],
+)

packages/kbn-pm/src/commands/bootstrap.ts

@@ -10,28 +10,29 @@ import { sep } from 'path';
 import { linkProjectExecutables } from '../utils/link_project_executables';
 import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
-import { topologicallyBatchProjects } from '../utils/projects';
+import { getNonBazelProjectsOnly, topologicallyBatchProjects } from '../utils/projects';
 import { Project } from '../utils/project';
 import { ICommand } from './';
 import { getAllChecksums } from '../utils/project_checksums';
 import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
 import { readYarnLock } from '../utils/yarn_lock';
 import { validateDependencies } from '../utils/validate_dependencies';
-import { installBazelTools } from '../utils/bazel';
+import { installBazelTools, runBazel } from '../utils/bazel';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
   name: 'bootstrap',
 
   async run(projects, projectGraph, { options, kbn, rootPath }) {
-    const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
+    const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
+    const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
     const kibanaProjectPath = projects.get('kibana')?.path;
 
     // Install bazel machinery tools if needed
     await installBazelTools(rootPath);
 
     // Install monorepo npm dependencies
-    for (const batch of batchedProjects) {
+    for (const batch of batchedNonBazelProjects) {
       for (const project of batch) {
         const isExternalPlugin = project.path.includes(`${kibanaProjectPath}${sep}plugins`);
 
@@ -62,9 +63,18 @@ export const BootstrapCommand: ICommand = {
     // copy those scripts into the top level node_modules folder
     await linkProjectExecutables(projects, projectGraph);
 
+    // Bootstrap process for Bazel packages
+    //
+    // NOTE: Bazel projects will be introduced incrementally
+    // And should begin from the ones with none dependencies forward.
+    // That way non bazel projects could depend on bazel projects but not the other way around
+    // That is only intended during the migration process while non Bazel projects are not removed at all.
+    await runBazel(['build', '//packages:build']);
+
+    // Bootstrap process for non Bazel packages
     /**
      * At the end of the bootstrapping process we call all `kbn:bootstrap` scripts
-     * in the list of projects. We do this because some projects need to be
+     * in the list of non Bazel projects. We do this because some projects need to be
      * transpiled before they can be used. Ideally we shouldn't do this unless we
      * have to, as it will slow down the bootstrapping process.
      */
@@ -73,8 +83,8 @@ export const BootstrapCommand: ICommand = {
     const caches = new Map<Project, { file: BootstrapCacheFile; valid: boolean }>();
     let cachedProjectCount = 0;
 
-    for (const project of projects.values()) {
-      if (project.hasScript('kbn:bootstrap')) {
+    for (const project of nonBazelProjectsOnly.values()) {
+      if (project.hasScript('kbn:bootstrap') && !project.isBazelPackage()) {
         const file = new BootstrapCacheFile(kbn, project, checksums);
         const valid = options.cache && file.isValid();
 
@@ -91,7 +101,7 @@ export const BootstrapCommand: ICommand = {
       log.success(`${cachedProjectCount} bootstrap builds are cached`);
     }
 
-    await parallelizeBatches(batchedProjects, async (project) => {
+    await parallelizeBatches(batchedNonBazelProjects, async (project) => {
       const cache = caches.get(project);
       if (cache && !cache.valid) {
         log.info(`[${project.name}] running [kbn:bootstrap] script`);

packages/kbn-pm/src/index.ts

@@ -7,7 +7,7 @@
  */
 
 export { run } from './cli';
-export { buildProductionProjects } from './production';
+export { buildBazelProductionProjects, buildNonBazelProductionProjects } from './production';
 export { getProjects } from './utils/projects';
 export { Project } from './utils/project';
 export { transformDependencies } from './utils/package_json';

packages/kbn-pm/src/production/build_bazel_production_projects.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import copy from 'cpy';
+import globby from 'globby';
+import { basename, join, relative, resolve } from 'path';
+
+import { buildProject, getProductionProjects } from './build_non_bazel_production_projects';
+import { chmod, isFile, isDirectory } from '../utils/fs';
+import { log } from '../utils/log';
+import {
+  createProductionPackageJson,
+  readPackageJson,
+  writePackageJson,
+} from '../utils/package_json';
+import { getBazelProjectsOnly } from '../utils/projects';
+import { Project } from '..';
+
+export async function buildBazelProductionProjects({
+  kibanaRoot,
+  buildRoot,
+  onlyOSS,
+}: {
+  kibanaRoot: string;
+  buildRoot: string;
+  onlyOSS?: boolean;
+}) {
+  const projects = await getBazelProjectsOnly(await getProductionProjects(kibanaRoot, onlyOSS));
+
+  const projectNames = [...projects.values()].map((project) => project.name);
+  log.info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
+
+  for (const project of projects.values()) {
+    await buildProject(project);
+    await copyToBuild(project, kibanaRoot, buildRoot);
+    await applyCorrectPermissions(project, kibanaRoot, buildRoot);
+  }
+}
+
+/**
+ * Copy all the project's files from its Bazel dist directory into the
+ * project build folder.
+ *
+ * When copying all the files into the build, we exclude `node_modules` because
+ * we want the Kibana build to be responsible for actually installing all
+ * dependencies. The primary reason for allowing the Kibana build process to
+ * manage dependencies is that it will "dedupe" them, so we don't include
+ * unnecessary copies of dependencies. We also exclude every related Bazel build
+ * files in order to get the most cleaner package module we can in the final distributable.
+ */
+async function copyToBuild(project: Project, kibanaRoot: string, buildRoot: string) {
+  // We want the package to have the same relative location within the build
+  const relativeProjectPath = relative(kibanaRoot, project.path);
+  const buildProjectPath = resolve(buildRoot, relativeProjectPath);
+
+  const bazelFilesToExclude = ['!*.params', '!*_mappings.json', '!*_options.optionsvalid.d.ts'];
+  await copy(['**/*', '!node_modules/**', ...bazelFilesToExclude], buildProjectPath, {
+    cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath)),
+    dot: true,
+    onlyFiles: true,
+    parents: true,
+  } as copy.Options);
+
+  // If a project is using an intermediate build directory, we special-case our
+  // handling of `package.json`, as the project build process might have copied
+  // (a potentially modified) `package.json` into the intermediate build
+  // directory already. If so, we want to use that `package.json` as the basis
+  // for creating the production-ready `package.json`. If it's not present in
+  // the intermediate build, we fall back to using the project's already defined
+  // `package.json`.
+  const packageJson = (await isFile(join(buildProjectPath, 'package.json')))
+    ? await readPackageJson(buildProjectPath)
+    : project.json;
+
+  const preparedPackageJson = createProductionPackageJson(packageJson);
+  await writePackageJson(buildProjectPath, preparedPackageJson);
+}
+
+async function applyCorrectPermissions(project: Project, kibanaRoot: string, buildRoot: string) {
+  const relativeProjectPath = relative(kibanaRoot, project.path);
+  const buildProjectPath = resolve(buildRoot, relativeProjectPath);
+  const allPluginPaths = await globby([`**/*`], {
+    onlyFiles: false,
+    cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath)),
+    dot: true,
+  });
+
+  for (const pluginPath of allPluginPaths) {
+    const resolvedPluginPath = resolve(buildRoot, pluginPath);
+    if (await isFile(resolvedPluginPath)) {
+      await chmod(resolvedPluginPath, 0o644);
+    }
+
+    if (await isDirectory(resolvedPluginPath)) {
+      await chmod(resolvedPluginPath, 0o755);
+    }
+  }
+}

packages/kbn-pm/src/production/build_non_bazel_production_projects.ts

@@ -20,13 +20,14 @@ import {
 } from '../utils/package_json';
 import {
   buildProjectGraph,
+  getNonBazelProjectsOnly,
   getProjects,
   includeTransitiveProjects,
   topologicallyBatchProjects,
 } from '../utils/projects';
 import { Project } from '..';
 
-export async function buildProductionProjects({
+export async function buildNonBazelProductionProjects({
   kibanaRoot,
   buildRoot,
   onlyOSS,
@@ -35,12 +36,12 @@ export async function buildProductionProjects({
   buildRoot: string;
   onlyOSS?: boolean;
 }) {
-  const projects = await getProductionProjects(kibanaRoot, onlyOSS);
+  const projects = await getNonBazelProjectsOnly(await getProductionProjects(kibanaRoot, onlyOSS));
   const projectGraph = buildProjectGraph(projects);
   const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
 
   const projectNames = [...projects.values()].map((project) => project.name);
-  log.info(`Preparing production build for [${projectNames.join(', ')}]`);
+  log.info(`Preparing non Bazel production build for [${projectNames.join(', ')}]`);
 
   for (const batch of batchedProjects) {
     for (const project of batch) {
@@ -58,7 +59,7 @@ export async function buildProductionProjects({
  * we only include Kibana's transitive _production_ dependencies. If onlyOSS
  * is supplied, we omit projects with build.oss in their package.json set to false.
  */
-async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
+export async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
   const projectPaths = getProjectPaths({ rootPath });
   const projects = await getProjects(rootPath, projectPaths);
   const projectsSubset = [projects.get('kibana')!];
@@ -93,7 +94,7 @@ async function deleteTarget(project: Project) {
   }
 }
 
-async function buildProject(project: Project) {
+export async function buildProject(project: Project) {
   if (project.hasScript('build')) {
     await project.runScript('build');
   }

packages/kbn-pm/src/production/index.ts

@@ -6,4 +6,5 @@
  * Side Public License, v 1.
  */
 
-export { buildProductionProjects } from './build_production_projects';
+export { buildBazelProductionProjects } from './build_bazel_production_projects';
+export { buildNonBazelProductionProjects } from './build_non_bazel_production_projects';

packages/kbn-pm/src/utils/package_json.ts

@@ -34,6 +34,9 @@ export const createProductionPackageJson = (pkgJson: IPackageJson) => ({
 
 export const isLinkDependency = (depVersion: string) => depVersion.startsWith('link:');
 
+export const isBazelPackageDependency = (depVersion: string) =>
+  depVersion.startsWith('link:bazel/bin/');
+
 /**
  * Replaces `link:` dependencies with `file:` dependencies. When installing
  * dependencies, these `file:` dependencies will be copied into `node_modules`
@@ -42,16 +45,27 @@ export const isLinkDependency = (depVersion: string) => depVersion.startsWith('l
  * This will allow us to copy packages into the build and run `yarn`, which
  * will then _copy_ the `file:` dependencies into `node_modules` instead of
  * symlinking like we do in development.
+ *
+ * Additionally it also taken care of replacing `link:bazel/bin/` with
+ * `file:` so we can also support the copy of the Bazel packages dist already into
+ * build/packages to be copied into the node_modules
  */
 export function transformDependencies(dependencies: IPackageDependencies = {}) {
   const newDeps: IPackageDependencies = {};
   for (const name of Object.keys(dependencies)) {
     const depVersion = dependencies[name];
-    if (isLinkDependency(depVersion)) {
-      newDeps[name] = depVersion.replace('link:', 'file:');
-    } else {
+
+    if (!isLinkDependency(depVersion)) {
       newDeps[name] = depVersion;
+      continue;
     }
+
+    if (isBazelPackageDependency(depVersion)) {
+      newDeps[name] = depVersion.replace('link:bazel/bin/', 'file:');
+      continue;
+    }
+
+    newDeps[name] = depVersion.replace('link:', 'file:');
   }
   return newDeps;
 }

packages/kbn-pm/src/utils/project.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import Fs from 'fs';
 import Path from 'path';
 import { inspect } from 'util';
 
@@ -56,6 +57,8 @@ export class Project {
   public readonly devDependencies: IPackageDependencies;
   /** scripts defined in the package.json file for the project [name => body] */
   public readonly scripts: IPackageScripts;
+  /** states if this project is a Bazel package */
+  public readonly bazelPackage: boolean;
 
   public isSinglePackageJsonProject = false;
 
@@ -77,6 +80,9 @@ export class Project {
     this.isSinglePackageJsonProject = this.json.name === 'kibana';
 
     this.scripts = this.json.scripts || {};
+
+    this.bazelPackage =
+      !this.isSinglePackageJsonProject && Fs.existsSync(Path.resolve(this.path, 'BUILD.bazel'));
   }
 
   public get name(): string {
@@ -85,21 +91,27 @@ export class Project {
 
   public ensureValidProjectDependency(project: Project) {
     const relativePathToProject = normalizePath(Path.relative(this.path, project.path));
+    const relativePathToProjectIfBazelPkg = normalizePath(
+      Path.relative(this.path, `bazel/bin/packages/${Path.basename(project.path)}`)
+    );
 
     const versionInPackageJson = this.allDependencies[project.name];
     const expectedVersionInPackageJson = `link:${relativePathToProject}`;
+    const expectedVersionInPackageJsonIfBazelPkg = `link:${relativePathToProjectIfBazelPkg}`;
 
-    // TODO: after introduce bazel to build packages do not allow child projects
-    // to hold dependencies
-
-    if (versionInPackageJson === expectedVersionInPackageJson) {
+    // TODO: after introduce bazel to build all the packages and completely remove the support for kbn packages
+    //  do not allow child projects to hold dependencies
+    if (
+      versionInPackageJson === expectedVersionInPackageJson ||
+      versionInPackageJson === expectedVersionInPackageJsonIfBazelPkg
+    ) {
       return;
     }
 
     const updateMsg = 'Update its package.json to the expected value below.';
     const meta = {
       actual: `"${project.name}": "${versionInPackageJson}"`,
-      expected: `"${project.name}": "${expectedVersionInPackageJson}"`,
+      expected: `"${project.name}": "${expectedVersionInPackageJson}" or "${project.name}": "${expectedVersionInPackageJsonIfBazelPkg}"`,
       package: `${this.name} (${this.packageJsonLocation})`,
     };
 
@@ -133,6 +145,10 @@ export class Project {
     return (this.json.kibana && this.json.kibana.clean) || {};
   }
 
+  public isBazelPackage() {
+    return this.bazelPackage;
+  }
+
   public isFlaggedAsDevOnly() {
     return !!(this.json.kibana && this.json.kibana.devOnly);
   }

packages/kbn-pm/src/utils/projects.ts

@@ -26,7 +26,8 @@ export interface IProjectsOptions {
 export async function getProjects(
   rootPath: string,
   projectsPathsPatterns: string[],
-  { include = [], exclude = [] }: IProjectsOptions = {}
+  { include = [], exclude = [] }: IProjectsOptions = {},
+  bazelOnly: boolean = false
 ) {
   const projects: ProjectMap = new Map();
 
@@ -39,7 +40,9 @@ export async function getProjects(
       const project = await Project.fromPath(projectDir);
 
       const excludeProject =
-        exclude.includes(project.name) || (include.length > 0 && !include.includes(project.name));
+        exclude.includes(project.name) ||
+        (include.length > 0 && !include.includes(project.name)) ||
+        (bazelOnly && !project.isBazelPackage());
 
       if (excludeProject) {
         continue;
@@ -59,6 +62,30 @@ export async function getProjects(
   return projects;
 }
 
+export async function getNonBazelProjectsOnly(projects: ProjectMap) {
+  const bazelProjectsOnly: ProjectMap = new Map();
+
+  for (const project of projects.values()) {
+    if (!project.isBazelPackage()) {
+      bazelProjectsOnly.set(project.name, project);
+    }
+  }
+
+  return bazelProjectsOnly;
+}
+
+export async function getBazelProjectsOnly(projects: ProjectMap) {
+  const bazelProjectsOnly: ProjectMap = new Map();
+
+  for (const project of projects.values()) {
+    if (project.isBazelPackage()) {
+      bazelProjectsOnly.set(project.name, project);
+    }
+  }
+
+  return bazelProjectsOnly;
+}
+
 function packagesFromGlobPattern({ pattern, rootPath }: { pattern: string; rootPath: string }) {
   const globOptions = {
     cwd: rootPath,

src/dev/build/build_distributables.ts

@@ -54,6 +54,7 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
   await run(Tasks.ReplaceFavicon);
   await run(Tasks.CreateEmptyDirsAndFiles);
   await run(Tasks.CreateReadme);
+  await run(Tasks.BuildBazelPackages);
   await run(Tasks.BuildPackages);
   await run(Tasks.BuildKibanaPlatformPlugins);
   await run(Tasks.TranspileBabel);

src/dev/build/tasks/build_packages_task.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { buildProductionProjects } from '@kbn/pm';
+import { buildBazelProductionProjects, buildNonBazelProductionProjects } from '@kbn/pm';
 
 import { mkdirp, Task } from '../lib';
 
@@ -56,11 +56,23 @@ import { mkdirp, Task } from '../lib';
  *   in some way by Kibana itself in production, as it won't otherwise be
  *   included in the production build.
  */
-export const BuildPackages: Task = {
-  description: 'Building distributable versions of packages',
+
+export const BuildBazelPackages: Task = {
+  description: 'Building distributable versions of Bazel packages',
   async run(config, log, build) {
-    await mkdirp(config.resolveFromRepo('target'));
-    await buildProductionProjects({
+    await buildBazelProductionProjects({
+      kibanaRoot: config.resolveFromRepo(),
+      buildRoot: build.resolvePath(),
+      onlyOSS: build.isOss(),
+    });
+  },
+};
+
+export const BuildPackages: Task = {
+  description: 'Building distributable versions of non Bazel packages',
+  async run(config, log, build) {
+    await mkdirp(config.resolveFromRepo('target'));
+    await buildNonBazelProductionProjects({
       kibanaRoot: config.resolveFromRepo(),
       buildRoot: build.resolvePath(),
       onlyOSS: build.isOss(),