[bazel] validate packages/BUILD.bazel in CI (#127241) (#127691)

(cherry picked from commit 9871111cf6)

Co-authored-by: Spencer <spencer@elastic.co>

.buildkite/scripts/steps/checks.sh

@@ -6,6 +6,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/bootstrap.sh
 
 .buildkite/scripts/steps/checks/commit/commit.sh
+.buildkite/scripts/steps/checks/bazel_packages.sh
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh

.buildkite/scripts/steps/checks/bazel_packages.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check Bazel Packages Manifest
+node scripts/generate packages_build_manifest --validate

packages/BUILD.bazel

@@ -1,6 +1,7 @@
 ################
 ################
-## This file is automatically generated, to create a new package use `node scripts/generate package --help`
+## This file is automatically generated, to create a new package use `node scripts/generate package --help` or run
+## `node scripts/generate packages_build_manifest` to regenerate it from the current state of the repo
 ################
 ################
 

packages/kbn-bazel-packages/BUILD.bazel

@@ -40,6 +40,7 @@ RUNTIME_DEPS = [
   "//packages/kbn-utils",
   "//packages/kbn-std",
   "@npm//globby",
+  "@npm//normalize-path",
 ]
 
 # In this array place dependencies necessary to build the types, which will include the
@@ -55,7 +56,9 @@ RUNTIME_DEPS = [
 TYPES_DEPS = [
   "//packages/kbn-utils:npm_module_types",
   "//packages/kbn-std:npm_module_types",
+  "@npm//@types/normalize-path",
   "@npm//globby",
+  "@npm//normalize-path",
 ]
 
 jsts_transpiler(

packages/kbn-bazel-packages/src/bazel_package.test.ts

@@ -15,24 +15,34 @@ const OWN_BAZEL_BUILD_FILE = Fs.readFileSync(Path.resolve(__dirname, '../BUILD.b
 
 describe('hasBuildRule()', () => {
   it('returns true if there is a rule with the name "build"', () => {
-    const pkg = new BazelPackage('foo', {}, OWN_BAZEL_BUILD_FILE);
+    const pkg = new BazelPackage('foo', { name: 'foo' }, OWN_BAZEL_BUILD_FILE);
     expect(pkg.hasBuildRule()).toBe(true);
   });
 
   it('returns false if there is no rule with name "build"', () => {
-    const pkg = new BazelPackage('foo', {}, ``);
+    const pkg = new BazelPackage('foo', { name: 'foo' }, ``);
+    expect(pkg.hasBuildRule()).toBe(false);
+  });
+
+  it('returns false if there is no BUILD.bazel file', () => {
+    const pkg = new BazelPackage('foo', { name: 'foo' });
     expect(pkg.hasBuildRule()).toBe(false);
   });
 });
 
 describe('hasBuildTypesRule()', () => {
   it('returns true if there is a rule with the name "build_types"', () => {
-    const pkg = new BazelPackage('foo', {}, OWN_BAZEL_BUILD_FILE);
+    const pkg = new BazelPackage('foo', { name: 'foo' }, OWN_BAZEL_BUILD_FILE);
     expect(pkg.hasBuildTypesRule()).toBe(true);
   });
 
   it('returns false if there is no rule with name "build_types"', () => {
-    const pkg = new BazelPackage('foo', {}, ``);
+    const pkg = new BazelPackage('foo', { name: 'foo' }, ``);
+    expect(pkg.hasBuildTypesRule()).toBe(false);
+  });
+
+  it('returns false if there is no BUILD.bazel file', () => {
+    const pkg = new BazelPackage('foo', { name: 'foo' });
     expect(pkg.hasBuildTypesRule()).toBe(false);
   });
 });

packages/kbn-bazel-packages/src/bazel_package.ts

@@ -6,14 +6,49 @@
  * Side Public License, v 1.
  */
 
+import { inspect } from 'util';
 import Path from 'path';
 import Fsp from 'fs/promises';
+
+import normalizePath from 'normalize-path';
 import { REPO_ROOT } from '@kbn/utils';
 
 const BUILD_RULE_NAME = /(^|\s)name\s*=\s*"build"/;
 const BUILD_TYPES_RULE_NAME = /(^|\s)name\s*=\s*"build_types"/;
 
+/**
+ * Simple parsed representation of a package.json file, validated
+ * by `assertParsedPackageJson()` and extensible as needed in the future
+ */
+export interface ParsedPackageJson {
+  /**
+   * The name of the package, usually `@kbn/`+something
+   */
+  name: string;
+  /**
+   * All other fields in the package.json are typed as unknown as all we need at this time is "name"
+   */
+  [key: string]: unknown;
+}
+
+function isObj(v: unknown): v is Record<string, unknown> {
+  return !!(typeof v === 'object' && v);
+}
+
+function assertParsedPackageJson(v: unknown): asserts v is ParsedPackageJson {
+  if (!isObj(v) || typeof v.name !== 'string') {
+    throw new Error('Expected parsed package.json to be an object with at least a "name" property');
+  }
+}
+
+/**
+ * Representation of a Bazel Package in the Kibana repository
+ */
 export class BazelPackage {
+  /**
+   * Create a BazelPackage object from a package directory. Reads some files from the package and returns
+   * a Promise for a BazelPackage instance
+   */
   static async fromDir(dir: string) {
     let pkg;
     try {
@@ -22,6 +57,8 @@ export class BazelPackage {
       throw new Error(`unable to parse package.json in [${dir}]: ${error.message}`);
     }
 
+    assertParsedPackageJson(pkg);
+
     let buildBazelContent;
     if (pkg.name !== '@kbn/pm') {
       try {
@@ -31,20 +68,43 @@ export class BazelPackage {
       }
     }
 
-    return new BazelPackage(Path.relative(REPO_ROOT, dir), pkg, buildBazelContent);
+    return new BazelPackage(normalizePath(Path.relative(REPO_ROOT, dir)), pkg, buildBazelContent);
   }
 
   constructor(
-    public readonly repoRelativeDir: string,
-    public readonly pkg: any,
-    public readonly buildBazelContent?: string
+    /**
+     * Relative path from the root of the repository to the package directory
+     */
+    public readonly normalizedRepoRelativeDir: string,
+    /**
+     * Parsed package.json file from the package
+     */
+    public readonly pkg: ParsedPackageJson,
+    /**
+     * Content of the BUILD.bazel file
+     */
+    private readonly buildBazelContent?: string
   ) {}
 
+  /**
+   * Returns true if the package includes a `:build` bazel rule
+   */
   hasBuildRule() {
     return !!(this.buildBazelContent && BUILD_RULE_NAME.test(this.buildBazelContent));
   }
 
+  /**
+   * Returns true if the package includes a `:build_types` bazel rule
+   */
   hasBuildTypesRule() {
     return !!(this.buildBazelContent && BUILD_TYPES_RULE_NAME.test(this.buildBazelContent));
   }
+
+  /**
+   * Custom inspect handler so that logging variables in scripts/generate doesn't
+   * print all the BUILD.bazel files
+   */
+  [inspect.custom]() {
+    return `BazelPackage<${this.normalizedRepoRelativeDir}>`;
+  }
 }

packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.test.ts

@@ -1,85 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { generatePackagesBuildBazelFile } from './generate_packages_build_bazel_file';
-
-import { BazelPackage } from './bazel_package';
-
-it('produces a valid BUILD.bazel file', () => {
-  const packages = [
-    new BazelPackage(
-      'foo',
-      {},
-      `
-        rule(
-          name = "build"
-        )
-        rule(
-          name =   "build_types"
-        )
-      `
-    ),
-    new BazelPackage(
-      'bar',
-      {},
-      `
-        rule(
-          name= "build_types"
-        )
-      `
-    ),
-    new BazelPackage(
-      'bar',
-      {},
-      `
-        rule(
-          name ="build"
-        )
-      `
-    ),
-    new BazelPackage('bar', {}),
-  ];
-
-  expect(generatePackagesBuildBazelFile(packages)).toMatchInlineSnapshot(`
-    "################
-    ################
-    ## This file is automatically generated, to create a new package use \`node scripts/generate package --help\`
-    ################
-    ################
-
-    # It will build all declared code packages
-    filegroup(
-      name = \\"build_pkg_code\\",
-      srcs = [
-        \\"//foo:build\\",
-        \\"//bar:build\\",
-      ],
-    )
-
-    # It will build all declared package types
-    filegroup(
-      name = \\"build_pkg_types\\",
-      srcs = [
-        \\"//foo:build_types\\",
-        \\"//bar:build_types\\",
-      ],
-    )
-
-    # Grouping target to call all underlying packages build
-    # targets so we can build them all at once
-    # It will auto build all declared code packages and types packages
-    filegroup(
-      name = \\"build\\",
-      srcs = [
-        \\":build_pkg_code\\",
-        \\":build_pkg_types\\"
-      ],
-    )
-    "
-  `);
-});

packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.ts

@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { BazelPackage } from './bazel_package';
-
-export function generatePackagesBuildBazelFile(packages: BazelPackage[]) {
-  return `################
-################
-## This file is automatically generated, to create a new package use \`node scripts/generate package --help\`
-################
-################
-
-# It will build all declared code packages
-filegroup(
-  name = "build_pkg_code",
-  srcs = [
-${packages
-  .flatMap((p) => (p.hasBuildRule() ? `    "//${p.repoRelativeDir}:build",` : []))
-  .join('\n')}
-  ],
-)
-
-# It will build all declared package types
-filegroup(
-  name = "build_pkg_types",
-  srcs = [
-${packages
-  .flatMap((p) => (p.hasBuildTypesRule() ? `    "//${p.repoRelativeDir}:build_types",` : []))
-  .join('\n')}
-  ],
-)
-
-# Grouping target to call all underlying packages build
-# targets so we can build them all at once
-# It will auto build all declared code packages and types packages
-filegroup(
-  name = "build",
-  srcs = [
-    ":build_pkg_code",
-    ":build_pkg_types"
-  ],
-)
-`;
-}

packages/kbn-bazel-packages/src/index.ts

@@ -8,4 +8,3 @@
 
 export * from './discover_packages';
 export type { BazelPackage } from './bazel_package';
-export * from './generate_packages_build_bazel_file';

packages/kbn-generate/src/cli.ts

@@ -12,6 +12,7 @@ import { Render } from './lib/render';
 import { ContextExtensions } from './generate_command';
 
 import { PackageCommand } from './commands/package_command';
+import { PackagesBuildManifestCommand } from './commands/packages_build_manifest_command';
 
 /**
  * Runs the generate CLI. Called by `node scripts/generate` and not intended for use outside of that script
@@ -26,6 +27,6 @@ export function runGenerateCli() {
         };
       },
     },
-    [PackageCommand]
+    [PackageCommand, PackagesBuildManifestCommand]
   ).execute();
 }

packages/kbn-generate/src/commands/package_command.ts

@@ -13,10 +13,10 @@ import normalizePath from 'normalize-path';
 import globby from 'globby';
 
 import { REPO_ROOT } from '@kbn/utils';
-import { discoverBazelPackages, generatePackagesBuildBazelFile } from '@kbn/bazel-packages';
+import { discoverBazelPackages } from '@kbn/bazel-packages';
 import { createFailError, createFlagError, isFailError, sortPackageJson } from '@kbn/dev-utils';
 
-import { ROOT_PKG_DIR, PKG_TEMPLATE_DIR } from '../paths';
+import { TEMPLATE_DIR, ROOT_PKG_DIR, PKG_TEMPLATE_DIR } from '../paths';
 import type { GenerateCommand } from '../generate_command';
 
 export const PackageCommand: GenerateCommand = {
@@ -49,7 +49,7 @@ export const PackageCommand: GenerateCommand = {
 
     const containingDir = flags.dir ? Path.resolve(`${flags.dir}`) : ROOT_PKG_DIR;
     const packageDir = Path.resolve(containingDir, name.slice(1).replace('/', '-'));
-    const repoRelativeDir = normalizePath(Path.relative(REPO_ROOT, packageDir));
+    const normalizedRepoRelativeDir = normalizePath(Path.relative(REPO_ROOT, packageDir));
 
     try {
       await Fsp.readdir(packageDir);
@@ -107,8 +107,8 @@ export const PackageCommand: GenerateCommand = {
           name,
           web,
           dev,
-          directoryName: Path.basename(repoRelativeDir),
-          repoRelativeDir,
+          directoryName: Path.basename(normalizedRepoRelativeDir),
+          normalizedRepoRelativeDir,
         },
       });
     }
@@ -122,17 +122,20 @@ export const PackageCommand: GenerateCommand = {
       ? [packageJson.devDependencies, packageJson.dependencies]
       : [packageJson.dependencies, packageJson.devDependencies];
 
-    addDeps[name] = `link:bazel-bin/${repoRelativeDir}`;
-    addDeps[typePkgName] = `link:bazel-bin/${repoRelativeDir}/npm_module_types`;
+    addDeps[name] = `link:bazel-bin/${normalizedRepoRelativeDir}`;
+    addDeps[typePkgName] = `link:bazel-bin/${normalizedRepoRelativeDir}/npm_module_types`;
     delete removeDeps[name];
     delete removeDeps[typePkgName];
 
     await Fsp.writeFile(packageJsonPath, sortPackageJson(JSON.stringify(packageJson)));
     log.info('Updated package.json file');
 
-    await Fsp.writeFile(
+    await render.toFile(
+      Path.resolve(TEMPLATE_DIR, 'packages_BUILD.bazel.ejs'),
       Path.resolve(REPO_ROOT, 'packages/BUILD.bazel'),
-      generatePackagesBuildBazelFile(await discoverBazelPackages())
+      {
+        packages: await discoverBazelPackages(),
+      }
     );
     log.info('Updated packages/BUILD.bazel');
 

packages/kbn-generate/src/commands/packages_build_manifest_command.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fsp from 'fs/promises';
+
+import { REPO_ROOT } from '@kbn/utils';
+import { discoverBazelPackages } from '@kbn/bazel-packages';
+
+import { TEMPLATE_DIR } from '../paths';
+import { GenerateCommand } from '../generate_command';
+import { validateFile } from '../lib/validate_file';
+
+const USAGE = `node scripts/generate packages_build_manifest`;
+
+export const PackagesBuildManifestCommand: GenerateCommand = {
+  name: 'packages_build_manifest',
+  usage: USAGE,
+  description: 'Generate the packages/BUILD.bazel file',
+  flags: {
+    boolean: ['validate'],
+    help: `
+      --validate         Rather than writing the generated output to disk, validate that the content on disk is in sync with the
+    `,
+  },
+  async run({ log, render, flags }) {
+    const validate = !!flags.validate;
+
+    const packages = await discoverBazelPackages();
+    const dest = Path.resolve(REPO_ROOT, 'packages/BUILD.bazel');
+    const relDest = Path.relative(process.cwd(), dest);
+
+    const content = await render.toString(
+      Path.join(TEMPLATE_DIR, 'packages_BUILD.bazel.ejs'),
+      dest,
+      {
+        packages,
+      }
+    );
+
+    if (validate) {
+      await validateFile(log, USAGE, dest, content);
+      return;
+    }
+
+    await Fsp.writeFile(dest, content);
+    log.success('Wrote', relDest);
+  },
+};

packages/kbn-generate/src/lib/validate_file.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fsp from 'fs/promises';
+import Path from 'path';
+
+import { ToolingLog, createFailError, diffStrings } from '@kbn/dev-utils';
+
+export async function validateFile(log: ToolingLog, usage: string, path: string, expected: string) {
+  const relPath = Path.relative(process.cwd(), path);
+
+  let current;
+  try {
+    current = await Fsp.readFile(path, 'utf8');
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      throw createFailError(`${relPath} is missing, please run "${usage}" and commit the result`);
+    }
+
+    throw error;
+  }
+
+  if (current !== expected) {
+    log.error(`${relPath} is outdated:\n${diffStrings(expected, current)}`);
+    throw createFailError(`${relPath} is outdated, please run "${usage}" and commit the result`);
+  }
+
+  log.success(`${relPath} is valid`);
+}

packages/kbn-generate/templates/package/jest.config.js.ejs

@@ -1,5 +1,5 @@
 module.exports = {
   preset: <%- js(pkg.web ? '@kbn/test' : '@kbn/test/jest_node') %>,
   rootDir: '../..',
-  roots: [<%- js(`<rootDir>/${pkg.repoRelativeDir}`) %>],
+  roots: [<%- js(`<rootDir>/${pkg.normalizedRepoRelativeDir}`) %>],
 };

packages/kbn-generate/templates/packages_BUILD.bazel.ejs

@@ -0,0 +1,37 @@
+################
+################
+## This file is automatically generated, to create a new package use `node scripts/generate package --help` or run
+## `node scripts/generate packages_build_manifest` to regenerate it from the current state of the repo
+################
+################
+
+# It will build all declared code packages
+filegroup(
+  name = "build_pkg_code",
+  srcs = [
+<% for (const p of packages.filter(p => p.hasBuildRule())) { _%>
+    "//<%- p.normalizedRepoRelativeDir %>:build",
+<% } _%>
+  ],
+)
+
+# It will build all declared package types
+filegroup(
+  name = "build_pkg_types",
+  srcs = [
+<% for (const p of packages.filter(p => p.hasBuildTypesRule())) { _%>
+    "//<%- p.normalizedRepoRelativeDir %>:build_types",
+<% } _%>
+  ],
+)
+
+# Grouping target to call all underlying packages build
+# targets so we can build them all at once
+# It will auto build all declared code packages and types packages
+filegroup(
+  name = "build",
+  srcs = [
+    ":build_pkg_code",
+    ":build_pkg_types"
+  ],
+)

src/dev/precommit_hook/casing_check_config.js

@@ -70,6 +70,9 @@ export const IGNORE_FILE_GLOBS = [
 
   // Buildkite
   '.buildkite/**/*',
+
+  // generator templates use weird filenames based on the requirements for the files they're generating
+  'packages/kbn-generate/templates/**/*',
 ];
 
 /**
@@ -110,10 +113,7 @@ export const IGNORE_DIRECTORY_GLOBS = [
  *
  * @type {Array}
  */
-export const REMOVE_EXTENSION = [
-  'packages/kbn-plugin-generator/template/**/*.ejs',
-  'packages/kbn-generate/templates/**/*.ejs',
-];
+export const REMOVE_EXTENSION = ['packages/kbn-plugin-generator/template/**/*.ejs'];
 
 /**
  * DO NOT ADD FILES TO THIS LIST!!