chore(NA): creates pkg_npm_types bazel rule (#116465)

* chore(NA): auto creation of the package.json for the new types pkg rule

* chore(NA): first alpha api extractor working version

* chore(NA): support kbn-analytics

* chore(NA): correctly read tsconfig files and deps from ts_config rule

* chore(NA): layed out pkg_npm_types tree artifact custom rule

* chore(NA): missing todos

* chore(NA): node modules link mapping

* chore(NA): fully working pkg_npm_types rule

* chore(NA): fix changes on new packages using elastic datemath pkgs

* docs(NA): remove todo

* docs(NA): last todo text correction

* chore(NA): removed commented lines

* fix(NA): include missing package version

* chore(NA): include license keys

* chore(NA): change mock types package into private

* chore(NA): disable validator on ts_project rule

* chore(NA): use the wrapper for ts_project

* commit using @elastic.co

* chore(NA): commit using @elastic.co

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

package.json

@@ -512,6 +512,7 @@
     "@types/delete-empty": "^2.0.0",
     "@types/ejs": "^3.0.6",
     "@types/elasticsearch": "^5.0.33",
+    "@types/elastic__datemath": "link:bazel-bin/packages/elastic-datemath/npm_module_types",
     "@types/enzyme": "^3.10.8",
     "@types/eslint": "^7.28.0",
     "@types/extract-zip": "^1.6.2",

packages/BUILD.bazel

@@ -1,7 +1,6 @@
-# Grouping target to call all underlying packages build
-# targets so we can build them all at once
+# It will build all declared code packages
 filegroup(
-  name = "build",
+  name = "build_pkg_code",
   srcs = [
       "//packages/elastic-apm-synthtrace:build",
       "//packages/elastic-datemath:build",
@@ -70,3 +69,24 @@ filegroup(
       "//packages/kbn-utils:build",
   ],
 )
+
+# It will build all declared package types
+filegroup(
+  name = "build_pkg_types",
+  srcs = [
+      "//packages/elastic-datemath:build_types",
+  ],
+)
+
+
+
+# Grouping target to call all underlying packages build
+# targets so we can build them all at once
+# It will auto build all declared code packages and types packages
+filegroup(
+  name = "build",
+  srcs = [
+      ":build_pkg_code",
+      ":build_pkg_types"
+  ],
+)

packages/elastic-apm-synthtrace/BUILD.bazel

@@ -37,7 +37,7 @@ RUNTIME_DEPS = [
 ]
 
 TYPES_DEPS = [
-  "//packages/elastic-datemath",
+  "//packages/elastic-datemath:npm_module_types",
   "@npm//@elastic/elasticsearch",
   "@npm//moment",
   "@npm//p-limit",

packages/elastic-datemath/BUILD.bazel

@@ -1,9 +1,10 @@
-load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
-load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
-load("//src/dev/bazel:index.bzl", "jsts_transpiler")
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "ts_project", "pkg_npm", "pkg_npm_types")
 
 PKG_BASE_NAME = "elastic-datemath"
 PKG_REQUIRE_NAME = "@elastic/datemath"
+TYPES_PKG_REQUIRE_NAME = "@types/elastic__datemath"
 
 SOURCE_FILES = glob([
   "src/index.ts",
@@ -26,7 +27,7 @@ TYPES_DEPS = [
   "@npm//@types/node",
 ]
 
-DEPS = TYPES_DEPS
+RUNTIME_DEPS = TYPES_DEPS
 
 jsts_transpiler(
   name = "target_node",
@@ -47,20 +48,20 @@ ts_project(
   name = "tsc_types",
   args = ['--pretty'],
   srcs = SRCS,
-  deps = DEPS,
+  deps = TYPES_DEPS,
   declaration = True,
   declaration_map = True,
   emit_declaration_only = True,
   out_dir = "target_types",
   source_map = True,
   root_dir = "src",
-  tsconfig = ":tsconfig",
+  tsconfig = ":tsconfig"
 )
 
 js_library(
   name = PKG_BASE_NAME,
   srcs = NPM_MODULE_EXTRA_FILES,
-  deps = DEPS + [":target_node", ":tsc_types"],
+  deps = RUNTIME_DEPS + [":target_node"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],
 )
@@ -79,3 +80,20 @@ filegroup(
   ],
   visibility = ["//visibility:public"],
 )
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = TYPES_PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [
+    ":npm_module_types",
+  ],
+  visibility = ["//visibility:public"],
+)

packages/elastic-datemath/package.json

@@ -4,7 +4,6 @@
   "description": "elasticsearch datemath parser, used in kibana",
   "license": "Apache-2.0",
   "main": "./target_node/index.js",
-  "types": "./target_types/index.d.ts",
   "peerDependencies": {
     "moment": "^2.24.0"
   }

packages/kbn-securitysolution-io-ts-utils/BUILD.bazel

@@ -37,7 +37,7 @@ RUNTIME_DEPS = [
 ]
 
 TYPES_DEPS = [
-  "//packages/elastic-datemath",
+  "//packages/elastic-datemath:npm_module_types",
   "@npm//fp-ts",
   "@npm//io-ts",
   "@npm//moment",

packages/kbn-ui-shared-deps-src/BUILD.bazel

@@ -41,7 +41,7 @@ RUNTIME_DEPS = [
 ]
 
 TYPES_DEPS = [
-  "//packages/elastic-datemath",
+  "//packages/elastic-datemath:npm_module_types",
   "//packages/elastic-safer-lodash-set",
   "//packages/kbn-analytics",
   "//packages/kbn-babel-preset",

src/dev/bazel/index.bzl

@@ -6,14 +6,16 @@
 # Side Public License, v 1.
 #
 
-"""Public API interface for Bazel custom rules.
+"""Public API interface for all Kibana Bazel custom rules.
 Please do not import from any other files when looking to use a custom rule
 """
 
 load("//src/dev/bazel:jsts_transpiler.bzl", _jsts_transpiler = "jsts_transpiler")
-load("//src/dev/bazel:ts_project.bzl", _ts_project = "ts_project")
 load("//src/dev/bazel:pkg_npm.bzl", _pkg_npm = "pkg_npm")
+load("//src/dev/bazel/pkg_npm_types:index.bzl", _pkg_npm_types = "pkg_npm_types")
+load("//src/dev/bazel:ts_project.bzl", _ts_project = "ts_project")
 
 jsts_transpiler = _jsts_transpiler
 pkg_npm = _pkg_npm
+pkg_npm_types = _pkg_npm_types
 ts_project = _ts_project

src/dev/bazel/pkg_npm_types/BUILD.bazel

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@build_bazel_rules_nodejs//internal/node:node.bzl", "nodejs_binary")
+
+filegroup(
+  name = "packager_all_files",
+  srcs = glob([
+    "packager/*",
+  ]),
+)
+
+exports_files(
+  [
+    "package_json.mustache",
+  ],
+  visibility = ["//visibility:public"]
+)
+
+nodejs_binary(
+  name = "_packager",
+  data = [
+    "@npm//@bazel/typescript",
+    "@npm//@microsoft/api-extractor",
+    "@npm//mustache",
+    ":packager_all_files"
+  ],
+  entry_point = ":packager/index.js",
+)

src/dev/bazel/pkg_npm_types/index.bzl

@@ -0,0 +1,15 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+"""Public API interface for pkg_npm_types rule.
+Please do not import from any other files when looking to this rule
+"""
+
+load(":pkg_npm_types.bzl", _pkg_npm_types = "pkg_npm_types")
+
+pkg_npm_types = _pkg_npm_types

src/dev/bazel/pkg_npm_types/package_json.mustache

@@ -0,0 +1,8 @@
+{
+  "name": "{{{NAME}}}",
+  "description": "Generated by Bazel",
+  "types": "./index.d.ts",
+  "private": true,
+  "license": "MIT",
+  "version": "1.1.0"
+}

src/dev/bazel/pkg_npm_types/packager/create_api_extraction.js

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+const { format, parseTsconfig } = require('@bazel/typescript');
+const { Extractor, ExtractorConfig } = require('@microsoft/api-extractor');
+const fs = require('fs');
+const path = require('path');
+
+function createApiExtraction(
+  tsConfig,
+  entryPoint,
+  dtsBundleOut,
+  apiReviewFolder,
+  acceptApiUpdates = false
+) {
+  const [parsedConfig, errors] = parseTsconfig(tsConfig);
+  if (errors && errors.length) {
+    console.error(format('', errors));
+    return 1;
+  }
+  const pkgJson = path.resolve(path.dirname(entryPoint), 'package.json');
+  if (!fs.existsSync(pkgJson)) {
+    fs.writeFileSync(
+      pkgJson,
+      JSON.stringify({
+        name: 'GENERATED-BY-BAZEL',
+        description: 'This is a dummy package.json as API Extractor always requires one.',
+        types: './index.d.ts',
+        private: true,
+        license: 'SSPL-1.0 OR Elastic License 2.0',
+        version: '1.0.0',
+      })
+    );
+  }
+  // API extractor doesn't always support the version of TypeScript used in the repo
+  // example: at the moment it is not compatable with 3.2
+  // to use the internal TypeScript we shall not create a program but rather pass a parsed tsConfig.
+  const parsedTsConfig = parsedConfig.config;
+  const extractorOptions = {
+    localBuild: acceptApiUpdates,
+  };
+  const configObject = {
+    compiler: {
+      overrideTsconfig: parsedTsConfig,
+    },
+    projectFolder: path.resolve(path.dirname(tsConfig)),
+    mainEntryPointFilePath: path.resolve(entryPoint),
+    apiReport: {
+      enabled: !!apiReviewFolder,
+      // TODO(alan-agius4): remove this folder name when the below issue is solved upstream
+      // See: https://github.com/microsoft/web-build-tools/issues/1470
+      reportFileName: (apiReviewFolder && path.resolve(apiReviewFolder)) || 'invalid',
+    },
+    docModel: {
+      enabled: false,
+    },
+    dtsRollup: {
+      enabled: !!dtsBundleOut,
+      untrimmedFilePath: dtsBundleOut && path.resolve(dtsBundleOut),
+    },
+    tsdocMetadata: {
+      enabled: false,
+    },
+  };
+  const options = {
+    configObject,
+    packageJson: undefined,
+    packageJsonFullPath: pkgJson,
+    configObjectFullPath: undefined,
+  };
+  const extractorConfig = ExtractorConfig.prepare(options);
+  const { succeeded } = Extractor.invoke(extractorConfig, extractorOptions);
+  // API extractor errors are emitted by it's logger.
+  return succeeded ? 0 : 1;
+}
+
+module.exports.createApiExtraction = createApiExtraction;

src/dev/bazel/pkg_npm_types/packager/generate_package_json.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const fs = require('fs');
+const Mustache = require('mustache');
+const path = require('path');
+
+function generatePackageJson(outputBasePath, packageJsonTemplatePath, rawPackageJsonTemplateArgs) {
+  const packageJsonTemplateArgsInTuples = rawPackageJsonTemplateArgs.reduce(
+    (a, v) => {
+      const lastTupleIdx = a.length - 1;
+      const lastTupleSize = a[lastTupleIdx].length;
+
+      if (lastTupleSize < 2) {
+        a[lastTupleIdx].push(v);
+
+        return a;
+      }
+
+      return a.push([v]);
+    },
+    [[]]
+  );
+  const packageJsonTemplateArgs = Object.fromEntries(new Map(packageJsonTemplateArgsInTuples));
+
+  try {
+    const template = fs.readFileSync(packageJsonTemplatePath);
+    const renderedTemplate = Mustache.render(template.toString(), packageJsonTemplateArgs);
+    fs.writeFileSync(path.resolve(outputBasePath, 'package.json'), renderedTemplate);
+  } catch (e) {
+    console.error(e);
+    return 1;
+  }
+
+  return 0;
+}
+
+module.exports.generatePackageJson = generatePackageJson;

src/dev/bazel/pkg_npm_types/packager/index.js

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { createApiExtraction } = require('./create_api_extraction');
+const { generatePackageJson } = require('./generate_package_json');
+const path = require('path');
+
+const DEBUG = false;
+
+if (require.main === module) {
+  if (DEBUG) {
+    console.error(`
+pkg_npm_types packager: running with
+  cwd: ${process.cwd()}
+  argv:
+    ${process.argv.join('\n    ')}
+  `);
+  }
+
+  // layout args
+  const [
+    outputBasePath,
+    packageJsonTemplatePath,
+    stringifiedPackageJsonTemplateArgs,
+    tsConfig,
+    entryPoint,
+  ] = process.argv.slice(2);
+  const dtsBundleOutput = path.resolve(outputBasePath, 'index.d.ts');
+
+  // generate pkg json output
+  const generatePackageJsonRValue = generatePackageJson(
+    outputBasePath,
+    packageJsonTemplatePath,
+    stringifiedPackageJsonTemplateArgs.split(',')
+  );
+  // create api extraction output
+  const createApiExtractionRValue = createApiExtraction(tsConfig, entryPoint, dtsBundleOutput);
+
+  // setup correct exit code
+  process.exitCode = generatePackageJsonRValue || createApiExtractionRValue;
+}

src/dev/bazel/pkg_npm_types/pkg_npm_types.bzl

@@ -0,0 +1,148 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+load("@npm//@bazel/typescript/internal:ts_config.bzl", "TsConfigInfo")
+load("@build_bazel_rules_nodejs//:providers.bzl", "run_node", "LinkablePackageInfo", "declaration_info")
+load("@build_bazel_rules_nodejs//internal/linker:link_node_modules.bzl", "module_mappings_aspect")
+
+
+#### TODO
+# Implement a way to produce source maps for api extractor
+# summarised types as referenced at (https://github.com/microsoft/rushstack/issues/1886#issuecomment-933997910)
+
+def _deps_inputs(ctx):
+  """Returns all transitively referenced files on deps """
+  deps_files_depsets = []
+  for dep in ctx.attr.deps:
+    # Collect whatever is in the "data"
+    deps_files_depsets.append(dep.data_runfiles.files)
+
+    # Only collect DefaultInfo files (not transitive)
+    deps_files_depsets.append(dep.files)
+
+  deps_files = depset(transitive = deps_files_depsets).to_list()
+  return deps_files
+
+def _calculate_entrypoint_path(ctx):
+  return _join(ctx.bin_dir.path, ctx.label.package, _get_types_outdir_name(ctx), ctx.attr.entrypoint_name)
+
+def _get_types_outdir_name(ctx):
+  base_out_folder = _join(ctx.bin_dir.path, ctx.label.package)
+  type_dep_path = ctx.files.deps[0].path
+  type_dep_path_without_base_out = type_dep_path.replace(base_out_folder + "/", "", 1)
+  types_outdir_name = type_dep_path_without_base_out.split("/")[0]
+  return types_outdir_name
+
+def _join(*elements):
+  segments = [f for f in elements if f]
+  if len(segments):
+    return "/".join(segments)
+  return "."
+
+def _tsconfig_inputs(ctx):
+  """Returns all transitively referenced tsconfig files from "tsconfig" """
+  all_inputs = []
+  if TsConfigInfo in ctx.attr.tsconfig:
+    all_inputs.extend(ctx.attr.tsconfig[TsConfigInfo].deps)
+  else:
+    all_inputs.append(ctx.file.tsconfig)
+  return all_inputs
+
+def _pkg_npm_types_impl(ctx):
+  # input declarations
+  deps_inputs = _deps_inputs(ctx)
+  tsconfig_inputs = _tsconfig_inputs(ctx)
+  inputs = ctx.files.srcs[:]
+  inputs.extend(tsconfig_inputs)
+  inputs.extend(deps_inputs)
+  inputs.append(ctx.file._generated_package_json_template)
+
+  # output dir declaration
+  package_path = ctx.label.package
+  package_dir = ctx.actions.declare_directory(ctx.label.name)
+  outputs = [package_dir]
+
+  # gathering template args
+  template_args = [
+    "NAME", ctx.attr.package_name
+  ]
+
+  # layout api extractor arguments
+  extractor_args = ctx.actions.args()
+
+  ## general args layout
+  ### [0] = base output dir
+  ### [1] = generated package json template input file path
+  ### [2] = stringified template args
+  ### [3] = tsconfig input file path
+  ### [4] = entry point from provided types to summarise
+  extractor_args.add(package_dir.path)
+  extractor_args.add(ctx.file._generated_package_json_template.path)
+  extractor_args.add_joined(template_args, join_with = ",", omit_if_empty = False)
+  extractor_args.add(tsconfig_inputs[0])
+  extractor_args.add(_calculate_entrypoint_path(ctx))
+
+  run_node(
+    ctx,
+    inputs = inputs,
+    arguments = [extractor_args],
+    outputs = outputs,
+    mnemonic = "AssembleNpmTypesPackage",
+    progress_message = "Assembling npm types package %s" % package_dir.short_path,
+    executable = "_packager",
+  )
+
+  # this is a tree artifact, so correctly build the return
+  package_dir_depset = depset([package_dir])
+
+  return [
+    DefaultInfo(
+      files = package_dir_depset,
+      runfiles = ctx.runfiles([package_dir]),
+    ),
+    declaration_info(
+      declarations = depset([package_dir])
+    ),
+    LinkablePackageInfo(
+      package_name = ctx.attr.package_name,
+      package_path = "",
+      path = package_dir.path,
+      files = package_dir_depset,
+    )
+  ]
+
+pkg_npm_types = rule(
+  implementation = _pkg_npm_types_impl,
+  attrs = {
+    "deps": attr.label_list(
+      doc = """Other targets which are the base types to summarise from""",
+      allow_files = True,
+      aspects = [module_mappings_aspect],
+    ),
+    "entrypoint_name": attr.string(
+      doc = """Entrypoint name of the types files group to summarise""",
+      default = "index.d.ts",
+    ),
+    "package_name": attr.string(),
+    "srcs": attr.label_list(
+      doc = """Files inside this directory which are inputs for the types to summarise.""",
+      allow_files = True,
+    ),
+    "tsconfig": attr.label(mandatory = True, allow_single_file = [".json"]),
+    "_packager": attr.label(
+      doc = "Target that executes the npm types package assembler binary",
+      executable = True,
+      cfg = "host",
+      default = Label("//src/dev/bazel/pkg_npm_types:_packager"),
+    ),
+    "_generated_package_json_template": attr.label(
+      allow_single_file = True,
+      default = "package_json.mustache",
+    ),
+  },
+)

yarn.lock

@@ -6101,6 +6101,10 @@
   resolved "https://registry.yarnpkg.com/@types/ejs/-/ejs-3.0.6.tgz#aca442289df623bfa8e47c23961f0357847b83fe"
   integrity sha512-fj1hi+ZSW0xPLrJJD+YNwIh9GZbyaIepG26E/gXvp8nCa2pYokxUYO1sK9qjGxp2g8ryZYuon7wmjpwE2cyASQ==
 
+"@types/elastic__datemath@link:bazel-bin/packages/elastic-datemath/npm_module_types":
+  version "0.0.0"
+  uid ""
+
 "@types/elasticsearch@^5.0.33":
   version "5.0.33"
   resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.33.tgz#b0fd37dc674f498223b6d68c313bdfd71f4d812b"