implement "plugin" package type (#149370)

This PR updates the core discovery logic to support loading plugins from
packages. This logic is additive, so that the existing plugins in the
repo and third-party plugins can continue to be loaded via the existing
mechanism, but with https://github.com/elastic/kibana/pull/148130 we
will be automatically migrating all plugins in the repo to packages,
which will use this logic.

The logic is already in-use in that PR, and was developed there, but
extracted here for easier review.

The logic is relatively simple, where a list of packages in the repo are
attached to the core `Env` and then filtered by core before converting
all plugin packages to `PluginWrapper`. The `PluginWrapper` still
exposes the plugin manifest to the rest of the code, and it is used in
many places, so rather than making changes to the `PluginWrapper` I'm
faking a legacy plugin manifest with the plugin package manifest.

@elastic/kibana-core: I'm going to need some help identifying what we
need to get test coverage for. This is a pretty simple addition to the
core IMO, and if it didn't work then nothing would work, so I'm pretty
confident in it, but would still appreciate your feedback.

dev_docs/operations/packages_idm.mdx

@@ -51,19 +51,16 @@ Package types allow us to have many different packages, pre-defined build tasks,
 : These packages can be imported from all other packages.
 
 `shared-browser`
-: These packages can be imported from `shared-browser` and `plugin-browser` packages. `shared-browser` packages may include Storybooks.
+: These packages can be imported from `shared-browser` and `public` directories within `plugin` packages. `shared-browser` packages may include Storybooks.
 
 `shared-server`
-: These packages can be imported from `shared-server` and `plugin-server` packages.
+: These packages can be imported from `shared-server` and `server` directories within `plugin` packages.
 
 `shared-scss`
-: These packages can be imported by `shared-browser` and `plugin-browser` packages, and expose an `index.scss` file to consumers instead of an `index.ts` file.
+: These packages can be imported by `shared-browser` and `public` directories within `plugin` packages, and expose an `index.scss` file to consumers instead of an `index.ts` file.
 
-`plugin-browser`
-: These packages expose types to other packages via a root `types.ts` file.  Module IDs must end with `-plugin-browser`. Consumers must use `import type` statements.
-
-`plugin-server`
-: These packages expose types to other packages via a root `types.ts` file. Module IDs must end with `-plugin-server`. Consumers must use `import type` statements.
+`plugin`
+: These packages were automatically created from the existing plugins at the time we switched everything over to packages. Module IDs must end with `-plugin`. Consumers must use `import type` statements.
 
 `functional-test`
 : These packages expose one or more functional testing configurations, including API integration tests, and can not be imported by other packages. Separating functional and integration tests allows us to iterate on tests without rebuilding the application. Similarly, iterating and updating the application should mostly mean the tests don't need to rebuild.
@@ -160,8 +157,7 @@ The solution to resolving a circular dependency has, thus far, been to break out
 
 There are a few package naming rules:
  - all packages must use the `@kbn/` namespace
- - `plugin-browser`-type packages must end with `-plugin-browser`
- - `plugin-server- type packages must end with `-plugin-server`
+ - `plugin`-type packages must end with `-plugin`
  - considering that we operate in a global namespace, avoid overly generic names
 
 Other than these rules, it's up to you and your team to decide on an appropriate name for your package.
@@ -205,4 +201,4 @@ We're now entering Phase 2 of the plan, more details about the phases of our pla
 
 [status]: #what-works-now
 [idm-rfc]: https://docs.google.com/document/d/1Bhg601MoGQjqGMGdLWSLnkopRexwrcbf_0MNcUkhx3I "Internal Dependency Management RFC on Google Docs"
-[pkgDirs]: https://github.com/elastic/kibana/blob/main/packages/kbn-bazel-packages/src/bazel_package_dirs.ts#L22
+[pkgDirs]: https://github.com/elastic/kibana/blob/main/packages/kbn-repo-packages/src/repo_package_dirs.js#L19

packages/core/base/core-base-server-mocks/src/core_context.mock.ts

@@ -23,7 +23,7 @@ function create({
   logger?: jest.Mocked<LoggerFactory>;
   configService?: jest.Mocked<IConfigService>;
 } = {}): DeeplyMockedKeys<CoreContext> {
-  return { coreId: Symbol(), env, logger, configService };
+  return { coreId: Symbol(), env: env as DeeplyMockedKeys<typeof env>, logger, configService };
 }
 
 export const mockCoreContext = {

packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_from_plugin_package.test.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { PluginPackageManifest } from '@kbn/repo-packages';
+import { PluginType } from '@kbn/core-base-common';
+import { pluginManifestFromPluginPackage } from './plugin_manifest_from_plugin_package';
+
+const kibanaVersion = `1.${Math.round(10 * Math.random())}.1`;
+const minimal: PluginPackageManifest = {
+  type: 'plugin',
+  id: '@kbn/some-legacy-plugin',
+  owner: ['@elastic/team-a', '@elastic/team-b'],
+  plugin: {
+    id: 'someLegacyPluginId',
+    browser: true,
+    server: true,
+  },
+};
+const basic: PluginPackageManifest = {
+  ...minimal,
+  plugin: {
+    ...minimal.plugin,
+    type: 'preboot',
+    configPath: ['some', 'legacy'],
+    enabledOnAnonymousPages: false,
+    extraPublicDirs: ['foo', 'bar'],
+    optionalPlugins: ['someOtherPlugin'],
+    requiredBundles: ['someRequiresBundlePlugin'],
+    requiredPlugins: ['someRequiredPlugin'],
+  },
+  serviceFolders: ['foo', 'bar'],
+};
+
+describe('pluginManifestFromPluginPackage()', () => {
+  it('consumes correct values from plugin package manifest', () => {
+    expect(pluginManifestFromPluginPackage('static', basic)).toMatchInlineSnapshot(`
+      Object {
+        "configPath": Array [
+          "some",
+          "legacy",
+        ],
+        "enabledOnAnonymousPages": false,
+        "id": "someLegacyPluginId",
+        "kibanaVersion": "static",
+        "optionalPlugins": Array [
+          "someOtherPlugin",
+        ],
+        "owner": Object {
+          "name": "@elastic/team-a & @elastic/team-b",
+        },
+        "requiredBundles": Array [
+          "someRequiresBundlePlugin",
+        ],
+        "requiredPlugins": Array [
+          "someRequiredPlugin",
+        ],
+        "server": true,
+        "serviceFolders": Array [
+          "foo",
+          "bar",
+        ],
+        "type": "preboot",
+        "ui": true,
+        "version": "1.0.0",
+      }
+    `);
+  });
+
+  it('applies correct defaults', () => {
+    const pm = pluginManifestFromPluginPackage(kibanaVersion, minimal);
+    expect(pm).toHaveProperty('type', PluginType.standard);
+    expect(pm.enabledOnAnonymousPages).toBeUndefined();
+    expect(pm.serviceFolders).toBeUndefined();
+    expect(pm).toHaveProperty('kibanaVersion', kibanaVersion);
+    expect(pm).toHaveProperty('optionalPlugins', []);
+    expect(pm).toHaveProperty('requiredBundles', []);
+    expect(pm).toHaveProperty('requiredPlugins', []);
+    expect(pm).toHaveProperty('owner', {
+      name: '@elastic/team-a & @elastic/team-b',
+    });
+    expect(pm).toHaveProperty('server', true);
+    expect(pm).toHaveProperty('ui', true);
+    expect(pm).toHaveProperty('configPath', 'some_legacy_plugin_id');
+  });
+
+  it('reflects plugin.server', () => {
+    expect(
+      pluginManifestFromPluginPackage(kibanaVersion, {
+        ...minimal,
+        plugin: { ...minimal.plugin, server: false },
+      })
+    ).toHaveProperty('server', false);
+    expect(
+      pluginManifestFromPluginPackage(kibanaVersion, {
+        ...minimal,
+        plugin: { ...minimal.plugin, server: true },
+      })
+    ).toHaveProperty('server', true);
+  });
+
+  it('reflects plugin.browser', () => {
+    expect(
+      pluginManifestFromPluginPackage(kibanaVersion, {
+        ...minimal,
+        plugin: { ...minimal.plugin, browser: false },
+      })
+    ).toHaveProperty('ui', false);
+    expect(
+      pluginManifestFromPluginPackage(kibanaVersion, {
+        ...minimal,
+        plugin: { ...minimal.plugin, browser: true },
+      })
+    ).toHaveProperty('ui', true);
+  });
+});

packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_from_plugin_package.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { snakeCase } from 'lodash';
+import { PluginPackageManifest } from '@kbn/repo-packages';
+import { PluginManifest } from '@kbn/core-plugins-server';
+import { PluginType } from '@kbn/core-base-common';
+
+export function pluginManifestFromPluginPackage(
+  kibanaVersion: string,
+  manifest: PluginPackageManifest
+): PluginManifest {
+  return {
+    type: manifest.plugin.type === 'preboot' ? PluginType.preboot : PluginType.standard,
+    id: manifest.plugin.id,
+    version: '1.0.0',
+    enabledOnAnonymousPages: manifest.plugin.enabledOnAnonymousPages,
+    serviceFolders: manifest.serviceFolders,
+    kibanaVersion,
+    optionalPlugins: manifest.plugin.optionalPlugins ?? [],
+    requiredBundles: manifest.plugin.requiredBundles ?? [],
+    requiredPlugins: manifest.plugin.requiredPlugins ?? [],
+    owner: {
+      name: manifest.owner.join(' & '),
+    },
+    server: manifest.plugin.server,
+    ui: manifest.plugin.browser,
+    configPath: manifest.plugin.configPath ?? snakeCase(manifest.plugin.id),
+  };
+}

packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.test.ts

@@ -12,6 +12,7 @@ import { mockPackage, scanPluginSearchPathsMock } from './plugins_discovery.test
 import mockFs from 'mock-fs';
 import { getEnvOptions, rawConfigServiceMock } from '@kbn/config-mocks';
 import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
+import type { Package } from '@kbn/repo-packages';
 
 import { firstValueFrom, from } from 'rxjs';
 import { map, toArray } from 'rxjs/operators';
@@ -25,6 +26,35 @@ import { discover } from './plugins_discovery';
 import { PluginType } from '@kbn/core-base-common';
 
 const KIBANA_ROOT = process.cwd();
+jest.mock('@kbn/repo-packages', () => ({
+  ...jest.requireActual('@kbn/repo-packages'),
+  getPackages: jest.fn().mockReturnValue([]),
+  getPluginPackagesFilter: jest.fn().mockReturnValue(() => true),
+}));
+
+jest.mock('./plugin_manifest_from_plugin_package', () => ({
+  pluginManifestFromPluginPackage: jest.fn((version, pkgManifest) => ({
+    version,
+    ...pkgManifest,
+  })),
+}));
+
+const getPluginPackagesFilterMock: jest.Mock =
+  jest.requireMock('@kbn/repo-packages').getPluginPackagesFilter;
+const pluginManifestFromPluginPackageMock: jest.Mock = jest.requireMock(
+  './plugin_manifest_from_plugin_package'
+).pluginManifestFromPluginPackage;
+
+function getMockPackage(id: string) {
+  return {
+    id,
+    manifest: {
+      id,
+      type: 'plugin',
+    },
+    directory: resolve(REPO_ROOT, `packages/${id}`),
+  } as Package;
+}
 
 const Plugins = {
   invalid: () => ({
@@ -180,6 +210,8 @@ describe('plugins discovery system', () => {
     jest.spyOn(console, 'log').mockImplementation((...args) => {
       process.stdout.write(args + '\n');
     });
+
+    jest.clearAllMocks();
   });
 
   afterEach(() => {
@@ -542,9 +574,61 @@ describe('plugins discovery system', () => {
     expect(loggingSystemMock.collect(logger).warn).toEqual([]);
   });
 
+  describe('plugin packages', () => {
+    it('filters repoPackages in the env and converts them to PluginWrappers', async () => {
+      const foo = getMockPackage('foo');
+      const bar = getMockPackage('bar');
+      coreContext.env = {
+        ...env,
+        pluginSearchPaths: [],
+        repoPackages: [foo, bar],
+      };
+      const filterFn = jest.fn((p: Package) => p === foo);
+      getPluginPackagesFilterMock.mockReturnValue(filterFn);
+
+      const { plugin$ } = discover({
+        config: new PluginsConfig(pluginConfig, coreContext.env),
+        coreContext,
+        instanceInfo,
+        nodeInfo,
+      });
+
+      const [plugin, ...empty] = await firstValueFrom(plugin$.pipe(toArray()));
+      expect(empty).toHaveLength(0);
+
+      expect(getPluginPackagesFilterMock).toHaveBeenCalledTimes(1);
+      const filterArgs = getPluginPackagesFilterMock.mock.calls[0];
+      expect(filterArgs).toEqual([
+        {
+          examples: false,
+          oss: false,
+          parentDirs: [],
+          paths: [],
+        },
+      ]);
+
+      expect(filterFn).toHaveBeenCalledTimes(2);
+      expect(filterFn.mock.calls[0]).toEqual([foo, 0]);
+      expect(filterFn.mock.calls[1]).toEqual([bar, 1]);
+      expect(filterFn.mock.results).toEqual([
+        { type: 'return', value: true },
+        { type: 'return', value: false },
+      ]);
+
+      expect(pluginManifestFromPluginPackageMock).toHaveBeenCalledTimes(1);
+      const manifestArgs = pluginManifestFromPluginPackageMock.mock.calls[0];
+      expect(manifestArgs).toEqual([coreContext.env.packageInfo.version, foo.manifest]);
+      expect(pluginManifestFromPluginPackageMock.mock.results[0]).toEqual({
+        type: 'return',
+        value: plugin.manifest,
+      });
+    });
+  });
+
   describe('discovery order', () => {
     beforeEach(() => {
       scanPluginSearchPathsMock.mockClear();
+      getPluginPackagesFilterMock.mockReturnValue(() => true);
     });
 
     it('returns the plugins in a deterministic order', async () => {
@@ -565,8 +649,13 @@ describe('plugins discovery system', () => {
         ])
       );
 
+      coreContext.env = {
+        ...env,
+        repoPackages: [getMockPackage('foo'), getMockPackage('bar')],
+      };
+
       let { plugin$ } = discover({
-        config: new PluginsConfig(pluginConfig, env),
+        config: new PluginsConfig(pluginConfig, coreContext.env),
         coreContext,
         instanceInfo,
         nodeInfo,
@@ -576,9 +665,8 @@ describe('plugins discovery system', () => {
       let plugins = await firstValueFrom(plugin$.pipe(toArray()));
       let pluginNames = plugins.map((plugin) => plugin.name);
 
-      expect(pluginNames).toHaveLength(3);
-      // order coming from `ROOT/plugin` -> `ROOT/src/plugins` -> // ROOT/x-pack
-      expect(pluginNames).toEqual(['pluginB', 'pluginA', 'pluginC']);
+      // order coming from `ROOT/packages` -> `ROOT/plugin` -> `ROOT/src/plugins` -> // ROOT/x-pack
+      expect(pluginNames).toEqual(['bar', 'foo', 'pluginB', 'pluginA', 'pluginC']);
 
       // second pass
       scanPluginSearchPathsMock.mockReturnValue(
@@ -589,6 +677,11 @@ describe('plugins discovery system', () => {
         ])
       );
 
+      coreContext.env = {
+        ...env,
+        repoPackages: [getMockPackage('bar'), getMockPackage('foo')],
+      };
+
       plugin$ = discover({
         config: new PluginsConfig(pluginConfig, env),
         coreContext,
@@ -600,9 +693,8 @@ describe('plugins discovery system', () => {
       plugins = await firstValueFrom(plugin$.pipe(toArray()));
       pluginNames = plugins.map((plugin) => plugin.name);
 
-      expect(pluginNames).toHaveLength(3);
-      // order coming from `ROOT/plugin` -> `ROOT/src/plugins` -> // ROOT/x-pack
-      expect(pluginNames).toEqual(['pluginB', 'pluginA', 'pluginC']);
+      // order coming from `ROOT/packages` -> `ROOT/plugin` -> `ROOT/src/plugins` -> // ROOT/x-pack
+      expect(pluginNames).toEqual(['bar', 'foo', 'pluginB', 'pluginA', 'pluginC']);
     });
   });
 });

packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.ts

@@ -6,12 +6,14 @@
  * Side Public License, v 1.
  */
 
-import { from, merge } from 'rxjs';
+import { from, merge, EMPTY } from 'rxjs';
 import { catchError, filter, map, mergeMap, concatMap, shareReplay, toArray } from 'rxjs/operators';
 import { Logger } from '@kbn/logging';
+import { getPluginPackagesFilter } from '@kbn/repo-packages';
 import type { CoreContext } from '@kbn/core-base-server-internal';
 import type { NodeInfo } from '@kbn/core-node-server';
 import { PluginWrapper } from '../plugin';
+import { pluginManifestFromPluginPackage } from './plugin_manifest_from_plugin_package';
 import { createPluginInitializerContext, InstanceInfo } from '../plugin_context';
 import { PluginsConfig } from '../plugins_config';
 import { PluginDiscoveryError } from './plugin_discovery_error';
@@ -48,23 +50,59 @@ export function discover({
     );
   }
 
-  const discoveryResults$ = merge(
+  const fsDiscovery$ = merge(
     from(config.additionalPluginPaths),
     scanPluginSearchPaths(config.pluginSearchPaths, log)
   ).pipe(
-    toArray(),
-    mergeMap((pathAndErrors) => {
-      return pathAndErrors.sort((a, b) => {
-        const pa = typeof a === 'string' ? a : a.path;
-        const pb = typeof b === 'string' ? b : b.path;
-        return pa < pb ? -1 : pa > pb ? 1 : 0;
-      });
-    }),
     concatMap((pluginPathOrError) => {
       return typeof pluginPathOrError === 'string'
         ? createPlugin$(pluginPathOrError, log, coreContext, instanceInfo, nodeInfo)
         : [pluginPathOrError];
-    }),
+    })
+  );
+
+  const pluginPkgDiscovery$ = from(coreContext.env.repoPackages ?? EMPTY).pipe(
+    filter(
+      getPluginPackagesFilter({
+        oss: coreContext.env.cliArgs.oss,
+        examples: coreContext.env.cliArgs.runExamples,
+        paths: config.additionalPluginPaths,
+        parentDirs: config.pluginSearchPaths,
+      })
+    ),
+    map((pkg) => {
+      log.debug(`Successfully discovered plugin package "${pkg.id}"`);
+      const manifest = pluginManifestFromPluginPackage(
+        coreContext.env.packageInfo.version,
+        pkg.manifest
+      );
+      const initializerContext = createPluginInitializerContext({
+        coreContext,
+        opaqueId: Symbol(pkg.id),
+        manifest,
+        instanceInfo,
+        nodeInfo,
+      });
+
+      return new PluginWrapper({
+        path: pkg.directory,
+        manifest,
+        opaqueId: initializerContext.opaqueId,
+        initializerContext,
+      });
+    })
+  );
+
+  const discoveryResults$ = merge(fsDiscovery$, pluginPkgDiscovery$).pipe(
+    toArray(),
+    // ensure that everything is always provided in a consistent order
+    mergeMap((pkgs) =>
+      pkgs.sort((a, b) => {
+        const aComp = typeof a !== 'string' ? a.path : a;
+        const bComp = typeof b !== 'string' ? b.path : b;
+        return aComp.localeCompare(bComp);
+      })
+    ),
     shareReplay()
   );
 

packages/core/plugins/core-plugins-server-internal/tsconfig.json

@@ -37,6 +37,7 @@
     "@kbn/core-environment-server-internal",
     "@kbn/core-node-server-internal",
     "@kbn/core-plugins-base-server-internal",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/core/root/core-root-server-internal/src/bootstrap.ts

@@ -7,6 +7,7 @@
  */
 
 import chalk from 'chalk';
+import { getPackages } from '@kbn/repo-packages';
 import { CliArgs, Env, RawConfigService } from '@kbn/config';
 import { CriticalError } from '@kbn/core-base-server-internal';
 import { Root } from './root';
@@ -39,6 +40,7 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot
   const env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
+    repoPackages: getPackages(REPO_ROOT),
   });
 
   const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);

packages/core/root/core-root-server-internal/tsconfig.json

@@ -67,6 +67,7 @@
     "@kbn/apm-config-loader",
     "@kbn/core-custom-branding-server-internal",
     "@kbn/core-custom-branding-server-mocks",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/core/test-helpers/core-test-helpers-kbn-server/src/create_root.ts

@@ -12,6 +12,7 @@ import { defaultsDeep } from 'lodash';
 import { BehaviorSubject } from 'rxjs';
 import supertest from 'supertest';
 
+import { getPackages } from '@kbn/repo-packages';
 import { ToolingLog } from '@kbn/tooling-log';
 import { REPO_ROOT } from '@kbn/repo-info';
 import {
@@ -71,6 +72,7 @@ export function createRootWithSettings(
         dist: false,
         ...cliArgs,
       },
+      repoPackages: getPackages(REPO_ROOT),
     },
     pkg
   );

packages/core/test-helpers/core-test-helpers-kbn-server/tsconfig.json

@@ -17,6 +17,7 @@
     "@kbn/core-lifecycle-server-internal",
     "@kbn/core-root-server-internal",
     "@kbn/repo-info",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/kbn-cli-dev-mode/src/bootstrap.ts

@@ -7,6 +7,7 @@
  */
 
 import { REPO_ROOT } from '@kbn/repo-info';
+import { getPackages } from '@kbn/repo-packages';
 import { CliArgs, Env, RawConfigAdapter } from '@kbn/config';
 import { CliDevMode } from './cli_dev_mode';
 import { CliLog } from './log';
@@ -25,6 +26,7 @@ export async function bootstrapDevMode({ configs, cliArgs, applyConfigOverrides
   const env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
+    repoPackages: getPackages(REPO_ROOT),
   });
 
   const config = await loadConfig({

packages/kbn-cli-dev-mode/tsconfig.json

@@ -24,6 +24,7 @@
     "@kbn/repo-source-classifier",
     "@kbn/import-resolver",
     "@kbn/picomatcher",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/kbn-config-mocks/src/env.mock.ts

@@ -7,6 +7,7 @@
  */
 
 import { REPO_ROOT } from '@kbn/repo-info';
+import { getPackages } from '@kbn/repo-packages';
 import { Env, type RawPackageInfo, type EnvOptions } from '@kbn/config';
 
 type DeepPartial<T> = {
@@ -28,6 +29,7 @@ export function getEnvOptions(options: DeepPartial<EnvOptions> = {}): EnvOptions
       runExamples: false,
       ...(options.cliArgs || {}),
     },
+    repoPackages: getPackages(REPO_ROOT),
   };
 }
 

packages/kbn-config-mocks/tsconfig.json

@@ -15,6 +15,7 @@
     "@kbn/utility-types",
     "@kbn/doc-links",
     "@kbn/repo-info",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/kbn-config/src/__snapshots__/env.test.ts.snap

@@ -39,6 +39,7 @@ Env {
     "/test/kibanaRoot/plugins",
     "/test/kibanaRoot/../kibana-extra",
   ],
+  "repoPackages": undefined,
 }
 `;
 
@@ -81,6 +82,7 @@ Env {
     "/test/kibanaRoot/plugins",
     "/test/kibanaRoot/../kibana-extra",
   ],
+  "repoPackages": undefined,
 }
 `;
 
@@ -122,6 +124,7 @@ Env {
     "/test/kibanaRoot/plugins",
     "/test/kibanaRoot/../kibana-extra",
   ],
+  "repoPackages": undefined,
 }
 `;
 
@@ -163,6 +166,7 @@ Env {
     "/test/kibanaRoot/plugins",
     "/test/kibanaRoot/../kibana-extra",
   ],
+  "repoPackages": undefined,
 }
 `;
 
@@ -204,6 +208,7 @@ Env {
     "/test/kibanaRoot/plugins",
     "/test/kibanaRoot/../kibana-extra",
   ],
+  "repoPackages": undefined,
 }
 `;
 
@@ -245,5 +250,9 @@ Env {
     "/some/home/dir/plugins",
     "/some/home/dir/../kibana-extra",
   ],
+  "repoPackages": Array [
+    "FakePackage1",
+    "FakePackage2",
+  ],
 }
 `;

packages/kbn-config/src/env.test.ts

@@ -7,6 +7,7 @@
  */
 
 import { mockPackage } from './env.test.mocks';
+import type { Package } from '@kbn/repo-packages';
 
 import { Env, RawPackageInfo } from './env';
 import { getEnvOptions } from './internal_mocks';
@@ -132,6 +133,7 @@ test('correctly creates environment with constructor.', () => {
     getEnvOptions({
       cliArgs: { dev: false },
       configs: ['/some/other/path/some-kibana.yml'],
+      repoPackages: ['FakePackage1', 'FakePackage2'] as unknown as Package[],
     })
   );
 

packages/kbn-config/src/env.ts

@@ -9,12 +9,14 @@
 import { resolve, join } from 'path';
 import loadJsonFile from 'load-json-file';
 import { getPluginSearchPaths } from '@kbn/plugin-discovery';
+import type { Package } from '@kbn/repo-packages';
 import { PackageInfo, EnvironmentMode } from './types';
 
 /** @internal */
 export interface EnvOptions {
   configs: string[];
   cliArgs: CliArgs;
+  repoPackages?: readonly Package[];
 }
 
 /** @internal */
@@ -64,6 +66,8 @@ export class Env {
   public readonly logDir: string;
   /** @internal */
   public readonly pluginSearchPaths: readonly string[];
+  /** @internal */
+  public readonly repoPackages?: readonly Package[];
 
   /**
    * Information about Kibana package (version, build number etc.).
@@ -100,6 +104,7 @@ export class Env {
       oss: options.cliArgs.oss,
       examples: options.cliArgs.runExamples,
     });
+    this.repoPackages = options.repoPackages;
 
     this.cliArgs = Object.freeze(options.cliArgs);
     this.configs = Object.freeze(options.configs);

packages/kbn-config/src/internal_mocks.ts

@@ -14,7 +14,11 @@ import type { RawConfigService } from './raw';
 import type { ConfigDeprecationContext } from './deprecation';
 
 type DeepPartial<T> = {
-  [P in keyof T]?: T[P] extends Array<infer R> ? Array<DeepPartial<R>> : DeepPartial<T[P]>;
+  [P in keyof T]?: P extends 'repoPackages'
+    ? T[P]
+    : T[P] extends Array<infer R>
+    ? Array<DeepPartial<R>>
+    : DeepPartial<T[P]>;
 };
 
 export function getEnvOptions(options: DeepPartial<EnvOptions> = {}): EnvOptions {
@@ -32,6 +36,7 @@ export function getEnvOptions(options: DeepPartial<EnvOptions> = {}): EnvOptions
       runExamples: false,
       ...(options.cliArgs || {}),
     },
+    repoPackages: options.repoPackages,
   };
 }
 

packages/kbn-config/tsconfig.json

@@ -19,7 +19,8 @@
     "@kbn/utility-types",
     "@kbn/i18n",
     "@kbn/plugin-discovery",
-    "@kbn/doc-links"
+    "@kbn/doc-links",
+    "@kbn/repo-packages"
   ],
   "exclude": [
     "target/**/*",

packages/kbn-health-gateway-server/src/config/config_service.ts

@@ -8,6 +8,7 @@
 
 import { fromRoot, REPO_ROOT } from '@kbn/repo-info';
 import type { LoggerFactory } from '@kbn/logging';
+import { getPackages } from '@kbn/repo-packages';
 import { ConfigService as KbnConfigService, CliArgs, Env, RawConfigService } from '@kbn/config';
 import { getArgValues } from './read_argv';
 
@@ -38,6 +39,7 @@ export function getConfigService({ logger }: { logger: LoggerFactory }) {
   const env = Env.createDefault(REPO_ROOT, {
     configs: configPath,
     cliArgs: KIBANA_CLI_ARGS,
+    repoPackages: getPackages(REPO_ROOT),
   });
 
   return new KbnConfigService(rawConfigService, env, logger);

packages/kbn-health-gateway-server/tsconfig.json

@@ -22,6 +22,7 @@
     "@kbn/server-http-tools",
     "@kbn/utility-types",
     "@kbn/repo-info",
+    "@kbn/repo-packages",
   ],
   "exclude": [
     "target/**/*",

packages/kbn-kibana-manifest-schema/src/kibana_json_v2_schema.ts

@@ -55,17 +55,59 @@ export const MANIFEST_V2: JSONSchema = {
       `,
       default: false,
     },
+    build: {
+      type: 'object',
+      properties: {
+        extraExcludes: {
+          type: 'array',
+          description: desc`
+            An array of micromatch patterns which will be used to exclude
+            files/directories in this package from the build.
+          `,
+          items: {
+            type: 'string',
+          },
+        },
+        noParse: {
+          type: 'array',
+          description: desc`
+            An array of micromatch patterns which will be used to exclude
+            files from being transformed automatically. Use this to skip large
+            assets which are already transpiled and do not need babel.
+          `,
+          items: {
+            type: 'string',
+          },
+        },
+      },
+    },
+    serviceFolders: {
+      description: desc`
+        Creates sections in the documentations based on the exports of the folders listed here.
+        If you need this you should probably split up your package, which is why this is deprecated.
+      `,
+      type: 'array',
+      items: { type: 'string' },
+      deprecated: true,
+    },
+    description: {
+      description: desc`
+        A brief description of what this package does and any capabilities it provides.
+      `,
+      type: 'string',
+    },
   },
   oneOf: [
     {
       type: 'object',
+      required: ['type', 'plugin'],
       properties: {
         type: {
-          enum: ['plugin-browser', 'plugin-server'],
+          const: 'plugin',
         },
         plugin: {
           type: 'object',
-          required: ['id'],
+          required: ['id', 'browser', 'server'],
           properties: {
             id: {
               type: 'string',
@@ -74,11 +116,13 @@ export const MANIFEST_V2: JSONSchema = {
             configPath: {
               description:
                 'Root configuration path used by the plugin, defaults to "id" in snake_case format.',
-              type: 'array',
-              items: {
-                type: 'string',
-                pattern: PLUGIN_ID_PATTERN.source,
-              },
+              oneOf: [
+                {
+                  type: 'array',
+                  items: { type: 'string' },
+                },
+                { type: 'string' },
+              ],
             },
             requiredPlugins: {
               type: 'array',
@@ -94,12 +138,6 @@ export const MANIFEST_V2: JSONSchema = {
                 pattern: PLUGIN_ID_PATTERN.source,
               },
             },
-            description: {
-              description: desc`
-                A brief description of what this plugin does and any capabilities it provides.
-              `,
-              type: 'string',
-            },
             enabledOnAnonymousPages: {
               description: desc`
                 Specifies whether this plugin - and its required dependencies - will be enabled for anonymous pages (login page, status page when
@@ -107,13 +145,25 @@ export const MANIFEST_V2: JSONSchema = {
               `,
               type: 'boolean',
             },
-            serviceFolders: {
+            type: {
               description: desc`
-                Only used for the automatically generated API documentation. Specifying service
-                folders will cause your plugin API reference to be broken up into sub sections.
+                Only used to distinguish "preboot" plugins from standard plugins.
+              `,
+              enum: ['preboot'],
+            },
+            browser: {
+              type: 'boolean',
+              description: desc`
+                Set this to true when your plugin has a browser-side component, causing the "public" directory
+                to be imported in a webpack bundle and the browser plugin to be started by core.
+              `,
+            },
+            server: {
+              type: 'boolean',
+              description: desc`
+                Set this to true when your plugin has a server-side component, causing the "server" directory
+                to be imported by the server and the plugin started by core.
               `,
-              type: 'array',
-              items: { type: 'string' },
             },
           },
         },

packages/kbn-repo-packages/index.js

@@ -13,15 +13,11 @@
 /** @typedef {import('./modern/types').KibanaPackageType} KibanaPackageType */
 /** @typedef {import('./modern/types').ParsedPackageJson} ParsedPackageJson */
 /** @typedef {import('./modern/types').KbnImportReq} KbnImportReq */
-/** @typedef {import('./modern/types').PluginTypeInfo} PluginTypeInfo */
+/** @typedef {import('./modern/types').PluginCategoryInfo} PluginCategoryInfo */
 /** @typedef {Map<string, string>} PackageMap */
 
 const { getPackages, findPackageInfoForPath, getPkgMap } = require('./modern/get_packages');
-const {
-  parsePackageManifest,
-  readPackageManifest,
-  validatePackageManifest,
-} = require('./modern/parse_package_manifest');
+const { readPackageManifest } = require('./modern/parse_package_manifest');
 const { Package } = require('./modern/package');
 const { parseKbnImportReq } = require('./modern/parse_kbn_import_req');
 const Jsonc = require('./utils/jsonc');
@@ -39,9 +35,7 @@ module.exports = {
   getPackages,
   findPackageInfoForPath,
   getPkgMap,
-  parsePackageManifest,
   readPackageManifest,
-  validatePackageManifest,
   Jsonc,
   getDistributablePacakgesFilter,
   getPluginPackagesFilter,

packages/kbn-repo-packages/modern/get_packages.js

@@ -69,7 +69,9 @@ function getPkgDirMap() {
 }
 
 /**
- * Find the package which contains this path, if one exists
+ * Find the package which contains this path, if one exists, and return
+ * basic info about that package.
+ *
  * @param {string} repoRoot
  * @param {string} path absolute path to a file
  */

packages/kbn-repo-packages/modern/package.js

@@ -115,13 +115,6 @@ class Package {
      */
     this.id = manifest.id;
 
-    /**
-     * Does this package expose a plugin, is it of one of the plugin types?
-     * @type {boolean}
-     * @readonly
-     */
-    this.isPlugin = manifest.type === 'plugin-browser' || manifest.type === 'plugin-server';
-
     /**
      * Is this package highlighted as a "dev only" package? If so it will always
      * be listed in the devDependencies and will never end up in the build
@@ -132,11 +125,19 @@ class Package {
   }
 
   /**
-   * Returns true if the package represents some type of plugin
-   * @returns {import('./types').PluginTypeInfo}
+   * Does this package expose a plugin, is it of one of the plugin types?
+   * @returns {this is import('./types').PluginPackageManifest}
    */
-  getPlguinType() {
-    if (!this.isPlugin) {
+  isPlugin() {
+    return this.manifest.type === 'plugin';
+  }
+
+  /**
+   * Returns true if the package represents some type of plugin
+   * @returns {import('./types').PluginCategoryInfo}
+   */
+  getPluginCategories() {
+    if (!this.isPlugin()) {
       throw new Error('package is not a plugin, check pkg.isPlugin before calling this method');
     }
 
@@ -156,7 +157,7 @@ class Package {
    * print all the BUILD.bazel files
    */
   [inspect.custom]() {
-    return `${this.isPlugin ? `PluginPackage` : `Package`}<${this.normalizedRepoRelativeDir}>`;
+    return `${this.isPlugin() ? `PluginPackage` : `Package`}<${this.normalizedRepoRelativeDir}>`;
   }
 }
 

packages/kbn-repo-packages/modern/parse_helpers.js

@@ -9,8 +9,7 @@
 /** @type {{ [k in import('./types').KibanaPackageType]: true }} */
 const PACKAGE_TYPE_MAP = {
   'functional-tests': true,
-  'plugin-browser': true,
-  'plugin-server': true,
+  plugin: true,
   'shared-browser': true,
   'shared-common': true,
   'shared-scss': true,

packages/kbn-repo-packages/modern/parse_package_manifest.js

@@ -41,35 +41,38 @@ const isValidOwner = (v) => typeof v === 'string' && v.startsWith('@');
 
 /**
  * @param {unknown} plugin
- * @returns {import('./types').PluginPackageManifest['plugin']}
+ * @returns {import('./types').PluginPackageManifest['plugin']} plugin
  */
 function validatePackageManifestPlugin(plugin) {
   if (!isObj(plugin)) {
-    throw err(`plugin`, plugin, `must be an object`);
+    throw err('plugin', plugin, 'must be an object');
   }
 
   const {
     id,
+    browser,
+    server,
+    extraPublicDirs,
     configPath,
     requiredPlugins,
     optionalPlugins,
     requiredBundles,
-    description,
     enabledOnAnonymousPages,
-    serviceFolders,
     type,
-    ...extra
   } = plugin;
 
-  const extraKeys = Object.keys(extra);
-  if (extraKeys.length) {
-    throw new Error(`unexpected keys in "plugin" of package [${extraKeys.join(', ')}]`);
-  }
-
   if (!isValidPluginId(id)) {
     throw err(`plugin.id`, id, `must be a string in camel or snake case`);
   }
-
+  if (typeof browser !== 'boolean') {
+    throw err('plugin.browser', browser, 'must be a boolean');
+  }
+  if (typeof server !== 'boolean') {
+    throw err('plugin.server', server, 'must be a boolean');
+  }
+  if (extraPublicDirs !== undefined && !isArrOfStrings(extraPublicDirs)) {
+    throw err(`plugin.extraPublicDirs`, extraPublicDirs, `must be an array of strings`);
+  }
   if (configPath !== undefined && !(isSomeString(configPath) || isArrOfStrings(configPath))) {
     throw err(
       `plugin.configPath`,
@@ -102,32 +105,25 @@ function validatePackageManifestPlugin(plugin) {
     );
   }
 
-  if (description !== undefined && !isSomeString(description)) {
-    throw err(`plugin.description`, description, `must be a non-empty string when specified`);
-  }
-
   if (enabledOnAnonymousPages !== undefined && typeof enabledOnAnonymousPages !== 'boolean') {
     throw err(`plugin.enabledOnAnonymousPages`, enabledOnAnonymousPages, `must be a boolean`);
   }
 
-  if (serviceFolders !== undefined && !isArrOfStrings(serviceFolders)) {
-    throw err(`plugin.serviceFolders`, serviceFolders, `must be an array of non-empty strings`);
-  }
-
   if (type !== undefined && type !== 'preboot') {
     throw err(`plugin.type`, type, `must be undefined or "preboot"`);
   }
 
   return {
     id,
+    browser,
+    server,
     type,
     configPath,
     requiredPlugins,
     optionalPlugins,
     requiredBundles,
-    description,
     enabledOnAnonymousPages,
-    serviceFolders,
+    extraPublicDirs,
   };
 }
 
@@ -179,7 +175,18 @@ function validatePackageManifest(parsed) {
     throw new Error('expected manifest root to be an object');
   }
 
-  const { type, id, owner, devOnly, plugin, sharedBrowserBundle, build, ...extra } = parsed;
+  const {
+    type,
+    id,
+    owner,
+    devOnly,
+    plugin,
+    sharedBrowserBundle,
+    build,
+    description,
+    serviceFolders,
+    ...extra
+  } = parsed;
 
   const extraKeys = Object.keys(extra);
   if (extraKeys.length) {
@@ -209,11 +216,21 @@ function validatePackageManifest(parsed) {
     throw err(`devOnly`, devOnly, `must be a boolean when defined`);
   }
 
+  if (description !== undefined && !isSomeString(description)) {
+    throw err(`description`, description, `must be a non-empty string when specified`);
+  }
+
+  if (serviceFolders !== undefined && !isArrOfStrings(serviceFolders)) {
+    throw err(`serviceFolders`, serviceFolders, `must be an array of non-empty strings`);
+  }
+
   const base = {
     id,
     owner: Array.isArray(owner) ? owner : [owner],
     devOnly,
     build: validatePackageManifestBuild(build),
+    description,
+    serviceFolders,
   };
 
   // return if this is one of the more basic types of package types
@@ -224,8 +241,7 @@ function validatePackageManifest(parsed) {
     };
   }
 
-  // handle the plugin field for plugin-* types
-  if (type === 'plugin-browser' || type === 'plugin-server') {
+  if (type === 'plugin') {
     return {
       type,
       ...base,
@@ -254,32 +270,25 @@ function readPackageManifest(path) {
     content = Fs.readFileSync(path, 'utf8');
   } catch (error) {
     if (error.code === 'ENOENT') {
-      throw new Error(`Missing kibana.jsonc file at ${path}`);
+      const err = new Error(`Missing kibana.jsonc file at ${path}`);
+      throw Object.assign(err, { code: 'ENOENT' });
     }
 
     throw error;
   }
 
   try {
-    return parsePackageManifest(content);
+    let parsed;
+    try {
+      parsed = parse(content);
+    } catch (error) {
+      throw new Error(`Invalid JSONc: ${error.message}`);
+    }
+
+    return validatePackageManifest(parsed);
   } catch (error) {
     throw new Error(`Unable to parse [${path}]: ${error.message}`);
   }
 }
 
-/**
- * Parse a kibana.jsonc file from a string
- * @param {string} content
- */
-function parsePackageManifest(content) {
-  let parsed;
-  try {
-    parsed = parse(content);
-  } catch (error) {
-    throw new Error(`Invalid JSONc: ${error.message}`);
-  }
-
-  return validatePackageManifest(parsed);
-}
-
-module.exports = { parsePackageManifest, readPackageManifest, validatePackageManifest };
+module.exports = { readPackageManifest };

packages/kbn-repo-packages/modern/plugins.js

@@ -18,7 +18,7 @@ function getPluginSearchPaths({ rootDir }) {
 
 /**
  * @param {import('./types').PluginSelector} selector
- * @param {import('./types').PluginTypeInfo} type
+ * @param {import('./types').PluginCategoryInfo} type
  */
 function matchType(selector, type) {
   if (!type.oss && selector.oss) {
@@ -79,9 +79,9 @@ function getPluginPackagesFilter(selector = {}) {
    * @returns {pkg is import('./types').PluginPackage}
    */
   return (pkg) =>
-    pkg.isPlugin &&
+    pkg.isPlugin() &&
     matchParentDirsLimit(selector, pkg.directory) &&
-    (matchType(selector, pkg.getPlguinType()) ||
+    (matchType(selector, pkg.getPluginCategories()) ||
       matchPluginPaths(selector, pkg.directory) ||
       matchPluginParentDirs(selector, pkg.directory));
 }
@@ -99,11 +99,11 @@ function getDistributablePacakgesFilter() {
       return false;
     }
 
-    if (!pkg.isPlugin) {
+    if (!pkg.isPlugin()) {
       return true;
     }
 
-    const type = pkg.getPlguinType();
+    const type = pkg.getPluginCategories();
     return !(type.example || type.testPlugin);
   };
 }

packages/kbn-repo-packages/modern/types.ts

@@ -33,8 +33,7 @@ export interface ParsedPackageJson {
 }
 
 export type KibanaPackageType =
-  | 'plugin-browser'
-  | 'plugin-server'
+  | 'plugin'
   | 'shared-browser'
   | 'shared-server'
   | 'shared-common'
@@ -79,23 +78,34 @@ interface PackageManifestBaseFields {
      */
     noParse?: string[];
   };
+  /**
+   * A breif description of the package and what it provides
+   */
+  description?: string;
+  /**
+   * Creates sections in the documentations based on the exports of the folders listed here.
+   * If you need this you should probably split up your package, which is why this is deprecated.
+   * @deprecated
+   */
+  serviceFolders?: string[];
 }
 
 export interface PluginPackageManifest extends PackageManifestBaseFields {
-  type: 'plugin-browser' | 'plugin-server';
+  type: 'plugin';
   /**
    * Details about the plugin which is contained within this package.
    */
   plugin: {
     id: string;
+    browser: boolean;
+    server: boolean;
     configPath?: string | string[];
     requiredPlugins?: string[];
     optionalPlugins?: string[];
     requiredBundles?: string[];
-    description?: string;
     enabledOnAnonymousPages?: boolean;
-    serviceFolders?: string[];
     type?: 'preboot';
+    extraPublicDirs?: string[];
   };
 }
 
@@ -149,6 +159,14 @@ export interface PluginSelector {
    * Absolute paths to parent directories of plugin packages which will always be included, regardless of the other settings
    */
   limitParentDirs?: readonly string[];
+  /**
+   * When set to true, only select plugins which have server-side components
+   */
+  server?: boolean;
+  /**
+   * When set to true, only select plugins which have browser-side components
+   */
+  browser?: boolean;
 }
 
 export interface KbnImportReq {
@@ -166,7 +184,7 @@ export interface KbnImportReq {
   full: string;
 }
 
-export interface PluginTypeInfo {
+export interface PluginCategoryInfo {
   /** is this an oss plugin? */
   oss: boolean;
   /** is this an example plugin? */

packages/kbn-repo-source-classifier/src/repo_source_classifier.ts

@@ -177,16 +177,17 @@ export class RepoSourceClassifier {
         case 'functional-tests':
         case 'test-helper':
           return 'tests or mocks';
-        case 'plugin-browser':
         case 'shared-browser':
           return 'browser package';
-        case 'plugin-server':
         case 'shared-server':
           return 'server package';
         case 'shared-scss':
           return 'static';
         case 'shared-common':
           return 'common package';
+        case 'plugin':
+          // classification in plugins is more complicated, fall through to remaining logic
+          break;
         default:
           // @ts-expect-error if there isn't an error here we are missing a case for a package type
           throw new Error(`unexpected package type [${manifest.type}]`);