fix(NA): external plugins development flow with the new modern package plugins in place (#153562)

This PR fixes the 3rd party external plugin development workflow by
introducing a dev step for plugins that allows for development usages of
those with Kibana.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

docs/developer/plugin/plugin-tooling.asciidoc

@@ -43,8 +43,10 @@ It will output a`zip` archive in `kibana/plugins/my_plugin_name/build/` folder.
 See <<install-plugin, How to install a plugin>>.
 
 === Run {kib} with your plugin in dev mode
-Run `yarn start` in the {kib} root folder. Make sure {kib} found and bootstrapped your plugin:
+If your plugin isn't server only and contains `ui` in order for Kibana to pick the browser bundles you need to run `yarn dev --watch` in the plugin root folder at a dedicated terminal.
+
+Then, in a second terminal, run `yarn start` at the {kib} root folder. Make sure {kib} found and bootstrapped your plugin by:
 ["source","shell"]
 -----------
-[info][plugins-system] Setting up […] plugins: […, myPluginName, …]
+[INFO ][plugins-system.standard] Setting up […] plugins: […, myPluginName, …]
 -----------

packages/kbn-optimizer/src/common/bundle.ts

@@ -21,8 +21,8 @@ const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
 const DEFAULT_IMPLICIT_BUNDLE_DEPS = ['core'];
 
-const toStringArray = (input: any): string[] =>
-  Array.isArray(input) && input.every((x) => typeof x === 'string') ? input : [];
+const toStringArray = (input: any): string[] | null =>
+  Array.isArray(input) && input.every((x) => typeof x === 'string') ? input : null;
 
 export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
@@ -164,19 +164,39 @@ export class Bundle {
       );
     }
 
-    if (isObj(parsed) && isObj(parsed.plugin)) {
-      return {
-        explicit: [...toStringArray(parsed.plugin.requiredBundles)],
-        implicit: [
-          ...DEFAULT_IMPLICIT_BUNDLE_DEPS,
-          ...toStringArray(parsed.plugin.requiredPlugins),
-        ],
-      };
+    // TODO: remove once we improve the third party plugin build workflow
+    // This is only used to build legacy third party plugins in the @kbn/plugin-helpers
+
+    if (!isObj(parsed)) {
+      throw new Error(`Expected [${this.manifestPath}] to be a jsonc parseable file`);
     }
 
-    throw new Error(
-      `Expected "requiredBundles" and "requiredPlugins" in manifest file [${this.manifestPath}] to be arrays of strings`
-    );
+    const requiredBundles = isObj(parsed.plugin)
+      ? parsed.plugin.requiredBundles
+      : parsed.requiredBundles;
+    const requiredPlugins = isObj(parsed.plugin)
+      ? parsed.plugin.requiredPlugins
+      : parsed.requiredPlugins;
+    const requiredBundlesStringArray = toStringArray(requiredBundles);
+    const requiredPluginsStringArray = toStringArray(requiredPlugins);
+    // END-OF-TD: we just need to check for parse.plugin and not for legacy plugins manifest types
+
+    if (!requiredBundlesStringArray && requiredBundles) {
+      throw new Error(
+        `Expected "requiredBundles" in manifest file [${this.manifestPath}] to be an array of strings`
+      );
+    }
+
+    if (!requiredPluginsStringArray && requiredPlugins) {
+      throw new Error(
+        `Expected "requiredPlugins" in manifest file [${this.manifestPath}] to be an array of strings`
+      );
+    }
+
+    return {
+      explicit: [...(requiredBundlesStringArray || [])],
+      implicit: [...DEFAULT_IMPLICIT_BUNDLE_DEPS, ...(requiredPluginsStringArray || [])],
+    };
   }
 }
 

packages/kbn-plugin-discovery/tsconfig.json

@@ -1,18 +0,0 @@
-{
-  "extends": "../../tsconfig.base.json",
-  "compilerOptions": {
-    "checkJs": true,
-    "outDir": "target/types",
-    "types": [
-      "jest",
-      "node"
-    ]
-  },
-  "include": [
-    "**/*.js",
-    "**/*.ts",
-  ],
-  "exclude": [
-    "target/**/*",
-  ]
-}

packages/kbn-plugin-generator/README.md

@@ -51,7 +51,7 @@ yarn kbn bootstrap
 
 Generated plugins receive a handful of scripts that can be used during development. Those scripts are detailed in the [README.md](template/README.md) file in each newly generated plugin, and expose the scripts provided by the [Kibana plugin helpers](../kbn-plugin-helpers), but here is a quick reference in case you need it:
 
-> ***NOTE:*** The following scripts should be run from the generated plugin.
+> ***NOTE:*** The following scripts should be run from the generated plugin root folder.
 
   - `yarn kbn bootstrap`
 
@@ -63,12 +63,16 @@ Generated plugins receive a handful of scripts that can be used during developme
 
     Build a distributable archive of your plugin.
 
+  - `yarn dev --watch`
+
+    Builds and starts the watch mode of your ui browser side plugin so it can be picked up by Kibana in development.
+
 
 To start kibana run the following command from Kibana root.
 
   - `yarn start`
 
-    Start kibana and it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
+    Start kibana and, if you had previously run in another terminal `yarn dev --watch` at the root of your plugin, it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
 
       ```
       yarn start --elasticsearch.hosts http://localhost:9220

packages/kbn-plugin-generator/template/.eslintrc.js.ejs

@@ -1,3 +1,5 @@
+require('@kbn/babel-register').install();
+
 module.exports = {
   root: true,
   extends: [

packages/kbn-plugin-generator/template/README.md.ejs

@@ -16,5 +16,8 @@ See the [kibana contributing guide](https://github.com/elastic/kibana/blob/main/
 
   <dt><code>yarn plugin-helpers build</code></dt>
   <dd>Execute this to create a distributable version of this plugin that can be installed in Kibana</dd>
+
+  <dt><code>yarn plugin-helpers dev --watch</code></dt>
+    <dd>Execute this to build your plugin ui browser side so Kibana could pick up when started in development</dd>
 </dl>
 <% } %>

packages/kbn-plugin-generator/template/package.json.ejs

@@ -4,6 +4,7 @@
   "private": true,
   "scripts": {
     "build": "yarn plugin-helpers build",
+    "dev": "yarn plugin-helpers dev",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "kbn": "node ../../scripts/kbn"
   }

packages/kbn-plugin-helpers/README.md

@@ -11,6 +11,7 @@ is already the case if you use the new `node scripts/generate_plugin` script.
 {
   "scripts" : {
     "build": "yarn plugin-helpers build",
+    "dev": "yarn plugin-helpers dev",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "kbn": "node ../../scripts/kbn"
   }
@@ -48,7 +49,15 @@ $ plugin-helpers help
   
         Options:
           --skip-archive        Don't create the zip file, just create the build/kibana directory
-          --kibana-version, -v  Kibana version that the
+          --kibana-version, -v  Kibana version this plugin will be built for
+          
+      dev
+        Builds the current plugin ui browser side so it can be picked up by Kibana
+        during development.
+        
+        Options:
+          --dist, -d  Outputs bundles in dist mode instead
+          --watch, -w  Starts the watch mode
     
 
   Global options:
@@ -92,7 +101,7 @@ Plugin code can be written in [TypeScript](http://www.typescriptlang.org/) if de
 ```js
 {
   // extend Kibana's tsconfig, or use your own settings
-  "extends": "../../kibana/tsconfig.json",
+  "extends": "../../tsconfig.json",
 
   // tell the TypeScript compiler where to find your source files
   "include": [

packages/kbn-plugin-helpers/src/cli.ts

@@ -14,7 +14,7 @@ import { createFlagError, createFailError } from '@kbn/dev-cli-errors';
 import { findPluginDir } from './find_plugin_dir';
 import { loadKibanaPlatformPlugin } from './load_kibana_platform_plugin';
 import * as Tasks from './tasks';
-import { BuildContext } from './build_context';
+import { TaskContext } from './task_context';
 import { resolveKibanaVersion } from './resolve_kibana_version';
 import { loadConfig } from './config';
 
@@ -42,7 +42,7 @@ export function runCli() {
         },
         help: `
           --skip-archive        Don't create the zip file, just create the build/kibana directory
-          --kibana-version, -v  Kibana version that the
+          --kibana-version, -v  Kibana version this plugin will be built for
         `,
       },
       async run({ log, flags }) {
@@ -56,7 +56,7 @@ export function runCli() {
           throw createFlagError('expected a single --skip-archive flag');
         }
 
-        const found = await findPluginDir();
+        const found = findPluginDir();
         if (!found) {
           throw createFailError(
             `Unable to find Kibana Platform plugin in [${process.cwd()}] or any of its parent directories. Has it been migrated properly? Does it have a kibana.json file?`
@@ -73,8 +73,10 @@ export function runCli() {
         const sourceDir = plugin.directory;
         const buildDir = Path.resolve(plugin.directory, 'build/kibana', plugin.manifest.id);
 
-        const context: BuildContext = {
+        const context: TaskContext = {
           log,
+          dev: false,
+          dist: true,
           plugin,
           config,
           sourceDir,
@@ -93,5 +95,73 @@ export function runCli() {
         }
       },
     })
+    .command({
+      name: 'dev',
+      description: `
+        Builds the current plugin ui browser side so it can be picked up by Kibana
+        during development
+
+      `,
+      flags: {
+        boolean: ['dist', 'watch'],
+        alias: {
+          d: 'dist',
+          w: 'watch',
+        },
+        help: `
+          --dist, -d  Outputs bundles in dist mode instead
+          --watch, -w  Starts the watch mode
+        `,
+      },
+      async run({ log, flags }) {
+        const dist = flags.dist;
+        if (dist !== undefined && typeof dist !== 'boolean') {
+          throw createFlagError('expected a single --dist flag');
+        }
+
+        const watch = flags.watch;
+        if (watch !== undefined && typeof watch !== 'boolean') {
+          throw createFlagError('expected a single --watch flag');
+        }
+
+        const found = findPluginDir();
+        if (!found) {
+          throw createFailError(
+            `Unable to find Kibana Platform plugin in [${process.cwd()}] or any of its parent directories. Has it been migrated properly? Does it have a kibana.json file?`
+          );
+        }
+
+        if (found.type === 'package') {
+          throw createFailError(`the plugin helpers do not currently support "package plugins"`);
+        }
+
+        const plugin = loadKibanaPlatformPlugin(found.dir);
+
+        if (!plugin.manifest.ui) {
+          log.info(
+            'Your plugin is server only and there is no need to run a dev task in order to get it ready to test. Please just run `yarn start` at the Kibana root and your plugin will be started.'
+          );
+          return;
+        }
+
+        const config = await loadConfig(log, plugin);
+        const sourceDir = plugin.directory;
+
+        const context: TaskContext = {
+          log,
+          dev: true,
+          dist,
+          watch,
+          plugin,
+          config,
+          sourceDir,
+          buildDir: '',
+          kibanaVersion: 'kibana',
+        };
+
+        await Tasks.initDev(context);
+        await Tasks.optimize(context);
+      },
+    })
     .execute();
 }

packages/kbn-plugin-helpers/src/integration_tests/build.test.ts

@@ -70,6 +70,7 @@ it('builds a generated plugin into a viable archive', async () => {
     " info deleting the build and target directories
      info running @kbn/optimizer
      │ succ browser bundle created at plugins/foo_test_plugin/build/kibana/fooTestPlugin/target/public
+     │ info stopping @kbn/optimizer
      info copying assets from \`public/assets\` to build
      info copying server source into the build and converting with babel
      info running yarn to install dependencies

packages/kbn-plugin-helpers/src/task_context.ts

@@ -11,8 +11,11 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { Plugin } from './load_kibana_platform_plugin';
 import { Config } from './config';
 
-export interface BuildContext {
+export interface TaskContext {
   log: ToolingLog;
+  dev: boolean;
+  dist?: boolean;
+  watch?: boolean;
   plugin: Plugin;
   config: Config;
   sourceDir: string;

packages/kbn-plugin-helpers/src/tasks/clean.ts

@@ -11,11 +11,11 @@ import { promisify } from 'util';
 
 import del from 'del';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncMkdir = promisify(Fs.mkdir);
 
-export async function initTargets({ log, sourceDir, buildDir }: BuildContext) {
+export async function initTargets({ log, sourceDir, buildDir }: TaskContext) {
   log.info('deleting the build and target directories');
   await del(['build', 'target'], {
     cwd: sourceDir,
@@ -24,3 +24,10 @@ export async function initTargets({ log, sourceDir, buildDir }: BuildContext) {
   log.debug(`creating build output dir [${buildDir}]`);
   await asyncMkdir(buildDir, { recursive: true });
 }
+
+export async function initDev({ log, sourceDir }: TaskContext) {
+  log.info('deleting the target folder');
+  await del(['target'], {
+    cwd: sourceDir,
+  });
+}

packages/kbn-plugin-helpers/src/tasks/create_archive.ts

@@ -14,11 +14,11 @@ import del from 'del';
 import vfs from 'vinyl-fs';
 import zip from 'gulp-zip';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncPipeline = promisify(pipeline);
 
-export async function createArchive({ kibanaVersion, plugin, log }: BuildContext) {
+export async function createArchive({ kibanaVersion, plugin, log }: TaskContext) {
   const {
     manifest: { id },
     directory,

packages/kbn-plugin-helpers/src/tasks/optimize.ts

@@ -12,37 +12,46 @@ import { fork } from 'child_process';
 
 import * as Rx from 'rxjs';
 import { REPO_ROOT } from '@kbn/repo-info';
-import { createFailError } from '@kbn/dev-cli-errors';
 import { OptimizerConfig } from '@kbn/optimizer';
 import { Bundle, BundleRemotes } from '@kbn/optimizer/src/common';
 import { observeLines } from '@kbn/stdio-dev-helpers';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 type WorkerMsg = { success: true; warnings: string } | { success: false; error: string };
 
-export async function optimize({ log, plugin, sourceDir, buildDir }: BuildContext) {
+export async function optimize({
+  log,
+  dev,
+  dist,
+  watch,
+  plugin,
+  sourceDir,
+  buildDir,
+}: TaskContext) {
   if (!plugin.manifest.ui) {
     return;
   }
 
-  log.info('running @kbn/optimizer');
+  log.info(`running @kbn/optimizer${!!watch ? ' in watch mode (use CTRL+C to quit)' : ''}`);
   await log.indent(2, async () => {
     const optimizerConfig = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       examples: false,
       testPlugins: false,
       includeCoreBundle: true,
-      dist: true,
+      dist: !!dist,
+      watch: !!watch,
     });
 
     const bundle = new Bundle({
       id: plugin.manifest.id,
       contextDir: sourceDir,
       ignoreMetrics: true,
-      outputDir: Path.resolve(buildDir, 'target/public'),
+      outputDir: Path.resolve(dev ? sourceDir : buildDir, 'target/public'),
       sourceRoot: sourceDir,
       type: 'plugin',
+      manifestPath: Path.resolve(sourceDir, 'kibana.json'),
       remoteInfo: {
         pkgId: 'not-importable',
         targets: ['public', 'common'],
@@ -58,56 +67,91 @@ export async function optimize({ log, plugin, sourceDir, buildDir }: BuildContex
       stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
     });
 
-    const result = await Rx.lastValueFrom(
-      Rx.race(
-        observeLines(proc.stdout!).pipe(
-          Rx.tap((line) => log.debug(line)),
-          Rx.ignoreElements()
-        ),
-        observeLines(proc.stderr!).pipe(
-          Rx.tap((line) => log.error(line)),
-          Rx.ignoreElements()
-        ),
-        Rx.defer(() => {
-          proc.send({
-            workerConfig: worker,
-            bundles: JSON.stringify([bundle.toSpec()]),
-            bundleRemotes: remotes.toSpecJson(),
-          });
+    const rel = Path.relative(REPO_ROOT, bundle.outputDir);
 
-          return Rx.merge(
-            Rx.fromEvent<[WorkerMsg]>(proc, 'message').pipe(
-              Rx.map((msg) => {
-                return msg[0];
-              })
-            ),
-            Rx.fromEvent<Error>(proc, 'error').pipe(
-              Rx.map((error) => {
-                throw error;
-              })
-            )
-          ).pipe(
-            Rx.take(1),
-            Rx.tap({
-              complete() {
-                proc.kill('SIGKILL');
-              },
-            })
-          );
-        })
-      )
+    // Observe all events from child process
+    const eventObservable = Rx.merge(
+      observeLines(proc.stdout!).pipe(Rx.map((line) => ({ type: 'stdout', data: line }))),
+      observeLines(proc.stderr!).pipe(Rx.map((line) => ({ type: 'stderr', data: line }))),
+      Rx.fromEvent<[WorkerMsg]>(proc, 'message').pipe(
+        Rx.map((msg) => ({ type: 'message', data: msg[0] }))
+      ),
+      Rx.fromEvent<Error>(proc, 'error').pipe(Rx.map((error) => ({ type: 'error', data: error })))
     );
 
-    // cleanup unnecessary files
-    Fs.unlinkSync(Path.resolve(bundle.outputDir, '.kbn-optimizer-cache'));
+    const simpleOrWatchObservable = watch
+      ? eventObservable
+      : eventObservable.pipe(
+          Rx.take(1),
+          Rx.tap({
+            complete() {
+              proc.kill('SIGKILL');
+            },
+          })
+        );
 
-    const rel = Path.relative(REPO_ROOT, bundle.outputDir);
-    if (!result.success) {
-      throw createFailError(`Optimizer failure: ${result.error}`);
-    } else if (result.warnings) {
-      log.warning(`browser bundle created at ${rel}, but with warnings:\n${result.warnings}`);
-    } else {
-      log.success(`browser bundle created at ${rel}`);
+    // Subscribe to eventObservable to log events
+    const eventSubscription = simpleOrWatchObservable.subscribe((event) => {
+      if (event.type === 'stdout') {
+        log.debug(event.data as string);
+      } else if (event.type === 'stderr') {
+        log.error(event.data as Error);
+      } else if (event.type === 'message') {
+        const result = event.data as WorkerMsg;
+        // Handle message event
+        if (!result.success) {
+          log.error(`Optimizer failure: ${result.error}`);
+        } else if (result.warnings) {
+          log.warning(`browser bundle created at ${rel}, but with warnings:\n${result.warnings}`);
+        } else {
+          log.success(`browser bundle created at ${rel}`);
+        }
+      } else if (event.type === 'error') {
+        log.error(event.data as Error);
+      }
+    });
+
+    // Send message to child process
+    proc.send({
+      workerConfig: worker,
+      bundles: JSON.stringify([bundle.toSpec()]),
+      bundleRemotes: remotes.toSpecJson(),
+    });
+
+    // Cleanup fn definition
+    const cleanup = () => {
+      // Cleanup unnecessary files
+      try {
+        Fs.unlinkSync(Path.resolve(bundle.outputDir, '.kbn-optimizer-cache'));
+      } catch {
+        // no-op
+      }
+
+      // Unsubscribe from eventObservable
+      eventSubscription.unsubscribe();
+
+      log.info('stopping @kbn/optimizer');
+    };
+
+    // if watch mode just wait for the first event then cleanup and exit
+    if (!watch) {
+      // Wait for parent process to exit if not in watch mode
+      await new Promise<void>((resolve) => {
+        proc.once('exit', () => {
+          cleanup();
+          resolve();
+        });
+      });
+
+      return;
     }
+
+    // Wait for parent process to exit if not in watch mode
+    await new Promise<void>((resolve) => {
+      process.once('exit', () => {
+        cleanup();
+        resolve();
+      });
+    });
   });
 }

packages/kbn-plugin-helpers/src/tasks/optimize_worker.ts

@@ -26,26 +26,33 @@ process.on('message', (msg: any) => {
   const webpackConfig = getWebpackConfig(bundle, remotes, workerConfig);
   const compiler = webpack(webpackConfig);
 
-  compiler.run((error, stats) => {
-    if (error) {
-      send.call(process, {
-        success: false,
-        error: error.message,
-      });
-      return;
-    }
+  compiler.watch(
+    {
+      // Example
+      aggregateTimeout: 300,
+      poll: undefined,
+    },
+    (error, stats) => {
+      if (error) {
+        send.call(process, {
+          success: false,
+          error: error.message,
+        });
+        return;
+      }
 
-    if (stats.hasErrors()) {
-      send.call(process, {
-        success: false,
-        error: `Failed to compile with webpack:\n${stats.toString()}`,
-      });
-      return;
-    }
+      if (stats.hasErrors()) {
+        send.call(process, {
+          success: false,
+          error: `Failed to compile with webpack:\n${stats.toString()}`,
+        });
+        return;
+      }
 
-    send.call(process, {
-      success: true,
-      warnings: stats.hasWarnings() ? stats.toString() : '',
-    });
-  });
+      send.call(process, {
+        success: true,
+        warnings: stats.hasWarnings() ? stats.toString() : '',
+      });
+    }
+  );
 });

packages/kbn-plugin-helpers/src/tasks/write_public_assets.ts

@@ -11,11 +11,11 @@ import { promisify } from 'util';
 
 import vfs from 'vinyl-fs';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncPipeline = promisify(pipeline);
 
-export async function writePublicAssets({ log, plugin, sourceDir, buildDir }: BuildContext) {
+export async function writePublicAssets({ log, plugin, sourceDir, buildDir }: TaskContext) {
   if (!plugin.manifest.ui) {
     return;
   }

packages/kbn-plugin-helpers/src/tasks/write_server_files.ts

@@ -13,7 +13,7 @@ import vfs from 'vinyl-fs';
 import { transformFileStream } from '@kbn/dev-utils';
 import { transformFileWithBabel } from './transform_file_with_babel';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const asyncPipeline = promisify(pipeline);
 
@@ -24,7 +24,7 @@ export async function writeServerFiles({
   sourceDir,
   buildDir,
   kibanaVersion,
-}: BuildContext) {
+}: TaskContext) {
   log.info('copying server source into the build and converting with babel');
 
   // copy source files and apply some babel transformations in the process

packages/kbn-plugin-helpers/src/tasks/yarn_install.ts

@@ -11,11 +11,11 @@ import Path from 'path';
 
 import execa from 'execa';
 
-import { BuildContext } from '../build_context';
+import { TaskContext } from '../task_context';
 
 const winVersion = (path: string) => (process.platform === 'win32' ? `${path}.cmd` : path);
 
-export async function yarnInstall({ log, buildDir, config }: BuildContext) {
+export async function yarnInstall({ log, buildDir, config }: TaskContext) {
   const pkgJson = Path.resolve(buildDir, 'package.json');
 
   if (config?.skipInstallDependencies || !Fs.existsSync(pkgJson)) {