[kbn-pm] Implement kbn watch. (#16892)

package.json

@@ -70,7 +70,8 @@
     "uiFramework:start": "cd packages/kbn-ui-framework && yarn docSiteStart",
     "uiFramework:build": "cd packages/kbn-ui-framework && yarn docSiteBuild",
     "uiFramework:createComponent": "cd packages/kbn-ui-framework && yarn createComponent",
-    "uiFramework:documentComponent": "cd packages/kbn-ui-framework && yarn documentComponent"
+    "uiFramework:documentComponent": "cd packages/kbn-ui-framework && yarn documentComponent",
+    "kbn:watch": "node scripts/kibana --dev --logging.json=false"
   },
   "repository": {
     "type": "git",

packages/kbn-datemath/package.json

@@ -7,7 +7,8 @@
   "main": "target/index.js",
   "scripts": {
     "build": "babel src --out-dir target",
-    "kbn:bootstrap": "yarn build"
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
   },
   "devDependencies": {
     "babel-cli": "^6.26.0",

packages/kbn-pm/package.json

@@ -6,6 +6,7 @@
   "private": true,
   "scripts": {
     "build": "webpack",
+    "kbn:watch": "webpack --watch --progress",
     "prettier": "prettier --write './src/**/*.ts'"
   },
   "devDependencies": {
@@ -52,6 +53,7 @@
     "ora": "^1.4.0",
     "prettier": "^1.11.1",
     "read-pkg": "^3.0.0",
+    "rxjs": "^5.5.7",
     "spawn-sync": "^1.0.15",
     "string-replace-loader": "^1.3.0",
     "strip-ansi": "^4.0.0",

packages/kbn-pm/src/commands/index.ts

@@ -20,9 +20,11 @@ export interface Command {
 import { BootstrapCommand } from './bootstrap';
 import { CleanCommand } from './clean';
 import { RunCommand } from './run';
+import { WatchCommand } from './watch';
 
 export const commands: { [key: string]: Command } = {
   bootstrap: BootstrapCommand,
   clean: CleanCommand,
   run: RunCommand,
+  watch: WatchCommand,
 };

packages/kbn-pm/src/commands/watch.ts

@@ -0,0 +1,75 @@
+import chalk from 'chalk';
+import { topologicallyBatchProjects, ProjectMap } from '../utils/projects';
+import { parallelizeBatches } from '../utils/parallelize';
+import { waitUntilWatchIsReady } from '../utils/watch';
+import { Command } from './';
+
+/**
+ * Name of the script in the package/project package.json file to run during `kbn watch`.
+ */
+const watchScriptName = 'kbn:watch';
+
+/**
+ * Name of the Kibana project.
+ */
+const kibanaProjectName = 'kibana';
+
+/**
+ * Command that traverses through list of available projects/packages that have `kbn:watch` script in their
+ * package.json files, groups them into topology aware batches and then processes theses batches one by one
+ * running `kbn:watch` scripts in parallel within the same batch.
+ *
+ * Command internally relies on the fact that most of the build systems that are triggered by `kbn:watch`
+ * will emit special "marker" once build/watch process is ready that we can use as completion condition for
+ * the `kbn:watch` script and eventually for the entire batch. Currently we support completion "markers" for
+ * `webpack` and `tsc` only, for the rest we rely on predefined timeouts.
+ */
+export const WatchCommand: Command = {
+  name: 'watch',
+  description: 'Runs `kbn:watch` script for every project.',
+
+  async run(projects, projectGraph) {
+    const projectsWithWatchScript: ProjectMap = new Map();
+    for (const project of projects.values()) {
+      if (project.hasScript(watchScriptName)) {
+        projectsWithWatchScript.set(project.name, project);
+      }
+    }
+
+    const projectNames = Array.from(projectsWithWatchScript.keys());
+    console.log(
+      chalk.bold(
+        chalk.green(
+          `Running ${watchScriptName} scripts for [${projectNames.join(', ')}].`
+        )
+      )
+    );
+
+    // Kibana should always be run the last, so we don't rely on automatic
+    // topological batching and push it to the last one-entry batch manually.
+    projectsWithWatchScript.delete(kibanaProjectName);
+
+    const batchedProjects = topologicallyBatchProjects(
+      projectsWithWatchScript,
+      projectGraph
+    );
+
+    if (projects.has(kibanaProjectName)) {
+      batchedProjects.push([projects.get(kibanaProjectName)!]);
+    }
+
+    await parallelizeBatches(batchedProjects, async pkg => {
+      const completionHint = await waitUntilWatchIsReady(
+        pkg.runScriptStreaming(watchScriptName).stdout
+      );
+
+      console.log(
+        chalk.bold(
+          `[${chalk.green(
+            pkg.name
+          )}] Initial build completed (${completionHint}).`
+        )
+      );
+    });
+  },
+};

packages/kbn-pm/src/utils/project.ts

@@ -157,7 +157,7 @@ export class Project {
     return runScriptInPackage(scriptName, args, this);
   }
 
-  async runScriptStreaming(scriptName: string, args: string[] = []) {
+  runScriptStreaming(scriptName: string, args: string[] = []) {
     return runScriptInPackageStreaming(scriptName, args, this);
   }
 

packages/kbn-pm/src/utils/scripts.ts

@@ -41,7 +41,7 @@ export async function runScriptInPackage(
 /**
  * Run script in the given directory
  */
-export async function runScriptInPackageStreaming(
+export function runScriptInPackageStreaming(
   script: string,
   args: string[],
   pkg: Project
@@ -50,7 +50,7 @@ export async function runScriptInPackageStreaming(
     cwd: pkg.path,
   };
 
-  await spawnStreaming(yarnPath, ['run', script, ...args], execOpts, {
+  return spawnStreaming(yarnPath, ['run', script, ...args], execOpts, {
     prefix: pkg.name,
   });
 }

packages/kbn-pm/src/utils/watch.test.ts

@@ -0,0 +1,57 @@
+import { EventEmitter } from 'events';
+import { waitUntilWatchIsReady } from './watch';
+
+describe('#waitUntilWatchIsReady', () => {
+  let buildOutputStream: EventEmitter;
+  let completionHintPromise: Promise<string>;
+  beforeEach(() => {
+    jest.useFakeTimers();
+
+    buildOutputStream = new EventEmitter();
+    completionHintPromise = waitUntilWatchIsReady(buildOutputStream, {
+      handlerDelay: 100,
+      handlerReadinessTimeout: 50,
+    });
+  });
+
+  test('`waitUntilWatchIsReady` correctly handles `webpack` output', async () => {
+    buildOutputStream.emit('data', Buffer.from('$ webpack'));
+    buildOutputStream.emit('data', Buffer.from('Chunk Names'));
+
+    jest.runAllTimers();
+
+    expect(await completionHintPromise).toBe('webpack');
+  });
+
+  test('`waitUntilWatchIsReady` correctly handles `tsc` output', async () => {
+    buildOutputStream.emit('data', Buffer.from('$ tsc'));
+    buildOutputStream.emit('data', Buffer.from('Compilation complete.'));
+
+    jest.runAllTimers();
+
+    expect(await completionHintPromise).toBe('tsc');
+  });
+
+  test('`waitUntilWatchIsReady` fallbacks to default output handler if output is not recognizable', async () => {
+    buildOutputStream.emit('data', Buffer.from('$ some-cli'));
+    buildOutputStream.emit('data', Buffer.from('Compilation complete.'));
+    buildOutputStream.emit('data', Buffer.from('Chunk Names.'));
+
+    jest.runAllTimers();
+
+    expect(await completionHintPromise).toBe('timeout');
+  });
+
+  test('`waitUntilWatchIsReady` fallbacks to default output handler if none output is detected', async () => {
+    jest.runAllTimers();
+    expect(await completionHintPromise).toBe('timeout');
+  });
+
+  test('`waitUntilWatchIsReady` fails if output stream receives error', async () => {
+    buildOutputStream.emit('error', new Error('Uh, oh!'));
+
+    jest.runAllTimers();
+
+    await expect(completionHintPromise).rejects.toThrow(/Uh, oh!/);
+  });
+});

packages/kbn-pm/src/utils/watch.ts

@@ -0,0 +1,85 @@
+import { Observable, Subject } from 'rxjs';
+
+/**
+ * Number of milliseconds we wait before we fall back to the default watch handler.
+ */
+const defaultHandlerDelay = 3000;
+
+/**
+ * If default watch handler is used, then it's the number of milliseconds we wait for
+ * any build output before we consider watch task ready.
+ */
+const defaultHandlerReadinessTimeout = 2000;
+
+/**
+ * Describes configurable watch options.
+ */
+interface WatchOptions {
+  /**
+   * Number of milliseconds to wait before we fall back to default watch handler.
+   */
+  handlerDelay?: number;
+
+  /**
+   * Number of milliseconds that default watch handler waits for any build output before
+   * it considers initial build completed. If build process outputs anything in a given
+   * time span, the timeout is restarted.
+   */
+  handlerReadinessTimeout?: number;
+}
+
+function getWatchHandlers(
+  buildOutput$: Observable<string>,
+  {
+    handlerDelay = defaultHandlerDelay,
+    handlerReadinessTimeout = defaultHandlerReadinessTimeout,
+  }: WatchOptions
+) {
+  const typescriptHandler = buildOutput$
+    .first(data => data.includes('$ tsc'))
+    .map(() =>
+      buildOutput$
+        .first(data => data.includes('Compilation complete.'))
+        .mapTo('tsc')
+    );
+
+  const webpackHandler = buildOutput$
+    .first(data => data.includes('$ webpack'))
+    .map(() =>
+      buildOutput$.first(data => data.includes('Chunk Names')).mapTo('webpack')
+    );
+
+  const defaultHandler = Observable.of(undefined)
+    .delay(handlerReadinessTimeout)
+    .map(() =>
+      buildOutput$.timeout(handlerDelay).catch(() => Observable.of('timeout'))
+    );
+
+  return [typescriptHandler, webpackHandler, defaultHandler];
+}
+
+export function waitUntilWatchIsReady(
+  stream: NodeJS.EventEmitter,
+  opts: WatchOptions = {}
+) {
+  const buildOutput$ = new Subject<string>();
+  const onDataListener = (data: Buffer) =>
+    buildOutput$.next(data.toString('utf-8'));
+  const onEndListener = () => buildOutput$.complete();
+  const onErrorListener = (e: Error) => buildOutput$.error(e);
+
+  stream.once('end', onEndListener);
+  stream.once('error', onErrorListener);
+  stream.on('data', onDataListener);
+
+  return Observable.race(getWatchHandlers(buildOutput$, opts))
+    .mergeMap(whenReady => whenReady)
+    .finally(() => {
+      stream.removeListener('data', onDataListener);
+      stream.removeListener('end', onEndListener);
+      stream.removeListener('error', onErrorListener);
+
+      buildOutput$.complete();
+    })
+    .toPromise();
+}

packages/kbn-pm/webpack.config.js

@@ -54,4 +54,8 @@ module.exports = {
     __filename: false,
     __dirname: false,
   },
+
+  watchOptions: {
+    ignored: [/node_modules/, /vendor/],
+  },
 };

packages/kbn-pm/yarn.lock

@@ -3156,6 +3156,12 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^2.0.0"
     inherits "^2.0.1"
 
+rxjs@^5.5.7:
+  version "5.5.7"
+  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-5.5.7.tgz#afb3d1642b069b2fbf203903d6501d1acb4cda27"
+  dependencies:
+    symbol-observable "1.0.1"
+
 safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
@@ -3449,6 +3455,10 @@ supports-color@^5.2.0:
   dependencies:
     has-flag "^3.0.0"
 
+symbol-observable@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.0.1.tgz#8340fc4702c3122df5d22288f88283f513d3fdd4"
+
 tapable@^0.2.7:
   version "0.2.8"
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-0.2.8.tgz#99372a5c999bf2df160afc0d74bed4f47948cd22"