[dev-utils/procRunner] wrap stop and close promises in timeouts (#17320) (#17321)

In an attempt to debug random stalls we are seeing in CI, the proc runner will now send SIGKILL if a process does not exit after receiving the standard exit signal it receives (usually SIGTERM). Additionally, if the child process object never emits "close" or "exit" another timeout will trigger an error

packages/kbn-dev-utils/src/proc_runner/proc.js

@@ -12,6 +12,28 @@ import { log } from './log';
 import { observeLines } from './observe_lines';
 import { observeChildProcess } from './observe_child_process';
 
+const SECOND = 1000;
+const STOP_TIMEOUT = 30 * SECOND;
+
+async function withTimeout(attempt, ms, onTimeout) {
+  const TIMEOUT = Symbol('timeout');
+  try {
+    await Promise.race([
+      attempt(),
+      new Promise((resolve, reject) => setTimeout(
+        () => reject(TIMEOUT),
+        STOP_TIMEOUT
+      ))
+    ]);
+  } catch (error) {
+    if (error === TIMEOUT) {
+      await onTimeout();
+    } else {
+      throw error;
+    }
+  }
+}
+
 export function createProc(name, { cmd, args, cwd, env, stdin }) {
   log.info('[%s] > %s', name, cmd, args.join(' '));
 
@@ -58,8 +80,26 @@ export function createProc(name, { cmd, args, cwd, env, stdin }) {
     closedPromise = this.outcomePromise.then(() => {}, () => {});
 
     async stop(signal) {
-      await treeKillAsync(childProcess.pid, signal);
-      await this.closedPromise;
+      await withTimeout(
+        async () => {
+          await treeKillAsync(childProcess.pid, signal);
+        },
+        STOP_TIMEOUT,
+        async () => {
+          log.warning(`Proc "${name}" was sent "${signal}" and didn't exit after ${STOP_TIMEOUT} ms, sending SIGKILL`);
+          await treeKillAsync(childProcess.pid, 'SIGKILL');
+        }
+      );
+
+      await withTimeout(
+        async () => {
+          await this.closedPromise;
+        },
+        STOP_TIMEOUT,
+        async () => {
+          throw new Error(`Proc "${name}" was stopped but never emiited either the "close" or "exit" events after ${STOP_TIMEOUT} ms`);
+        }
+      );
     }
   }();
 }