[9.0] [Build] Fix parallel stderr (#223177) (#223481)

# Backport This will backport the following commits from `main` to `9.0`: - [[Build] Fix parallel stderr (#223177)](https://github.com/elastic/kibana/pull/223177)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Brad White <Ikuni17@users.noreply.github.com>
2025-06-27 18:51:07 -04:00 · 2025-06-12 02:32:20 +02:00 · 2025-06-12 02:32:20 +02:00 · b1f63fb45c
commit b1f63fb45c
parent a441fd60e7
6 changed files with 188 additions and 108 deletions
--- a/src/dev/build/build_distributables.ts
+++ b/src/dev/build/build_distributables.ts
@ -186,14 +186,19 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
    artifactTasks.push(Tasks.CreateDockerContexts);
  }

-  await Promise.allSettled(
+  const results = await Promise.allSettled(
    // createRunner for each task to ensure each task gets its own Build instance
    artifactTasks.map(async (task) => await createRunner({ config, log, bufferLogs: true })(task))
  );

+  log.write('--- Finalizing Kibana artifacts');
+
+  if (results.some((result) => result.status === 'rejected')) {
+    throw new Error('One or more artifact tasks failed. Check the logs for details.');
+  }
+
  /**
   * finalize artifacts by writing sha1sums of each into the target directory
   */
-  log.write('--- Finalizing Kibana artifacts');
  await globalRun(Tasks.WriteShaSums);
 }
--- a/src/dev/build/lib/mocks/get_config.ts
+++ b/src/dev/build/lib/mocks/get_config.ts
@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { REPO_ROOT } from '@kbn/repo-info';
+import { Config } from '../config';
+
+export function getMockConfig() {
+  return new Config(
+    true,
+    false,
+    {
+      version: '8.0.0',
+      engines: {
+        node: '*',
+      },
+      workspaces: {
+        packages: [],
+      },
+    } as any,
+    '1.2.3',
+    REPO_ROOT,
+    {
+      buildNumber: 1234,
+      buildSha: 'abcd1234',
+      buildShaShort: 'abcd',
+      buildVersion: '8.0.0',
+      buildDate: '2023-05-15T23:12:09.000Z',
+    },
+    false,
+    false,
+    null,
+    '',
+    '',
+    false,
+    true,
+    true,
+    {},
+    {}
+  );
+}
--- a/src/dev/build/lib/build.test.ts
+++ b/src/dev/build/lib/build.test.ts
@ -7,46 +7,14 @@
 * License v3.0 only", or the "Server Side Public License, v 1".
 */

-import { REPO_ROOT } from '@kbn/repo-info';
 import { createAbsolutePathSerializer } from '@kbn/jest-serializers';

-import { Config } from './config';
 import { Build } from './build';
+import { getMockConfig } from './__mocks__/get_config';

 expect.addSnapshotSerializer(createAbsolutePathSerializer());

-const config = new Config(
-  true,
-  false,
-  {
-    version: '8.0.0',
-    engines: {
-      node: '*',
-    },
-    workspaces: {
-      packages: [],
-    },
-  } as any,
-  '1.2.3',
-  REPO_ROOT,
-  {
-    buildNumber: 1234,
-    buildSha: 'abcd1234',
-    buildShaShort: 'abcd',
-    buildVersion: '8.0.0',
-    buildDate: '2023-05-15T23:12:09+0000',
-  },
-  false,
-  false,
-  null,
-  '',
-  '',
-  false,
-  true,
-  true,
-  {},
-  {}
-);
+const config = getMockConfig();

 const linuxPlatform = config.getPlatform('linux', 'x64');
 const linuxArmPlatform = config.getPlatform('linux', 'arm64');
--- a/src/dev/build/lib/exec.test.ts
+++ b/src/dev/build/lib/exec.test.ts
@ -13,6 +13,8 @@ import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
 import { createStripAnsiSerializer, createRecursiveSerializer } from '@kbn/jest-serializers';

 import { exec } from './exec';
+import { Build } from './build';
+import { getMockConfig } from './__mocks__/get_config';

 const testWriter = new ToolingLogCollectingWriter();
 const log = new ToolingLog();
@ -26,28 +28,88 @@ expect.addSnapshotSerializer(
  )
 );

-beforeEach(() => {
-  testWriter.messages.length = 0;
-});
+jest.mock('./build', () => ({
+  Build: jest.fn().mockImplementation(() => ({
+    getBufferLogs: jest.fn().mockReturnValue(true),
+    getBuildDesc: jest.fn().mockReturnValue('test-build'),
+    getBuildArch: jest.fn().mockReturnValue('x64'),
+  })),
+}));

-it('executes a command, logs the command, and logs the output', async () => {
-  await exec(log, process.execPath, ['-e', 'console.log("hi")']);
-  expect(testWriter.messages).toMatchInlineSnapshot(`
-    Array [
-      " debg $ <nodedir>/node -e console.log(\\"hi\\")",
-      " debg hi",
-    ]
-  `);
-});
+const config = getMockConfig();

-it('logs using level: option', async () => {
-  await exec(log, process.execPath, ['-e', 'console.log("hi")'], {
-    level: 'info',
+describe('exec', () => {
+  let mockBuild: jest.Mocked<Build>;
+
+  beforeEach(() => {
+    testWriter.messages.length = 0;
+
+    jest.clearAllMocks();
+    mockBuild = new Build(config, true) as jest.Mocked<Build>;
+  });
+
+  it('executes a command, logs the command, and logs the output', async () => {
+    await exec(log, process.execPath, ['-e', 'console.log("hi")']);
+    expect(testWriter.messages).toMatchInlineSnapshot(`
+          Array [
+            " debg $ <nodedir>/node -e console.log(\\"hi\\")",
+            " debg hi",
+          ]
+      `);
+  });
+
+  it('logs using level: option', async () => {
+    await exec(log, process.execPath, ['-e', 'console.log("hi")'], {
+      level: 'info',
+    });
+    expect(testWriter.messages).toMatchInlineSnapshot(`
+          Array [
+            " info $ <nodedir>/node -e console.log(\\"hi\\")",
+            " info hi",
+          ]
+      `);
+  });
+
+  it('collects and logs output when bufferLogs is true', async () => {
+    await exec(log, process.execPath, ['-e', 'console.log("buffered output")'], {
+      build: mockBuild,
+    });
+
+    expect(testWriter.messages).toMatchInlineSnapshot(`
+      Array [
+        "--- ✅ test-build [x64]",
+        "   │ debg $ <nodedir>/node -e console.log(\\"buffered output\\")",
+        "   │ debg buffered output",
+      ]
+    `);
+  });
+
+  it('throws error when command fails when bufferLogs is true', async () => {
+    try {
+      await expect(
+        await exec(log, process.execPath, ['-e', 'process.exit(1)'], {
+          build: mockBuild,
+        })
+      ).rejects.toThrow();
+    } catch (error) {
+      expect(error).toBeTruthy();
+      expect(error.message).toMatchInlineSnapshot(
+        `"Command failed with exit code 1: <nodedir>/node -e process.exit(1)"`
+      );
+    }
+  });
+
+  it('handles stderr output when bufferLogs is true', async () => {
+    await exec(log, process.execPath, ['-e', 'console.error("error output: exit code 123")'], {
+      build: mockBuild,
+    });
+
+    expect(testWriter.messages).toMatchInlineSnapshot(`
+      Array [
+        "--- ✅ test-build [x64]",
+        "   │ debg $ <nodedir>/node -e console.error(\\"error output: exit code 123\\")",
+        "   │ERROR error output: exit code 123",
+      ]
+    `);
  });
-  expect(testWriter.messages).toMatchInlineSnapshot(`
-    Array [
-      " info $ <nodedir>/node -e console.log(\\"hi\\")",
-      " info hi",
-    ]
-  `);
 });
--- a/src/dev/build/lib/exec.ts
+++ b/src/dev/build/lib/exec.ts
@ -23,12 +23,43 @@ interface Options {
  build?: Build;
 }

+interface LogLine {
+  level: Exclude<LogLevel, 'silent'>;
+  chunk: string;
+}
+
+const handleBufferChunk = (chunk: Buffer, level: LogLine['level']): LogLine => {
+  return {
+    level,
+    chunk: chunk.toString().trim(),
+  };
+};
+
+const outputBufferedLogs = (
+  log: ToolingLog,
+  build: Build,
+  logBuildCmd: () => void,
+  logs: LogLine[] | undefined,
+  success: boolean
+) => {
+  log.write(`--- ${success ? '✅' : '❌'} ${build.getBuildDesc()} [${build.getBuildArch()}]`);
+
+  log.indent(4, () => {
+    logBuildCmd();
+
+    if (logs?.length) {
+      logs.forEach((line) => log[line.level](line.chunk));
+    }
+  });
+};
+
 export async function exec(
  log: ToolingLog,
  cmd: string,
  args: string[],
  { level = 'debug', cwd, env, exitAfter, build }: Options = {}
 ) {
+  const logBuildCmd = () => log[level](chalk.dim('$'), cmd, ...args);
  const bufferLogs = build && build?.getBufferLogs();

  const proc = execa(cmd, args, {
@ -39,26 +70,26 @@ export async function exec(
  });

  if (bufferLogs) {
-    const stdout$ = fromEvent<Buffer>(proc.stdout!, 'data').pipe(map((chunk) => chunk.toString()));
-    const stderr$ = fromEvent<Buffer>(proc.stderr!, 'data').pipe(map((chunk) => chunk.toString()));
+    const isDockerBuild = cmd === './build_docker.sh';
+    const stdout$ = fromEvent<Buffer>(proc.stdout!, 'data').pipe<LogLine>(
+      map((chunk) => handleBufferChunk(chunk, level))
+    );
+    // docker build uses stderr as a normal output stream
+    const stderr$ = fromEvent<Buffer>(proc.stderr!, 'data').pipe<LogLine>(
+      map((chunk) => handleBufferChunk(chunk, isDockerBuild ? level : 'error'))
+    );
    const close$ = fromEvent(proc, 'close');
-
-    await merge(stdout$, stderr$)
-      .pipe(takeUntil(close$), toArray())
-      .toPromise()
-      .then((logs) => {
-        log.write(`--- ${build.getBuildDesc()} [${build.getBuildArch()}]`);
-
-        log.indent(4, () => {
-          log[level](chalk.dim('$'), cmd, ...args);
-
-          if (logs?.length) {
-            logs.forEach((line: string) => log[level](line.trim()));
-          }
-        });
+    const logs = await merge(stdout$, stderr$).pipe(takeUntil(close$), toArray()).toPromise();
+    await proc
+      .then(() => {
+        outputBufferedLogs(log, build, logBuildCmd, logs, true);
+      })
+      .catch((error) => {
+        outputBufferedLogs(log, build, logBuildCmd, logs, false);
+        throw error;
      });
  } else {
-    log[level](chalk.dim('$'), cmd, ...args);
+    logBuildCmd();

    await watchStdioForLine(proc, (line: string) => log[level](line), exitAfter);
  }
--- a/src/dev/build/tasks/fetch_agent_versions_list.test.ts
+++ b/src/dev/build/tasks/fetch_agent_versions_list.test.ts
@ -10,48 +10,17 @@
 import fetch from 'node-fetch';
 import pRetry from 'p-retry';

-import { REPO_ROOT } from '@kbn/repo-info';
 import { ToolingLog } from '@kbn/tooling-log';

 import { FetchAgentVersionsList } from './fetch_agent_versions_list';
-import { Build, Config, write } from '../lib';
+import { Build, write } from '../lib';
+import { getMockConfig } from '../lib/__mocks__/get_config';

 jest.mock('node-fetch');
 jest.mock('p-retry');
 jest.mock('../lib');

-const config = new Config(
-  true,
-  false,
-  {
-    version: '8.0.0',
-    engines: {
-      node: '*',
-    },
-    workspaces: {
-      packages: [],
-    },
-  } as any,
-  '1.2.3',
-  REPO_ROOT,
-  {
-    buildNumber: 1234,
-    buildSha: 'abcd1234',
-    buildShaShort: 'abcd',
-    buildVersion: '8.0.0',
-    buildDate: '2023-05-15T23:12:09.000Z',
-  },
-  false,
-  false,
-  null,
-  '',
-  '',
-  false,
-  true,
-  true,
-  {},
-  {}
-);
+const config = getMockConfig();

 const mockedFetch = fetch as jest.MockedFunction<typeof fetch>;
 const mockedPRetry = pRetry as jest.MockedFunction<typeof pRetry>;