[buildkite] Add more logging and debug information to PR pipeline generation (#100709)

.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap

@@ -3,6 +3,8 @@
 exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
 [
   {
+    "name": "bwc-snapshots",
+    "pipeline": {
       "steps": [
         {
           "group": "bwc-snapshots",
@@ -34,7 +36,10 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
         },
       ],
     },
+  },
   {
+    "name": "using-defaults",
+    "pipeline": {
       "env": {
         "CUSTOM_ENV_VAR": "value",
       },
@@ -45,12 +50,15 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
         },
       ],
     },
+  },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
 [
   {
+    "name": "docs-check",
+    "pipeline": {
       "steps": [
         {
           "agents": {
@@ -65,12 +73,15 @@ exports[`generatePipelines should generate correct pipelines with only docs chan
         },
       ],
     },
+  },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
 [
   {
+    "name": "full-bwc",
+    "pipeline": {
       "steps": [
         {
           "group": "bwc",
@@ -154,7 +165,10 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
         },
       ],
     },
+  },
   {
+    "name": "using-defaults",
+    "pipeline": {
       "env": {
         "CUSTOM_ENV_VAR": "value",
       },
@@ -165,12 +179,15 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
         },
       ],
     },
+  },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
 [
   {
+    "name": "using-defaults",
+    "pipeline": {
       "env": {
         "CUSTOM_ENV_VAR": "value",
       },
@@ -181,5 +198,6 @@ exports[`generatePipelines should generate correct pipeline when using a trigger
         },
       ],
     },
+  },
 ]
 `;

.buildkite/scripts/pull-request/pipeline.generate.ts

@@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline";
 const pipelines = generatePipelines();
 
 for (const pipeline of pipelines) {
-  if (!process.env.CI) {
-    // Just for local debugging purposes
+  const yaml = stringify(pipeline.pipeline);
+
+  console.log(`--- Generated pipeline: ${pipeline.name}`);
+  console.log(yaml);
+
+  // Only do the pipeline upload if we're actually in CI
+  // This lets us run the tool locally and see the output
+  if (process.env.CI) {
     console.log("");
-    console.log(stringify(pipeline));
-  } else {
+    console.log("Uploading pipeline...");
+
     execSync(`buildkite-agent pipeline upload`, {
-      input: stringify(pipeline),
+      input: yaml,
       stdio: ["pipe", "inherit", "inherit"],
     });
   }

.buildkite/scripts/pull-request/pipeline.sh

@@ -2,5 +2,8 @@
 
 set -euo pipefail
 
+echo --- Installing bun
 npm install -g bun
+
+echo --- Generating pipeline
 bun .buildkite/scripts/pull-request/pipeline.generate.ts

.buildkite/scripts/pull-request/pipeline.ts

@@ -116,6 +116,7 @@ export const generatePipelines = (
     .filter((x) => x);
 
   if (!changedFiles?.length) {
+    console.log("Doing git fetch and getting merge-base");
     const mergeBase = execSync(
       `git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`,
       { cwd: PROJECT_ROOT }
@@ -123,12 +124,17 @@ export const generatePipelines = (
       .toString()
       .trim();
 
+    console.log(`Merge base: ${mergeBase}`);
+
     const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim();
 
     changedFiles = changedFilesOutput
       .split("\n")
       .map((x) => x.trim())
       .filter((x) => x);
+
+    console.log("Changed files (first 50):");
+    console.log(changedFiles.slice(0, 50).join("\n"));
   }
 
   let filters: ((pipeline: EsPipeline) => boolean)[] = [
@@ -154,9 +160,9 @@ export const generatePipelines = (
   pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
 
   const finalPipelines = pipelines.map((pipeline) => {
-    const finalPipeline = { ...pipeline };
-    delete finalPipeline.config;
-    delete finalPipeline.name;
+    const finalPipeline = { name: pipeline.name, pipeline: { ...pipeline } };
+    delete finalPipeline.pipeline.config;
+    delete finalPipeline.pipeline.name;
 
     return finalPipeline;
   });