mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-06-28 09:28:55 -04:00
[buildkite] Add more logging and debug information to PR pipeline generation (#100709)
This commit is contained in:
Brian Seeders
GitHub
parent
731c253058
commit
81a441e636
4 changed files with 182 additions and 149 deletions
|
|
@ -3,47 +3,53 @@
|
||||||
exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
|
exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"steps": [
|
"name": "bwc-snapshots",
|
||||||
{
|
"pipeline": {
|
||||||
"group": "bwc-snapshots",
|
"steps": [
|
||||||
"steps": [
|
{
|
||||||
{
|
"group": "bwc-snapshots",
|
||||||
"agents": {
|
"steps": [
|
||||||
"buildDirectory": "/dev/shm/bk",
|
{
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
"agents": {
|
||||||
"machineType": "custom-32-98304",
|
"buildDirectory": "/dev/shm/bk",
|
||||||
"provider": "gcp",
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
},
|
"machineType": "custom-32-98304",
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
|
"provider": "gcp",
|
||||||
"env": {
|
|
||||||
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
|
|
||||||
},
|
|
||||||
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
|
|
||||||
"matrix": {
|
|
||||||
"setup": {
|
|
||||||
"BWC_VERSION": [
|
|
||||||
"7.17.14",
|
|
||||||
"8.10.3",
|
|
||||||
"8.11.0",
|
|
||||||
],
|
|
||||||
},
|
},
|
||||||
|
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
|
||||||
|
"env": {
|
||||||
|
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
|
||||||
|
},
|
||||||
|
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
|
||||||
|
"matrix": {
|
||||||
|
"setup": {
|
||||||
|
"BWC_VERSION": [
|
||||||
|
"7.17.14",
|
||||||
|
"8.10.3",
|
||||||
|
"8.11.0",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"timeout_in_minutes": 300,
|
],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
],
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"env": {
|
"name": "using-defaults",
|
||||||
"CUSTOM_ENV_VAR": "value",
|
"pipeline": {
|
||||||
},
|
"env": {
|
||||||
"steps": [
|
"CUSTOM_ENV_VAR": "value",
|
||||||
{
|
|
||||||
"command": "echo 'hello world'",
|
|
||||||
"label": "test-step",
|
|
||||||
},
|
},
|
||||||
],
|
"steps": [
|
||||||
|
{
|
||||||
|
"command": "echo 'hello world'",
|
||||||
|
"label": "test-step",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
`;
|
`;
|
||||||
|
|
@ -51,19 +57,22 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
|
||||||
exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
|
exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"steps": [
|
"name": "docs-check",
|
||||||
{
|
"pipeline": {
|
||||||
"agents": {
|
"steps": [
|
||||||
"buildDirectory": "/dev/shm/bk",
|
{
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
"agents": {
|
||||||
"machineType": "custom-32-98304",
|
"buildDirectory": "/dev/shm/bk",
|
||||||
"provider": "gcp",
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
|
"machineType": "custom-32-98304",
|
||||||
|
"provider": "gcp",
|
||||||
|
},
|
||||||
|
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
|
||||||
|
"label": "docs-check",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
|
],
|
||||||
"label": "docs-check",
|
},
|
||||||
"timeout_in_minutes": 300,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
`;
|
`;
|
||||||
|
|
@ -71,99 +80,105 @@ exports[`generatePipelines should generate correct pipelines with only docs chan
|
||||||
exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
|
exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"steps": [
|
"name": "full-bwc",
|
||||||
{
|
"pipeline": {
|
||||||
"group": "bwc",
|
"steps": [
|
||||||
"steps": [
|
{
|
||||||
{
|
"group": "bwc",
|
||||||
"agents": {
|
"steps": [
|
||||||
"buildDirectory": "/dev/shm/bk",
|
{
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
"agents": {
|
||||||
"machineType": "custom-32-98304",
|
"buildDirectory": "/dev/shm/bk",
|
||||||
"provider": "gcp",
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
|
"machineType": "custom-32-98304",
|
||||||
|
"provider": "gcp",
|
||||||
|
},
|
||||||
|
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
|
||||||
|
"env": {
|
||||||
|
"BWC_VERSION": "7.0.0",
|
||||||
|
},
|
||||||
|
"key": "full-bwc:7_0_0",
|
||||||
|
"label": "7.0.0 / bwc",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
|
{
|
||||||
"env": {
|
"agents": {
|
||||||
"BWC_VERSION": "7.0.0",
|
"buildDirectory": "/dev/shm/bk",
|
||||||
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
|
"machineType": "custom-32-98304",
|
||||||
|
"provider": "gcp",
|
||||||
|
},
|
||||||
|
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
|
||||||
|
"env": {
|
||||||
|
"BWC_VERSION": "7.0.1",
|
||||||
|
},
|
||||||
|
"key": "full-bwc:7_0_1",
|
||||||
|
"label": "7.0.1 / bwc",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"key": "full-bwc:7_0_0",
|
{
|
||||||
"label": "7.0.0 / bwc",
|
"agents": {
|
||||||
"timeout_in_minutes": 300,
|
"buildDirectory": "/dev/shm/bk",
|
||||||
},
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
{
|
"machineType": "custom-32-98304",
|
||||||
"agents": {
|
"provider": "gcp",
|
||||||
"buildDirectory": "/dev/shm/bk",
|
},
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
|
||||||
"machineType": "custom-32-98304",
|
"env": {
|
||||||
"provider": "gcp",
|
"BWC_VERSION": "7.1.0",
|
||||||
|
},
|
||||||
|
"key": "full-bwc:7_1_0",
|
||||||
|
"label": "7.1.0 / bwc",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
|
{
|
||||||
"env": {
|
"agents": {
|
||||||
"BWC_VERSION": "7.0.1",
|
"buildDirectory": "/dev/shm/bk",
|
||||||
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
|
"machineType": "custom-32-98304",
|
||||||
|
"provider": "gcp",
|
||||||
|
},
|
||||||
|
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
|
||||||
|
"env": {
|
||||||
|
"BWC_VERSION": "8.10.0",
|
||||||
|
},
|
||||||
|
"key": "full-bwc:8_10_0",
|
||||||
|
"label": "8.10.0 / bwc",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"key": "full-bwc:7_0_1",
|
{
|
||||||
"label": "7.0.1 / bwc",
|
"agents": {
|
||||||
"timeout_in_minutes": 300,
|
"buildDirectory": "/dev/shm/bk",
|
||||||
},
|
"image": "family/elasticsearch-ubuntu-2004",
|
||||||
{
|
"machineType": "custom-32-98304",
|
||||||
"agents": {
|
"provider": "gcp",
|
||||||
"buildDirectory": "/dev/shm/bk",
|
},
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
|
||||||
"machineType": "custom-32-98304",
|
"env": {
|
||||||
"provider": "gcp",
|
"BWC_VERSION": "8.11.0",
|
||||||
|
},
|
||||||
|
"key": "full-bwc:8_11_0",
|
||||||
|
"label": "8.11.0 / bwc",
|
||||||
|
"timeout_in_minutes": 300,
|
||||||
},
|
},
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
|
],
|
||||||
"env": {
|
},
|
||||||
"BWC_VERSION": "7.1.0",
|
],
|
||||||
},
|
},
|
||||||
"key": "full-bwc:7_1_0",
|
|
||||||
"label": "7.1.0 / bwc",
|
|
||||||
"timeout_in_minutes": 300,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"agents": {
|
|
||||||
"buildDirectory": "/dev/shm/bk",
|
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
|
||||||
"machineType": "custom-32-98304",
|
|
||||||
"provider": "gcp",
|
|
||||||
},
|
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
|
|
||||||
"env": {
|
|
||||||
"BWC_VERSION": "8.10.0",
|
|
||||||
},
|
|
||||||
"key": "full-bwc:8_10_0",
|
|
||||||
"label": "8.10.0 / bwc",
|
|
||||||
"timeout_in_minutes": 300,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"agents": {
|
|
||||||
"buildDirectory": "/dev/shm/bk",
|
|
||||||
"image": "family/elasticsearch-ubuntu-2004",
|
|
||||||
"machineType": "custom-32-98304",
|
|
||||||
"provider": "gcp",
|
|
||||||
},
|
|
||||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
|
|
||||||
"env": {
|
|
||||||
"BWC_VERSION": "8.11.0",
|
|
||||||
},
|
|
||||||
"key": "full-bwc:8_11_0",
|
|
||||||
"label": "8.11.0 / bwc",
|
|
||||||
"timeout_in_minutes": 300,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"env": {
|
"name": "using-defaults",
|
||||||
"CUSTOM_ENV_VAR": "value",
|
"pipeline": {
|
||||||
},
|
"env": {
|
||||||
"steps": [
|
"CUSTOM_ENV_VAR": "value",
|
||||||
{
|
|
||||||
"command": "echo 'hello world'",
|
|
||||||
"label": "test-step",
|
|
||||||
},
|
},
|
||||||
],
|
"steps": [
|
||||||
|
{
|
||||||
|
"command": "echo 'hello world'",
|
||||||
|
"label": "test-step",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
`;
|
`;
|
||||||
|
|
@ -171,15 +186,18 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
|
||||||
exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
|
exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
"env": {
|
"name": "using-defaults",
|
||||||
"CUSTOM_ENV_VAR": "value",
|
"pipeline": {
|
||||||
},
|
"env": {
|
||||||
"steps": [
|
"CUSTOM_ENV_VAR": "value",
|
||||||
{
|
|
||||||
"command": "echo 'hello world'",
|
|
||||||
"label": "test-step",
|
|
||||||
},
|
},
|
||||||
],
|
"steps": [
|
||||||
|
{
|
||||||
|
"command": "echo 'hello world'",
|
||||||
|
"label": "test-step",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
`;
|
`;
|
||||||
|
|
|
||||||
|
|
@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline";
|
||||||
const pipelines = generatePipelines();
|
const pipelines = generatePipelines();
|
||||||
|
|
||||||
for (const pipeline of pipelines) {
|
for (const pipeline of pipelines) {
|
||||||
if (!process.env.CI) {
|
const yaml = stringify(pipeline.pipeline);
|
||||||
// Just for local debugging purposes
|
|
||||||
|
console.log(`--- Generated pipeline: ${pipeline.name}`);
|
||||||
|
console.log(yaml);
|
||||||
|
|
||||||
|
// Only do the pipeline upload if we're actually in CI
|
||||||
|
// This lets us run the tool locally and see the output
|
||||||
|
if (process.env.CI) {
|
||||||
console.log("");
|
console.log("");
|
||||||
console.log(stringify(pipeline));
|
console.log("Uploading pipeline...");
|
||||||
} else {
|
|
||||||
execSync(`buildkite-agent pipeline upload`, {
|
execSync(`buildkite-agent pipeline upload`, {
|
||||||
input: stringify(pipeline),
|
input: yaml,
|
||||||
stdio: ["pipe", "inherit", "inherit"],
|
stdio: ["pipe", "inherit", "inherit"],
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,5 +2,8 @@
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
echo --- Installing bun
|
||||||
npm install -g bun
|
npm install -g bun
|
||||||
|
|
||||||
|
echo --- Generating pipeline
|
||||||
bun .buildkite/scripts/pull-request/pipeline.generate.ts
|
bun .buildkite/scripts/pull-request/pipeline.generate.ts
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,7 @@ export const generatePipelines = (
|
||||||
.filter((x) => x);
|
.filter((x) => x);
|
||||||
|
|
||||||
if (!changedFiles?.length) {
|
if (!changedFiles?.length) {
|
||||||
|
console.log("Doing git fetch and getting merge-base");
|
||||||
const mergeBase = execSync(
|
const mergeBase = execSync(
|
||||||
`git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`,
|
`git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`,
|
||||||
{ cwd: PROJECT_ROOT }
|
{ cwd: PROJECT_ROOT }
|
||||||
|
|
@ -123,12 +124,17 @@ export const generatePipelines = (
|
||||||
.toString()
|
.toString()
|
||||||
.trim();
|
.trim();
|
||||||
|
|
||||||
|
console.log(`Merge base: ${mergeBase}`);
|
||||||
|
|
||||||
const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim();
|
const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim();
|
||||||
|
|
||||||
changedFiles = changedFilesOutput
|
changedFiles = changedFilesOutput
|
||||||
.split("\n")
|
.split("\n")
|
||||||
.map((x) => x.trim())
|
.map((x) => x.trim())
|
||||||
.filter((x) => x);
|
.filter((x) => x);
|
||||||
|
|
||||||
|
console.log("Changed files (first 50):");
|
||||||
|
console.log(changedFiles.slice(0, 50).join("\n"));
|
||||||
}
|
}
|
||||||
|
|
||||||
let filters: ((pipeline: EsPipeline) => boolean)[] = [
|
let filters: ((pipeline: EsPipeline) => boolean)[] = [
|
||||||
|
|
@ -154,9 +160,9 @@ export const generatePipelines = (
|
||||||
pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
|
pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
|
||||||
|
|
||||||
const finalPipelines = pipelines.map((pipeline) => {
|
const finalPipelines = pipelines.map((pipeline) => {
|
||||||
const finalPipeline = { ...pipeline };
|
const finalPipeline = { name: pipeline.name, pipeline: { ...pipeline } };
|
||||||
delete finalPipeline.config;
|
delete finalPipeline.pipeline.config;
|
||||||
delete finalPipeline.name;
|
delete finalPipeline.pipeline.name;
|
||||||
|
|
||||||
return finalPipeline;
|
return finalPipeline;
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue