mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-06-28 01:22:26 -04:00
[buildkite] Add more logging and debug information to PR pipeline generation (#100709)
This commit is contained in:
Brian Seeders
GitHub
parent
731c253058
commit
81a441e636
4 changed files with 182 additions and 149 deletions
|
|
@ -3,47 +3,53 @@
|
|||
exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
|
||||
[
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"group": "bwc-snapshots",
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
|
||||
},
|
||||
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
|
||||
"matrix": {
|
||||
"setup": {
|
||||
"BWC_VERSION": [
|
||||
"7.17.14",
|
||||
"8.10.3",
|
||||
"8.11.0",
|
||||
],
|
||||
"name": "bwc-snapshots",
|
||||
"pipeline": {
|
||||
"steps": [
|
||||
{
|
||||
"group": "bwc-snapshots",
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "{{matrix.BWC_VERSION}}",
|
||||
},
|
||||
"label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
|
||||
"matrix": {
|
||||
"setup": {
|
||||
"BWC_VERSION": [
|
||||
"7.17.14",
|
||||
"8.10.3",
|
||||
"8.11.0",
|
||||
],
|
||||
},
|
||||
},
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
"name": "using-defaults",
|
||||
"pipeline": {
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -51,19 +57,22 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
|
|||
exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
|
||||
[
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
"name": "docs-check",
|
||||
"pipeline": {
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
|
||||
"label": "docs-check",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
|
||||
"label": "docs-check",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -71,99 +80,105 @@ exports[`generatePipelines should generate correct pipelines with only docs chan
|
|||
exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
|
||||
[
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"group": "bwc",
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
"name": "full-bwc",
|
||||
"pipeline": {
|
||||
"steps": [
|
||||
{
|
||||
"group": "bwc",
|
||||
"steps": [
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.0.0",
|
||||
},
|
||||
"key": "full-bwc:7_0_0",
|
||||
"label": "7.0.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.0.0",
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.0.1",
|
||||
},
|
||||
"key": "full-bwc:7_0_1",
|
||||
"label": "7.0.1 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"key": "full-bwc:7_0_0",
|
||||
"label": "7.0.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.1.0",
|
||||
},
|
||||
"key": "full-bwc:7_1_0",
|
||||
"label": "7.1.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.0.1",
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "8.10.0",
|
||||
},
|
||||
"key": "full-bwc:8_10_0",
|
||||
"label": "8.10.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"key": "full-bwc:7_0_1",
|
||||
"label": "7.0.1 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "8.11.0",
|
||||
},
|
||||
"key": "full-bwc:8_11_0",
|
||||
"label": "8.11.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "7.1.0",
|
||||
},
|
||||
"key": "full-bwc:7_1_0",
|
||||
"label": "7.1.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "8.10.0",
|
||||
},
|
||||
"key": "full-bwc:8_10_0",
|
||||
"label": "8.10.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
{
|
||||
"agents": {
|
||||
"buildDirectory": "/dev/shm/bk",
|
||||
"image": "family/elasticsearch-ubuntu-2004",
|
||||
"machineType": "custom-32-98304",
|
||||
"provider": "gcp",
|
||||
},
|
||||
"command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
|
||||
"env": {
|
||||
"BWC_VERSION": "8.11.0",
|
||||
},
|
||||
"key": "full-bwc:8_11_0",
|
||||
"label": "8.11.0 / bwc",
|
||||
"timeout_in_minutes": 300,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
"name": "using-defaults",
|
||||
"pipeline": {
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
@ -171,15 +186,18 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
|
|||
exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
|
||||
[
|
||||
{
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
"name": "using-defaults",
|
||||
"pipeline": {
|
||||
"env": {
|
||||
"CUSTOM_ENV_VAR": "value",
|
||||
},
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"command": "echo 'hello world'",
|
||||
"label": "test-step",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
|
|
|||
|
|
@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline";
|
|||
const pipelines = generatePipelines();
|
||||
|
||||
for (const pipeline of pipelines) {
|
||||
if (!process.env.CI) {
|
||||
// Just for local debugging purposes
|
||||
const yaml = stringify(pipeline.pipeline);
|
||||
|
||||
console.log(`--- Generated pipeline: ${pipeline.name}`);
|
||||
console.log(yaml);
|
||||
|
||||
// Only do the pipeline upload if we're actually in CI
|
||||
// This lets us run the tool locally and see the output
|
||||
if (process.env.CI) {
|
||||
console.log("");
|
||||
console.log(stringify(pipeline));
|
||||
} else {
|
||||
console.log("Uploading pipeline...");
|
||||
|
||||
execSync(`buildkite-agent pipeline upload`, {
|
||||
input: stringify(pipeline),
|
||||
input: yaml,
|
||||
stdio: ["pipe", "inherit", "inherit"],
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,5 +2,8 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
echo --- Installing bun
|
||||
npm install -g bun
|
||||
|
||||
echo --- Generating pipeline
|
||||
bun .buildkite/scripts/pull-request/pipeline.generate.ts
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ export const generatePipelines = (
|
|||
.filter((x) => x);
|
||||
|
||||
if (!changedFiles?.length) {
|
||||
console.log("Doing git fetch and getting merge-base");
|
||||
const mergeBase = execSync(
|
||||
`git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`,
|
||||
{ cwd: PROJECT_ROOT }
|
||||
|
|
@ -123,12 +124,17 @@ export const generatePipelines = (
|
|||
.toString()
|
||||
.trim();
|
||||
|
||||
console.log(`Merge base: ${mergeBase}`);
|
||||
|
||||
const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim();
|
||||
|
||||
changedFiles = changedFilesOutput
|
||||
.split("\n")
|
||||
.map((x) => x.trim())
|
||||
.filter((x) => x);
|
||||
|
||||
console.log("Changed files (first 50):");
|
||||
console.log(changedFiles.slice(0, 50).join("\n"));
|
||||
}
|
||||
|
||||
let filters: ((pipeline: EsPipeline) => boolean)[] = [
|
||||
|
|
@ -154,9 +160,9 @@ export const generatePipelines = (
|
|||
pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
|
||||
|
||||
const finalPipelines = pipelines.map((pipeline) => {
|
||||
const finalPipeline = { ...pipeline };
|
||||
delete finalPipeline.config;
|
||||
delete finalPipeline.name;
|
||||
const finalPipeline = { name: pipeline.name, pipeline: { ...pipeline } };
|
||||
delete finalPipeline.pipeline.config;
|
||||
delete finalPipeline.pipeline.name;
|
||||
|
||||
return finalPipeline;
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue