elasticsearch/.buildkite/scripts/pull-request/pipeline.ts

import { parse } from "yaml";
import { readFileSync, readdirSync } from "fs";
import { basename, resolve } from "path";
import { execSync } from "child_process";

import { BuildkitePipeline, BuildkiteStep, EsPipeline, EsPipelineConfig } from "./types";
import { getBwcVersions, getSnapshotBwcVersions } from "./bwc-versions";

const PROJECT_ROOT = resolve(`${import.meta.dir}/../../..`);

const getArray = (strOrArray: string | string[] | undefined): string[] => {
  if (typeof strOrArray === "undefined") {
    return [];
  }

  return typeof strOrArray === "string" ? [strOrArray] : strOrArray;
};

const labelCheckAllow = (pipeline: EsPipeline, labels: string[]): boolean => {
  if (pipeline.config?.["allow-labels"]) {
    return getArray(pipeline.config["allow-labels"]).some((label) => labels.includes(label));
  }
  return true;
};

const labelCheckSkip = (pipeline: EsPipeline, labels: string[]): boolean => {
  if (pipeline.config?.["skip-labels"]) {
    return !getArray(pipeline.config["skip-labels"]).some((label) => labels.includes(label));
  }
  return true;
};

// Exclude the pipeline if all of the changed files in the PR are in at least one excluded region
const changedFilesExcludedCheck = (pipeline: EsPipeline, changedFiles: string[]): boolean => {
  if (pipeline.config?.["excluded-regions"]) {
    return !changedFiles.every((file) =>
      getArray(pipeline.config?.["excluded-regions"]).some((region) => file.match(region))
    );
  }
  return true;
};

// Include the pipeline if all of the changed files in the PR are in at least one included region
const changedFilesIncludedCheck = (pipeline: EsPipeline, changedFiles: string[]): boolean => {
  if (pipeline.config?.["included-regions"]) {
    return changedFiles.every((file) =>
      getArray(pipeline.config?.["included-regions"]).some((region) => file.match(region))
    );
  }
  return true;
};

const triggerCommentCheck = (pipeline: EsPipeline): boolean => {
  if (process.env["GITHUB_PR_TRIGGER_COMMENT"] && pipeline.config?.["trigger-phrase"]) {
    return !!process.env["GITHUB_PR_TRIGGER_COMMENT"].match(pipeline.config["trigger-phrase"]);
  }
  return false;
};

// There are so many BWC versions that we can't use the matrix feature in Buildkite, as it's limited to 20 elements per dimension
// So we need to duplicate the steps instead
// Recursively check for any steps that have a bwc_template attribute and expand them out into multiple steps, one for each BWC_VERSION
const doBwcTransforms = (step: BuildkitePipeline | BuildkiteStep) => {
  const stepsToExpand = (step.steps || []).filter((s) => s.bwc_template);
  step.steps = (step.steps || []).filter((s) => !s.bwc_template);

  for (const s of step.steps) {
    if (s.steps?.length) {
      doBwcTransforms(s);
    }
  }

  for (const stepToExpand of stepsToExpand) {
    for (const bwcVersion of getBwcVersions()) {
      let newStepJson = JSON.stringify(stepToExpand).replaceAll("$BWC_VERSION_SNAKE", bwcVersion.replaceAll(".", "_"));
      newStepJson = newStepJson.replaceAll("$BWC_VERSION", bwcVersion);
      const newStep = JSON.parse(newStepJson);
      delete newStep.bwc_template;
      step.steps.push(newStep);
    }
  }
};

export const generatePipelines = (
  directory: string = `${PROJECT_ROOT}/.buildkite/pipelines/pull-request`,
  changedFiles: string[] = []
) => {
  let defaults: EsPipelineConfig = { config: {} };
  defaults = parse(readFileSync(`${directory}/.defaults.yml`, "utf-8"));
  defaults.config = defaults.config || {};

  let pipelines: EsPipeline[] = [];
  const files = readdirSync(directory);
  for (const file of files) {
    if (!file.endsWith(".yml") || file.endsWith(".defaults.yml")) {
      continue;
    }

    let yaml = readFileSync(`${directory}/${file}`, "utf-8");
    yaml = yaml.replaceAll("$SNAPSHOT_BWC_VERSIONS", JSON.stringify(getSnapshotBwcVersions()));
    const pipeline: EsPipeline = parse(yaml) || {};

    pipeline.config = { ...defaults.config, ...(pipeline.config || {}) };

    // '.../build-benchmark.yml' => 'build-benchmark'
    const name = basename(file).split(".", 2)[0];
    pipeline.name = name;
    pipeline.config["trigger-phrase"] = pipeline.config["trigger-phrase"] || `.*run\\W+elasticsearch-ci/${name}.*`;

    pipelines.push(pipeline);
  }

  const labels = (process.env["GITHUB_PR_LABELS"] || "")
    .split(",")
    .map((x) => x.trim())
    .filter((x) => x);

  if (!changedFiles?.length) {
    console.log("Doing git fetch and getting merge-base");
    const mergeBase = execSync(
      `git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`,
      { cwd: PROJECT_ROOT }
    )
      .toString()
      .trim();

    console.log(`Merge base: ${mergeBase}`);

    const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim();

    changedFiles = changedFilesOutput
      .split("\n")
      .map((x) => x.trim())
      .filter((x) => x);

    console.log("Changed files (first 50):");
    console.log(changedFiles.slice(0, 50).join("\n"));
  }

  let filters: ((pipeline: EsPipeline) => boolean)[] = [
    (pipeline) => labelCheckAllow(pipeline, labels),
    (pipeline) => labelCheckSkip(pipeline, labels),
    (pipeline) => changedFilesExcludedCheck(pipeline, changedFiles),
    (pipeline) => changedFilesIncludedCheck(pipeline, changedFiles),
  ];

  // When triggering via the "run elasticsearch-ci/step-name" comment, we ONLY want to run pipelines that match the trigger phrase, regardless of labels, etc
  // However, if we're using the overall CI trigger "[buildkite] test this [please]", we should use the regular filters above
  if (
    process.env["GITHUB_PR_TRIGGER_COMMENT"] &&
    !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(
      /^\s*((@elastic(search)?machine|buildkite)\s*)?test\s+this(\s+please)?/i
    )
  ) {
    filters = [triggerCommentCheck];
  }

  for (const filter of filters) {
    pipelines = pipelines.filter(filter);
  }

  for (const pipeline of pipelines) {
    doBwcTransforms(pipeline);
  }

  pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));

  const finalPipelines = pipelines.map((pipeline) => {
    const finalPipeline = { name: pipeline.name, pipeline: { ...pipeline } };
    delete finalPipeline.pipeline.config;
    delete finalPipeline.pipeline.name;

    return finalPipeline;
  });

  return finalPipelines;
};