elasticsearch/build.gradle

/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the "Elastic License
 * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
 * Public License v 1"; you may not use this file except in compliance with, at
 * your election, the "Elastic License 2.0", the "GNU Affero General Public
 * License v3.0 only", or the "Server Side Public License, v 1".
 */


import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import com.avast.gradle.dockercompose.tasks.ComposePull
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper

import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin
import org.elasticsearch.gradle.internal.ResolveAllDependencies
import org.elasticsearch.gradle.util.GradleUtils
import org.gradle.plugins.ide.eclipse.model.AccessRule
import org.gradle.plugins.ide.eclipse.model.ProjectDependency
import org.elasticsearch.gradle.DistributionDownloadPlugin

import java.nio.file.Files

import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure

buildscript {
  repositories {
    mavenCentral()
  }
}

plugins {
  id 'lifecycle-base'
  id 'elasticsearch.docker-support'
  id 'elasticsearch.global-build-info'
  id 'elasticsearch.build-complete'
  id 'elasticsearch.build-scan'
  id 'elasticsearch.jdk-download'
  id 'elasticsearch.internal-distribution-download'
  id 'elasticsearch.runtime-jdk-provision'
  id 'elasticsearch.ide'
  id 'elasticsearch.forbidden-dependencies'
  id 'elasticsearch.local-distribution'
  id 'elasticsearch.fips'
  id 'elasticsearch.internal-testclusters'
  id 'elasticsearch.run'
  id 'elasticsearch.run-ccs'
  id 'elasticsearch.release-tools'
  id 'elasticsearch.versions'
}

/**
 * This is a convenient method for declaring test artifact dependencies provided by the internal
 * test artifact plugin. It replaces basically the longer dependency notation with explicit capability
 * declaration like this:
 *
 * testImplementation(project(xpackModule('repositories-metering-api'))) {
 *    capabilities {
 *         requireCapability("org.elasticsearch.gradle:repositories-metering-api-test-artifacts")
 *    }
 * }
 *
 * */
ext.testArtifact = { p, String name = "test" ->
  def projectDependency = p.dependencies.create(p)
  projectDependency.capabilities {
    requireCapabilities("org.elasticsearch.gradle:${projectDependency.name}-${name}-artifacts")
  };
}

class StepExpansion {
  String templatePath
  List<Version> versions
  String variable
}

class ListExpansion {
  List<Version> versions
  String variable
}

// Filters out intermediate patch releases to reduce the load of CI testing
def filterIntermediatePatches = { List<Version> versions ->
  versions.groupBy {"${it.major}.${it.minor}"}.values().collect {it.max()}
}

tasks.register("updateCIBwcVersions") {
  def writeVersions = { File file, List<Version> versions ->
    file.text = ""
    file << "BWC_VERSION:\n"
    versions.each {
      file << "  - \"$it\"\n"
    }
  }

  def writeBuildkitePipeline = { String outputFilePath, String pipelineTemplatePath, List<ListExpansion> listExpansions, List<StepExpansion> stepExpansions = [] ->
    def outputFile = file(outputFilePath)
    def pipelineTemplate = file(pipelineTemplatePath)

    def pipeline = pipelineTemplate.text

    listExpansions.each { expansion ->
      def listString = "[" + expansion.versions.collect { "\"${it}\"" }.join(", ") + "]"
      pipeline = pipeline.replaceAll('\\$' + expansion.variable, listString)
    }

    stepExpansions.each { expansion ->
      def steps = ""
      expansion.versions.each {
        steps += "\n" + file(expansion.templatePath).text.replaceAll('\\$BWC_VERSION', it.toString())
      }
      pipeline = pipeline.replaceAll(' *\\$' + expansion.variable, steps)
    }

    outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline
  }

  // Writes a Buildkite pipelime from a template, and replaces a variable with an array of versions
  // Useful for writing a list of versions in a matrix configuration
  def expandList = { String outputFilePath, String pipelineTemplatePath, String variable, List<Version> versions ->
    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: variable)])
  }

  // Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version
  // Useful when you need to configure more versions than are allowed in a matrix configuration
  def expandBwcSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [], [new StepExpansion(templatePath: stepTemplatePath, versions: versions, variable: "BWC_STEPS")])
  }

  doLast {
    writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible))
    writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible))
    expandList(
      ".buildkite/pipelines/intake.yml",
      ".buildkite/pipelines/intake.template.yml",
      "BWC_LIST",
      filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)
    )
    expandList(
      ".buildkite/pipelines/periodic-fwc.yml",
      ".buildkite/pipelines/periodic-fwc.template.yml",
      "FWC_LIST",
      buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor }
    )
    writeBuildkitePipeline(
      ".buildkite/pipelines/periodic.yml",
      ".buildkite/pipelines/periodic.template.yml",
      [
        new ListExpansion(versions: filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"),
      ],
      [
        new StepExpansion(
          templatePath: ".buildkite/pipelines/periodic.bwc.template.yml",
          versions: filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible),
          variable: "BWC_STEPS"
        ),
      ]
    )

    expandBwcSteps(
      ".buildkite/pipelines/periodic-packaging.yml",
      ".buildkite/pipelines/periodic-packaging.template.yml",
      ".buildkite/pipelines/periodic-packaging.bwc.template.yml",
      filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible)
    )
  }
}

tasks.register("verifyVersions") {
  def verifyCiYaml = { File file, List<Version> versions ->
    String ciYml = file.text
    versions.each {
      if (ciYml.contains("\"$it\"\n") == false) {
        throw new Exception("${file} is outdated, run `./gradlew updateCIBwcVersions` and check in the results")
      }
    }
  }
  doLast {
    if (gradle.startParameter.isOffline()) {
      throw new GradleException("Must run in online mode to verify versions")
    }
    // Read the list from maven central.
    // Fetch the metadata and parse the xml into Version instances because it's more straight forward here
    // rather than bwcVersion ( VersionCollection ).
    new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
      buildParams.bwcVersions.compareToAuthoritative(
        new XmlParser().parse(s)
          .versioning.versions.version
          .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
          .collect { Version.fromString(it) }
      )
    }
    verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible))
    verifyCiYaml(file(".ci/snapshotBwcVersions"), buildParams.bwcVersions.unreleasedIndexCompatible)

    // Make sure backport bot config file is up to date
    JsonNode backportConfig = new ObjectMapper().readTree(file(".backportrc.json"))
    buildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion ->
      boolean valid = backportConfig.get("targetBranchChoices").elements().any { branchChoice ->
        if (branchChoice.isObject()) {
          return branchChoice.get("name").textValue() == unreleasedVersion.branch
        } else {
          return branchChoice.textValue() == unreleasedVersion.branch
        }
      }
      if (valid == false) {
        throw new GradleException("No branch choice exists for development branch ${unreleasedVersion.branch} in .backportrc.json.")
      }
    }
    String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == '8.x' }.key
    String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
    if (versionMapping != expectedMapping) {
      throw new GradleException(
        "Backport label mapping for branch '8.x' is '${versionMapping}' but should be " +
          "'${expectedMapping}'. Update .backportrc.json."
      )
    }
  }
}

def generateUpgradeCompatibilityFile = tasks.register("generateUpgradeCompatibilityFile") {
  def outputFile = project.layout.buildDirectory.file("rolling-upgrade-compatible-${VersionProperties.elasticsearch}.json")
  def rollingUpgradeCompatibleVersions = buildParams.bwcVersions.wireCompatible - VersionProperties.elasticsearchVersion
  inputs.property("rollingUpgradeCompatibleVersions", rollingUpgradeCompatibleVersions)
  outputs.file(outputFile)
  doLast {
    def versionsString = rollingUpgradeCompatibleVersions.collect { "\"${it.toString()}\"" }.join(', ')
    outputFile.get().asFile.write("""{"rolling_upgrade_compatible_versions" : [${versionsString}]}""")
  }
}

def upgradeCompatibilityZip = tasks.register("upgradeCompatibilityZip", Zip) {
  archiveFile.set(project.layout.buildDirectory.file("distributions/rolling-upgrade-compatible-${VersionProperties.elasticsearch}.zip"))
  from(generateUpgradeCompatibilityFile)
}

/*
 * When adding backcompat behavior that spans major versions, temporarily
 * disabling the backcompat tests is necessary. This flag controls
 * the enabled state of every bwc task. It should be set back to true
 * after the backport of the backcompat code is complete.
 */

boolean bwc_tests_enabled = true
// place a PR link here when committing bwc changes:
String bwc_tests_disabled_issue = ""
if (bwc_tests_enabled == false) {
  if (bwc_tests_disabled_issue.isEmpty()) {
    throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
  }
  println "========================= WARNING ========================="
  println "         Backwards compatibility tests are disabled!"
  println "See ${bwc_tests_disabled_issue}"
  println "==========================================================="
}
if (project.gradle.startParameter.taskNames.any { it.startsWith("checkPart") || it == 'functionalTests' }) {
  // Disable BWC tests for checkPart* tasks and platform support tests as it's expected that this will run on it's own check
  bwc_tests_enabled = false
}

subprojects { proj ->
  apply plugin: 'elasticsearch.base'
}

allprojects {
  // We disable this plugin for now till we shaked out the issues we see
  // e.g. see https://github.com/elastic/elasticsearch/issues/72169
  // apply plugin:'elasticsearch.internal-test-rerun'

  plugins.withType(BaseInternalPluginBuildPlugin).whenPluginAdded {
    project.dependencies {
      compileOnly project(":server")
      testImplementation project(":test:framework")
    }
  }

  ext.bwc_tests_enabled = bwc_tests_enabled

  // eclipse configuration
  apply plugin: 'elasticsearch.eclipse'

  /*
   * Allow accessing com/sun/net/httpserver in projects that have
   * configured forbidden apis to allow it.
   */
  plugins.withType(ForbiddenApisPlugin) {
    eclipse.classpath.file.whenMerged { classpath ->
      if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
        classpath.entries
          .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
          .each {
            it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
          }
      }
    }
  }

  tasks.register('resolveAllDependencies', ResolveAllDependencies) {
    def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
    configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
    resolveJavaToolChain = true
    if (project.path.contains("fixture")) {
      dependsOn tasks.withType(ComposePull)
    }
    if (project.path.contains(":distribution:docker")) {
      enabled = false
    }
    if (project.path.contains(":libs:cli")) {
      // ensure we resolve p2 dependencies for the spotless eclipse formatter
      dependsOn "spotlessJavaCheck"
    }
  }

  ext.withReleaseBuild = { Closure config ->
    if(buildParams.snapshotBuild == false) {
      config.call()
    }
  }

  plugins.withId('lifecycle-base') {
    if (project.path.startsWith(":x-pack:")) {
      if (project.path.contains("security") || project.path.contains(":ml")) {
        tasks.register('checkPart4') {
          dependsOn 'check'
          withReleaseBuild {
            dependsOn 'assemble'
          }
        }
      } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
        tasks.register('checkPart3') {
          dependsOn 'check'
          withReleaseBuild {
            dependsOn 'assemble'
          }
        }
      } else if (project.path.contains("multi-node")) {
        tasks.register('checkPart5') {
          dependsOn 'check'
          withReleaseBuild {
            dependsOn 'assemble'
          }
        }
      } else {
        tasks.register('checkPart2') {
          dependsOn 'check'
          withReleaseBuild {
            dependsOn 'assemble'
          }
        }
      }
    } else {
      tasks.register('checkPart1') {
        dependsOn 'check'
        withReleaseBuild {
          dependsOn 'assemble'
        }
      }
    }
    tasks.register('functionalTests') {
      dependsOn 'check'
      withReleaseBuild {
        dependsOn 'assemble'
      }
    }
  }

  /*
   * Remove assemble/dependenciesInfo on all qa projects because we don't
   * need to publish artifacts for them.
   */
  if (project.name.equals('qa') || project.path.contains(':qa:')) {
    maybeConfigure(project.tasks, 'assemble') {
      it.enabled = false
    }
    maybeConfigure(project.tasks, 'dependenciesInfo') {
      it.enabled = false
    }
  }

  project.afterEvaluate {
    // Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
    // important because, while dependencies.all will pickup future dependencies,
    // it is not necessarily true that the task exists in both projects at the time
    // the dependency is added.
    if (project.path == ':test:framework') {
      // :test:framework:test cannot run before and after :server:test
      return
    }
    tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
      integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
    }

/*    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
      dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
        Project upstreamProject = dep.dependencyProject
        if (project.path != upstreamProject?.path) {
          for (String taskName : ['test', 'integTest']) {
            project.tasks.matching { it.name == taskName }.configureEach { task ->
              task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
            }
          }
        }
      }
    }*/
  }

  apply plugin: 'elasticsearch.formatting'
}

tasks.named("updateDaemonJvm") {
  def myPlatforms = [
    BuildPlatformFactory.of(
      org.gradle.platform.Architecture.AARCH64,
      org.gradle.platform.OperatingSystem.MAC_OS
    ),
    BuildPlatformFactory.of(
      org.gradle.platform.Architecture.AARCH64,
      org.gradle.platform.OperatingSystem.LINUX
    ),
    BuildPlatformFactory.of(
      org.gradle.platform.Architecture.X86_64,
      org.gradle.platform.OperatingSystem.LINUX
    ),
    BuildPlatformFactory.of(
      org.gradle.platform.Architecture.X86_64,
      org.gradle.platform.OperatingSystem.WINDOWS
    ),
    // anyone still using x86 osx?
    BuildPlatformFactory.of(
      org.gradle.platform.Architecture.X86_64,
      org.gradle.platform.OperatingSystem.MAC_OS
    )
  ]
  toolchainPlatforms.set(myPlatforms)
  languageVersion = JavaLanguageVersion.of(21)
  vendor = JvmVendorSpec.ADOPTIUM
}

tasks.register("verifyBwcTestsEnabled") {
  doLast {
    if (bwc_tests_enabled == false) {
      throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
    }
  }
}

tasks.register("branchConsistency") {
  description = 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
  group = 'Verification'
  dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
}

tasks.named("wrapper").configure {
  distributionType = 'ALL'
  def minimumGradleVersionFile = project.file('build-tools-internal/src/main/resources/minimumGradleVersion')
  doLast {
    // copy wrapper properties file to build-tools-internal to allow seamless idea integration
    def file = new File("build-tools-internal/gradle/wrapper/gradle-wrapper.properties")
    Files.copy(getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING)
    // copy wrapper properties file to plugins/examples to allow seamless idea integration
    def examplePluginsWrapperProperties = new File("plugins/examples/gradle/wrapper/gradle-wrapper.properties")
    Files.copy(getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING)
    // Update build-tools to reflect the Gradle upgrade
    // TODO: we can remove this once we have tests to make sure older versions work.
    minimumGradleVersionFile.text = gradleVersion
    println "Updated minimum Gradle Version"
  }
}

gradle.projectsEvaluated {
  // Having the same group and name for distinct projects causes Gradle to consider them equal when resolving
  // dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening.
  // see: https://github.com/gradle/gradle/issues/847
  Map coordsToProject = [:]
  project.allprojects.forEach { p ->
    String coords = "${p.group}:${p.name}"
    if (false == coordsToProject.putIfAbsent(coords, p)) {
      throw new GradleException(
        "Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " +
          "have the same name and group: ${coords}. " +
          "This doesn't currently work correctly in Gradle, see: " +
          "https://github.com/gradle/gradle/issues/847"
      )
    }
  }
}

tasks.named("validateChangelogs").configure {
  def triggeredTaskNames = gradle.startParameter.taskNames
  onlyIf {
    triggeredTaskNames.any { it.startsWith("checkPart") || it == 'functionalTests' } == false
  }
}

tasks.named("precommit") {
  dependsOn gradle.includedBuild('build-tools').task(':precommit')
  dependsOn gradle.includedBuild('build-tools-internal').task(':precommit')
}

tasks.named("checkPart1").configure {
  dependsOn gradle.includedBuild('build-tools').task(':check')
  dependsOn gradle.includedBuild('build-tools-internal').task(':check')
}

tasks.named("assemble").configure {
  dependsOn gradle.includedBuild('build-tools').task(':assemble')
}

tasks.named("cleanEclipse").configure {
  dependsOn gradle.includedBuild('build-conventions').task(':cleanEclipse')
  dependsOn gradle.includedBuild('build-tools').task(':cleanEclipse')
  dependsOn gradle.includedBuild('build-tools-internal').task(':cleanEclipse')
}

tasks.named("eclipse").configure {
  dependsOn gradle.includedBuild('build-conventions').task(':eclipse')
  dependsOn gradle.includedBuild('build-tools').task(':eclipse')
  dependsOn gradle.includedBuild('build-tools-internal').task(':eclipse')
}

tasks.register("buildReleaseArtifacts").configure {
  group = 'build'
  description = 'Builds all artifacts required for release manager'

  dependsOn allprojects.findAll {
    it.path.startsWith(':distribution:docker') == false
      && it.path.startsWith(':ml-cpp') == false
      && it.path.startsWith(':distribution:bwc') == false
      && it.path.startsWith(':test:fixture') == false
  }
    .collect { GradleUtils.findByName(it.tasks, 'assemble') }
    .findAll { it != null }
  dependsOn upgradeCompatibilityZip
}

tasks.register("spotlessApply").configure {
  dependsOn gradle.includedBuild('build-tools').task(':spotlessApply')
  dependsOn gradle.includedBuild('build-tools').task(':reaper:spotlessApply')
  dependsOn gradle.includedBuild('build-tools-internal').task(':spotlessApply')
}