From fad8c191fc92d680279a9f415dd25e66edc2b86b Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 6 Jun 2024 07:30:21 +0200 Subject: [PATCH] Make BWC build logic configuration cache compatible (#109354) More refactoring towards configuration cache compatibility --- .../main/groovy/elasticsearch.bwc-test.gradle | 7 +- .../gradle/internal/BwcSetupExtension.java | 72 ++++++++++++++----- .../gradle/internal/InternalBwcGitPlugin.java | 8 +-- .../InternalDistributionBwcSetupPlugin.java | 42 +++++++++-- .../org/elasticsearch/gradle/LoggedExec.java | 6 +- 5 files changed, 99 insertions(+), 36 deletions(-) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle index 5512b06d0ab8..ff9b6fe7a526 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle @@ -33,7 +33,8 @@ tasks.register("bwcTest") { plugins.withType(ElasticsearchTestBasePlugin) { tasks.withType(Test).matching { it.name ==~ /v[0-9\.]+#.*/ }.configureEach { - onlyIf("BWC tests enabled") { project.bwc_tests_enabled } + boolean bwcEnabled = project.bwc_tests_enabled + onlyIf("BWC tests enabled") { bwcEnabled } nonInputProperties.systemProperty 'tests.bwc', 'true' } } @@ -50,5 +51,5 @@ plugins.withType(InternalJavaRestTestPlugin) { } } -tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) } -tasks.matching { it.name.equals("test") }.configureEach {enabled = false} +tasks.matching { it.name.equals("check") }.configureEach { dependsOn(bwcTestSnapshots) } +tasks.matching { it.name.equals("test") }.configureEach { enabled = false } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 3d6d37575eca..7010ed92d4c5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -20,6 +20,9 @@ import org.gradle.api.logging.LogLevel; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.provider.ValueSource; +import org.gradle.api.provider.ValueSourceParameters; import org.gradle.api.tasks.TaskProvider; import org.gradle.jvm.toolchain.JavaLanguageVersion; import org.gradle.jvm.toolchain.JavaToolchainService; @@ -41,6 +44,7 @@ public class BwcSetupExtension { private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0"); private final Project project; private final ObjectFactory objectFactory; + private final ProviderFactory providerFactory; private final JavaToolchainService toolChainService; private final Provider unreleasedVersionInfo; @@ -49,12 +53,14 @@ public class BwcSetupExtension { public BwcSetupExtension( Project project, ObjectFactory objectFactory, + ProviderFactory providerFactory, JavaToolchainService toolChainService, Provider unreleasedVersionInfo, Provider checkoutDir ) { this.project = project; this.objectFactory = objectFactory; + this.providerFactory = providerFactory; this.toolChainService = toolChainService; this.unreleasedVersionInfo = unreleasedVersionInfo; this.checkoutDir = checkoutDir; @@ -65,11 +71,26 @@ public class BwcSetupExtension { } TaskProvider bwcTask(String name, Action configuration, boolean useUniqueUserHome) { - return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome); + return createRunBwcGradleTask( + project, + checkoutDir, + providerFactory, + unreleasedVersionInfo, + objectFactory, + toolChainService, + name, + configuration, + useUniqueUserHome + ); } - private TaskProvider createRunBwcGradleTask( + private static TaskProvider createRunBwcGradleTask( Project project, + Provider checkoutDir, + ProviderFactory providerFactory, + Provider unreleasedVersionInfo, + ObjectFactory objectFactory, + JavaToolchainService toolChainService, String name, Action configAction, boolean useUniqueUserHome @@ -78,10 +99,10 @@ public class BwcSetupExtension { loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.getWorkingDir().set(checkoutDir.get()); - loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> { - String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version()))); - return getJavaHome(Integer.parseInt(minimumCompilerVersion)); - })); + loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> { + spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); + spec.getParameters().getCheckoutDir().set(checkoutDir); + }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); if (BuildParams.isCi() && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted @@ -137,10 +158,13 @@ public class BwcSetupExtension { }); } - private String minimumCompilerVersionPath(Version bwcVersion) { - return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION)) - ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH - : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; + /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ + private static Provider getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) { + Property value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version)); + return toolChainService.launcherFor(javaToolchainSpec -> { + javaToolchainSpec.getLanguageVersion().value(value); + javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE); + }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } private static String readFromFile(File file) { @@ -151,13 +175,25 @@ public class BwcSetupExtension { } } - /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ - public String getJavaHome(final int version) { - Property value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version)); - return toolChainService.launcherFor(javaToolchainSpec -> { - javaToolchainSpec.getLanguageVersion().value(value); - javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE); - }).get().getMetadata().getInstallationPath().getAsFile().getAbsolutePath(); - } + public static abstract class JavaHomeValueSource implements ValueSource { + private String minimumCompilerVersionPath(Version bwcVersion) { + return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION)) + ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH + : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; + } + + @Override + public String obtain() { + return readFromFile( + new File(getParameters().getCheckoutDir().get(), minimumCompilerVersionPath(getParameters().getVersion().get())) + ); + } + + public interface Params extends ValueSourceParameters { + Property getVersion(); + + Property getCheckoutDir(); + } + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java index 71c76b204500..7add1e615f57 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java @@ -93,13 +93,6 @@ public class InternalBwcGitPlugin implements Plugin { String remoteRepo = remote.get(); // for testing only we can override the base remote url String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo") - .orElse( - providerFactory.provider( - () -> addRemote.getExtensions().getExtraProperties().has("remote") - ? addRemote.getExtensions().getExtraProperties().get("remote").toString() - : null - ) - ) .getOrElse("https://github.com/" + remoteRepo + "/" + rootProjectName); spec.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl); }); @@ -213,6 +206,7 @@ public class InternalBwcGitPlugin implements Plugin { private void writeFile(File file, String content) { try { + file.getParentFile().mkdirs(); Files.writeString(file.toPath(), content, CREATE, TRUNCATE_EXISTING); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index f727dc165a8a..a2247adcf7b9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -16,6 +16,7 @@ import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; +import org.gradle.api.file.ProjectLayout; import org.gradle.api.model.ObjectFactory; import org.gradle.api.plugins.JvmToolchainsPlugin; import org.gradle.api.provider.Provider; @@ -63,15 +64,39 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { project.getPlugins().apply(JvmToolchainsPlugin.class); toolChainService = project.getExtensions().getByType(JavaToolchainService.class); BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { - configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion); + configureBwcProject( + project.project(unreleasedVersion.gradleProjectPath()), + unreleasedVersion, + providerFactory, + objectFactory, + toolChainService + ); }); } - private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) { + private static void configureBwcProject( + Project project, + BwcVersions.UnreleasedVersionInfo versionInfo, + ProviderFactory providerFactory, + ObjectFactory objectFactory, + JavaToolchainService toolChainService + ) { + ProjectLayout layout = project.getLayout(); Provider versionInfoProvider = providerFactory.provider(() -> versionInfo); - Provider checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch())); + Provider checkoutDir = versionInfoProvider.map( + info -> new File(layout.getBuildDirectory().get().getAsFile(), "bwc/checkout-" + info.branch()) + ); BwcSetupExtension bwcSetupExtension = project.getExtensions() - .create("bwcSetup", BwcSetupExtension.class, project, objectFactory, toolChainService, versionInfoProvider, checkoutDir); + .create( + "bwcSetup", + BwcSetupExtension.class, + project, + objectFactory, + providerFactory, + toolChainService, + versionInfoProvider, + checkoutDir + ); BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension(); Provider bwcVersion = versionInfoProvider.map(info -> info.version()); gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version())); @@ -157,7 +182,7 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { } } - private void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) { + private static void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) { String projectName = distributionProject.name; String buildBwcTask = buildBwcTaskName(projectName); @@ -174,7 +199,11 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { } } - private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) { + private static void registerDistributionArchiveArtifact( + Project bwcProject, + DistributionProject distributionProject, + String buildBwcTask + ) { File distFile = distributionProject.expectedBuildArtifact.distFile; String artifactFileName = distFile.getName(); String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch"; @@ -363,5 +392,4 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { this.expandedDistDir = expandedDistDir; } } - } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 4fda91d33211..6087482db278 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -65,6 +65,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat @Optional abstract public MapProperty getEnvironment(); + @Internal + abstract public MapProperty getNonTrackedEnvironment(); + @Input abstract public Property getExecutable(); @@ -139,7 +142,8 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat execSpec.setStandardOutput(finalOutputStream); execSpec.setErrorOutput(finalOutputStream); execSpec.setExecutable(getExecutable().get()); - execSpec.setEnvironment(getEnvironment().get()); + execSpec.environment(getEnvironment().get()); + execSpec.environment(getNonTrackedEnvironment().get()); if (getArgs().isPresent()) { execSpec.setArgs(getArgs().get()); }