From d118cbf529ec4da9f9d7378cb48e4355ee61c9ab Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 18 Dec 2024 09:22:08 +0100 Subject: [PATCH] [Build] Make ThirdPartyAudit task uptodate more effective (#118723) We should basically be able to skip the third party dependency audit tasks if no third party dependency has changed. Filtering out the project dependencies allows way better uptodate and caching behaviour. We are only interested in thirdparty libs anyhow in this context. A positive side effect of the reduced class path is a quicker execution of the task --- .../ThirdPartyAuditPrecommitPlugin.java | 9 +++-- .../precommit/ThirdPartyAuditTask.java | 15 ++------ .../internal/util/DependenciesUtils.java | 36 ++++++++++++++++--- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index e45a1d3dd25b..7046a22204ef 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -16,12 +16,14 @@ import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.component.ModuleComponentIdentifier; +import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.TaskProvider; import java.io.File; import java.nio.file.Path; import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; +import static org.elasticsearch.gradle.internal.util.DependenciesUtils.thirdPartyDependenciesView; import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { @@ -47,7 +49,6 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, elasticsearchCoreProject); } } - TaskProvider resourcesTask = project.getTasks() .register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class); Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config"); @@ -59,9 +60,11 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { // usually only one task is created. but this construct makes our integTests easier to setup project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> { Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath"); + FileCollection runtimeThirdParty = thirdPartyDependenciesView(runtimeConfiguration); Configuration compileOnly = project.getConfigurations() .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); - t.setClasspath(runtimeConfiguration.plus(compileOnly)); + FileCollection compileOnlyThirdParty = thirdPartyDependenciesView(compileOnly); + t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty); t.getJarsToScan() .from( createFileCollectionFromNonTransitiveArtifactsView( @@ -78,7 +81,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath)); t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile()); t.getJdkJarHellClasspath().from(jdkJarHellConfig); - t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); + t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty)); }); return audit; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 442797775de2..59ba9bae0a57 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -17,7 +17,6 @@ import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; import org.gradle.api.file.ArchiveOperations; import org.gradle.api.file.ConfigurableFileCollection; -import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; import org.gradle.api.file.ProjectLayout; @@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { private final ProjectLayout projectLayout; - private FileCollection classpath; - @Inject public ThirdPartyAuditTask( ArchiveOperations archiveOperations, @@ -198,9 +195,7 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { public abstract Property getRuntimeJavaVersion(); @Classpath - public FileCollection getClasspath() { - return classpath; - } + public abstract ConfigurableFileCollection getThirdPartyClasspath(); @TaskAction public void runThirdPartyAudit() throws IOException { @@ -345,7 +340,7 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { if (javaHome.isPresent()) { spec.setExecutable(javaHome.get() + "/bin/java"); } - spec.classpath(getForbiddenAPIsClasspath(), classpath); + spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath()); // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module. if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) { spec.jvmArgs("--add-modules", "jdk.incubator.vector"); @@ -383,7 +378,7 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { private Set runJdkJarHellCheck() throws IOException { ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); ExecResult execResult = execOperations.javaexec(spec -> { - spec.classpath(getJdkJarHellClasspath(), classpath); + spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath()); spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS); spec.args(getJarExpandDir()); spec.setIgnoreExitValue(true); @@ -402,8 +397,4 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))); } - public void setClasspath(FileCollection classpath) { - this.classpath = classpath; - } - } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java index 9080f62f1993..5d7386e2c215 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java @@ -9,12 +9,16 @@ package org.elasticsearch.gradle.internal.util; +import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; + import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolvableDependencies; import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.component.ProjectComponentIdentifier; import org.gradle.api.artifacts.result.ResolvedComponentResult; import org.gradle.api.artifacts.result.ResolvedDependencyResult; import org.gradle.api.file.FileCollection; +import org.gradle.api.provider.Provider; import org.gradle.api.specs.AndSpec; import org.gradle.api.specs.Spec; @@ -29,7 +33,7 @@ public class DependenciesUtils { ) { ResolvableDependencies incoming = configuration.getIncoming(); return incoming.artifactView(viewConfiguration -> { - Set firstLevelDependencyComponents = incoming.getResolutionResult() + Provider> firstLevelDependencyComponents = incoming.getResolutionResult() .getRootComponent() .map( rootComponent -> rootComponent.getDependencies() @@ -39,12 +43,36 @@ public class DependenciesUtils { .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult) .map(dependency -> dependency.getSelected().getId()) .collect(Collectors.toSet()) - ) - .get(); + ); viewConfiguration.componentFilter( - new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter) + new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter) ); }).getFiles(); } + /** + * This method gives us an artifact view of a configuration that filters out all + * project dependencies that are not shadowed jars. + * Basically a thirdparty only view of the dependency tree. + */ + public static FileCollection thirdPartyDependenciesView(Configuration configuration) { + ResolvableDependencies incoming = configuration.getIncoming(); + return incoming.artifactView(v -> { + // resolve componentIdentifier for all shadowed project dependencies + Provider> shadowedDependencies = incoming.getResolutionResult() + .getRootComponent() + .map( + root -> root.getDependencies() + .stream() + .filter(dep -> dep instanceof ResolvedDependencyResult) + .map(dep -> (ResolvedDependencyResult) dep) + .filter(dep -> dep.getResolvedVariant().getDisplayName() == ShadowBasePlugin.COMPONENT_NAME) + .filter(dep -> dep.getSelected() instanceof ResolvedComponentResult) + .map(dep -> dep.getSelected().getId()) + .collect(Collectors.toSet()) + ); + // filter out project dependencies if they are not a shadowed dependency + v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false || shadowedDependencies.get().contains(i))); + }).getFiles(); + } }