diff --git a/build-tools/build.gradle b/build-tools/build.gradle index e457999fedfe..9d9ec139b175 100644 --- a/build-tools/build.gradle +++ b/build-tools/build.gradle @@ -51,6 +51,10 @@ gradlePlugin { id = 'elasticsearch.stable-esplugin' implementationClass = 'org.elasticsearch.gradle.plugin.StablePluginBuildPlugin' } + testBuildInfo { + id = 'elasticsearch.test-build-info' + implementationClass = 'org.elasticsearch.gradle.test.TestBuildInfoPlugin' + } javaRestTest { id = 'elasticsearch.java-rest-test' implementationClass = 'org.elasticsearch.gradle.test.JavaRestTestPlugin' diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy new file mode 100644 index 000000000000..a12e6aba7b6f --- /dev/null +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy @@ -0,0 +1,63 @@ +package org.elasticsearch.gradle.test + +import com.fasterxml.jackson.databind.ObjectMapper + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome + +class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest { + def "works"() { + given: + file("src/main/java/com/example/Example.java") << """ + package com.example; + + public class Example { + } + """ + + file("src/main/java/module-info.java") << """ + module com.example { + exports com.example; + } + """ + + buildFile << """ + import org.elasticsearch.gradle.plugin.GenerateTestBuildInfoTask; + + plugins { + id 'java' + id 'elasticsearch.test-build-info' + } + + repositories { + mavenCentral() + } + + tasks.withType(GenerateTestBuildInfoTask.class) { + componentName = 'example-component' + outputFile = new File('build/generated-build-info/plugin-test-build-info.json') + } + """ + + when: + def result = gradleRunner('generateTestBuildInfo').build() + def task = result.task(":generateTestBuildInfo") + + + then: + task.outcome == TaskOutcome.SUCCESS + + def output = file("build/generated-build-info/plugin-test-build-info.json") + output.exists() == true + + def location = Map.of( + "module", "com.example", + "representative_class", "com/example/Example.class" + ) + def expectedOutput = Map.of( + "component", "example-component", + "locations", List.of(location) + ) + new ObjectMapper().readValue(output, Map.class) == expectedOutput + } +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java index 9e20ce64ed88..e00a4b45f8a6 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java @@ -121,7 +121,10 @@ public class BasePluginBuildPlugin implements Plugin { task.getIsLicensed().set(providerFactory.provider(extension::isLicensed)); var mainSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName(SourceSet.MAIN_SOURCE_SET_NAME); - FileCollection moduleInfoFile = mainSourceSet.getOutput().getAsFileTree().matching(p -> p.include("module-info.class")); + FileCollection moduleInfoFile = mainSourceSet.getOutput() + .getClassesDirs() + .getAsFileTree() + .matching(p -> p.include("module-info.class")); task.getModuleInfoFile().setFrom(moduleInfoFile); }); diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateTestBuildInfoTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateTestBuildInfoTask.java new file mode 100644 index 000000000000..c0f9eda18b21 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateTestBuildInfoTask.java @@ -0,0 +1,351 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.plugin; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.SerializationFeature; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.jetbrains.annotations.NotNull; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ModuleVisitor; +import org.objectweb.asm.Opcodes; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.CodeSource; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; + +import static java.nio.file.FileVisitResult.CONTINUE; +import static java.nio.file.FileVisitResult.TERMINATE; + +/** + * This task generates a file with a class to module mapping + * used to imitate modular behavior during unit tests so + * entitlements can lookup correct policies. + */ +@CacheableTask +public abstract class GenerateTestBuildInfoTask extends DefaultTask { + + public static final String DESCRIPTION = "generates plugin test dependencies file"; + + public static final String META_INF_VERSIONS_PREFIX = "META-INF/versions/"; + public static final String JAR_DESCRIPTOR_SUFFIX = ".jar"; + + public GenerateTestBuildInfoTask() { + setDescription(DESCRIPTION); + } + + @Input + @Optional + public abstract Property getModuleName(); + + @Input + public abstract Property getComponentName(); + + @Classpath + public abstract Property getCodeLocations(); + + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @TaskAction + public void generatePropertiesFile() throws IOException { + Path outputFile = getOutputFile().get().getAsFile().toPath(); + Files.createDirectories(outputFile.getParent()); + + try (var writer = Files.newBufferedWriter(outputFile, StandardCharsets.UTF_8)) { + ObjectMapper mapper = new ObjectMapper().configure(SerializationFeature.INDENT_OUTPUT, true) + .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); + mapper.writeValue(writer, new OutputFileContents(getComponentName().get(), buildLocationList())); + } + } + + /** + * The output of this task is a JSON file formatted according to this record. + * @param component the entitlements component name of the artifact we're describing + * @param locations a {@link Location} for each code directory/jar in this artifact + */ + record OutputFileContents(String component, List locations) {} + + /** + * Our analog of a single {@link CodeSource#getLocation()}. + * All classes in any single location (a directory or jar) + * are considered to be part of the same Java module for entitlements purposes. + * Since tests run without Java modules, and entitlements are all predicated on modules, + * this info lets us determine what the module would have been + * so we can look up the appropriate entitlements. + * + * @param module the name of the Java module corresponding to this {@code Location}. + * @param representativeClass an example of any .class file within this {@code Location} + * whose name will be unique within its {@link ClassLoader} at run time. + */ + record Location(String module, String representativeClass) {} + + /** + * Build the list of {@link Location}s for all {@link #getCodeLocations() code locations}. + * There are different methods for finding these depending on if the + * classpath entry is a jar or a directory + */ + private List buildLocationList() throws IOException { + List locations = new ArrayList<>(); + for (File file : getCodeLocations().get().getFiles()) { + if (file.exists()) { + if (file.getName().endsWith(JAR_DESCRIPTOR_SUFFIX)) { + extractLocationsFromJar(file, locations); + } else if (file.isDirectory()) { + extractLocationsFromDirectory(file, locations); + } else { + throw new IllegalArgumentException("unrecognized classpath entry: " + file); + } + } + } + return List.copyOf(locations); + } + + /** + * find the first class and module when the class path entry is a jar + */ + private void extractLocationsFromJar(File file, List locations) throws IOException { + try (JarFile jarFile = new JarFile(file)) { + var className = extractClassNameFromJar(jarFile); + + if (className.isPresent()) { + String moduleName = extractModuleNameFromJar(file, jarFile); + locations.add(new Location(moduleName, className.get())); + } + } + } + + /** + * look through the jar to find the first unique class that isn't + * in META-INF (those may not be unique) and isn't module-info.class + * (which is also not unique) and avoid anonymous classes + */ + private java.util.Optional extractClassNameFromJar(JarFile jarFile) { + return jarFile.stream() + .filter( + je -> je.getName().startsWith("META-INF") == false + && je.getName().equals("module-info.class") == false + && je.getName().contains("$") == false + && je.getName().endsWith(".class") + ) + .findFirst() + .map(ZipEntry::getName); + } + + /** + * Look through the jar for the module name using a succession of techniques corresponding + * to how the JDK itself determines module names, + * as documented in {@link java.lang.module.ModuleFinder#of}. + */ + private String extractModuleNameFromJar(File file, JarFile jarFile) throws IOException { + String moduleName = null; + + if (jarFile.isMultiRelease()) { + StringBuilder dir = versionDirectoryIfExists(jarFile); + if (dir != null) { + dir.append("/module-info.class"); + moduleName = getModuleNameFromModuleInfoFile(dir.toString(), jarFile); + } + } + + if (moduleName == null) { + moduleName = getModuleNameFromModuleInfoFile("module-info.class", jarFile); + } + + if (moduleName == null) { + moduleName = getAutomaticModuleNameFromManifest(jarFile); + } + + if (moduleName == null) { + moduleName = deriveModuleNameFromJarFileName(file); + } + + return moduleName; + } + + /** + * if the jar is multi-release, there will be a set versions + * under the path META-INF/versions/; + * each version will have its own module-info.class if this is a modular jar; + * look for the module name in the module-info from the latest version + * fewer than or equal to the current JVM version + * + * @return a {@link StringBuilder} with the {@code META-INF/versions/} if it exists; otherwise null + */ + private static StringBuilder versionDirectoryIfExists(JarFile jarFile) { + List versions = jarFile.stream() + .filter(je -> je.getName().startsWith(META_INF_VERSIONS_PREFIX) && je.getName().endsWith("/module-info.class")) + .map( + je -> Integer.parseInt( + je.getName().substring(META_INF_VERSIONS_PREFIX.length(), je.getName().length() - META_INF_VERSIONS_PREFIX.length()) + ) + ) + .toList(); + versions = new ArrayList<>(versions); + versions.sort(Integer::compareTo); + versions = versions.reversed(); + int major = Runtime.version().feature(); + StringBuilder path = new StringBuilder(META_INF_VERSIONS_PREFIX); + for (int version : versions) { + if (version <= major) { + return path.append(version); + } + } + return null; + } + + /** + * Looks into the specified {@code module-info.class} file, if it exists, and extracts the declared name of the module. + * @return the module name, or null if there is no such {@code module-info.class} file. + */ + private String getModuleNameFromModuleInfoFile(String moduleInfoFileName, JarFile jarFile) throws IOException { + JarEntry moduleEntry = jarFile.getJarEntry(moduleInfoFileName); + if (moduleEntry != null) { + try (InputStream inputStream = jarFile.getInputStream(moduleEntry)) { + return extractModuleNameFromModuleInfo(inputStream); + } + } + return null; + } + + /** + * Looks into the {@code MANIFEST.MF} file and returns the {@code Automatic-Module-Name} value if there is one. + * @return the module name, or null if the manifest is nonexistent or has no {@code Automatic-Module-Name} value + */ + private static String getAutomaticModuleNameFromManifest(JarFile jarFile) throws IOException { + JarEntry manifestEntry = jarFile.getJarEntry("META-INF/MANIFEST.MF"); + if (manifestEntry != null) { + try (InputStream inputStream = jarFile.getInputStream(manifestEntry)) { + Manifest manifest = new Manifest(inputStream); + String amn = manifest.getMainAttributes().getValue("Automatic-Module-Name"); + if (amn != null) { + return amn; + } + } + } + return null; + } + + /** + * Compose a module name from the given {@code jarFile} name, + * as documented in {@link java.lang.module.ModuleFinder#of}. + */ + private static @NotNull String deriveModuleNameFromJarFileName(File jarFile) { + String jn = jarFile.getName().substring(0, jarFile.getName().length() - JAR_DESCRIPTOR_SUFFIX.length()); + Matcher matcher = Pattern.compile("-(\\d+(\\.|$))").matcher(jn); + if (matcher.find()) { + jn = jn.substring(0, matcher.start()); + } + jn = jn.replaceAll("[^A-Za-z0-9]", "."); + return jn; + } + + /** + * find the first class and module when the class path entry is a directory + */ + private void extractLocationsFromDirectory(File dir, List locations) throws IOException { + String className = extractClassNameFromDirectory(dir); + String moduleName = extractModuleNameFromDirectory(dir); + + if (className != null && moduleName != null) { + locations.add(new Location(moduleName, className)); + } + } + + /** + * look through the directory to find the first unique class that isn't + * module-info.class (which may not be unique) and avoid anonymous classes + */ + private String extractClassNameFromDirectory(File dir) throws IOException { + var visitor = new SimpleFileVisitor() { + String result = null; + + @Override + public @NotNull FileVisitResult visitFile(@NotNull Path candidate, @NotNull BasicFileAttributes attrs) { + String name = candidate.getFileName().toString(); // Just the part after the last dir separator + if (name.endsWith(".class") && (name.equals("module-info.class") || name.contains("$")) == false) { + result = candidate.toAbsolutePath().toString().substring(dir.getAbsolutePath().length() + 1); + return TERMINATE; + } else { + return CONTINUE; + } + } + }; + Files.walkFileTree(dir.toPath(), visitor); + return visitor.result; + } + + /** + * look through the directory to find the module name in either module-info.class + * if it exists or the preset one derived from the jar task + */ + private String extractModuleNameFromDirectory(File dir) throws IOException { + List files = new ArrayList<>(List.of(dir)); + while (files.isEmpty() == false) { + File find = files.removeFirst(); + if (find.exists()) { + if (find.getName().equals("module-info.class")) { + try (InputStream inputStream = new FileInputStream(find)) { + return extractModuleNameFromModuleInfo(inputStream); + } + } else if (find.isDirectory()) { + files.addAll(Arrays.asList(find.listFiles())); + } + } + } + return getModuleName().getOrNull(); + } + + /** + * a helper method to extract the module name from module-info.class + * using an ASM ClassVisitor + */ + private String extractModuleNameFromModuleInfo(InputStream inputStream) throws IOException { + String[] moduleName = new String[1]; + ClassReader cr = new ClassReader(inputStream); + cr.accept(new ClassVisitor(Opcodes.ASM9) { + @Override + public ModuleVisitor visitModule(String name, int access, String version) { + moduleName[0] = name; + return super.visitModule(name, access, version); + } + }, Opcodes.ASM9); + return moduleName[0]; + } +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java index d5fec104c063..667bed768854 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -10,11 +10,16 @@ package org.elasticsearch.gradle.plugin; import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.test.TestBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.file.RegularFile; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; +import org.gradle.jvm.tasks.Jar; +import org.gradle.language.jvm.tasks.ProcessResources; + +import java.util.concurrent.Callable; import javax.inject.Inject; @@ -33,6 +38,7 @@ public class PluginBuildPlugin implements Plugin { @Override public void apply(final Project project) { project.getPluginManager().apply(BasePluginBuildPlugin.class); + project.getPluginManager().apply(TestBuildInfoPlugin.class); var dependencies = project.getDependencies(); dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch()); @@ -51,6 +57,30 @@ public class PluginBuildPlugin implements Plugin { task.getOutputFile().set(file); }); - } + project.getTasks().withType(GenerateTestBuildInfoTask.class).named("generateTestBuildInfo").configure(task -> { + var jarTask = project.getTasks().withType(Jar.class).named("jar").get(); + String moduleName = (String) jarTask.getManifest().getAttributes().get("Automatic-Module-Name"); + if (moduleName == null) { + moduleName = jarTask.getArchiveBaseName().getOrNull(); + } + if (moduleName != null) { + task.getModuleName().set(moduleName); + } + var propertiesExtension = project.getExtensions().getByType(PluginPropertiesExtension.class); + task.getComponentName().set(providerFactory.provider(propertiesExtension::getName)); + task.getOutputFile().set(project.getLayout().getBuildDirectory().file("generated-build-info/plugin-test-build-info.json")); + }); + project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> { + task.into( + (Callable) () -> "META-INF/es-plugins/" + + project.getExtensions().getByType(PluginPropertiesExtension.class).getName() + + "/", + copy -> { + copy.from(project.getTasks().withType(GeneratePluginPropertiesTask.class).named("pluginProperties")); + copy.from(project.getLayout().getProjectDirectory().file("src/main/plugin-metadata/entitlement-policy.yaml")); + } + ); + }); + } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java new file mode 100644 index 000000000000..3cab57a333d2 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.test; + +import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; +import org.elasticsearch.gradle.plugin.GenerateTestBuildInfoTask; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileCollection; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.language.jvm.tasks.ProcessResources; + +import javax.inject.Inject; + +/** + * This plugin configures the {@link GenerateTestBuildInfoTask} task + * with customizations for component name and output file name coming + * from the source using the plugin (server or ES plugin). + */ +public class TestBuildInfoPlugin implements Plugin { + + protected final ProviderFactory providerFactory; + + @Inject + public TestBuildInfoPlugin(ProviderFactory providerFactory) { + this.providerFactory = providerFactory; + } + + @Override + public void apply(Project project) { + var testBuildInfoTask = project.getTasks().register("generateTestBuildInfo", GenerateTestBuildInfoTask.class, task -> { + FileCollection codeLocations = project.getConfigurations().getByName("runtimeClasspath"); + Configuration compileOnly = project.getConfigurations() + .findByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); + if (compileOnly != null) { + codeLocations = codeLocations.minus(compileOnly); + } + var sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + codeLocations = codeLocations.plus(sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput().getClassesDirs()); + task.getCodeLocations().set(codeLocations); + }); + + project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> { + task.into("META-INF", copy -> copy.from(testBuildInfoTask)); + }); + } +} diff --git a/server/build.gradle b/server/build.gradle index 1b86cd639e4a..be2b43745d0b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -11,6 +11,7 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.test-build-info' publishing { publications { @@ -283,3 +284,10 @@ tasks.named("licenseHeaders").configure { } tasks.withType(Checkstyle.class).configureEach { t -> t.getMaxHeapSize().set("2g") } + +tasks.named("generateTestBuildInfo").configure { + t -> { + t.getComponentName().set("server") + t.getOutputFile().set(project.getLayout().getBuildDirectory().file("generated-build-info/server-test-build-info.json")) + } +}