From 1aa72377819b7e6b73068d936c95fa7d7afe084c Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 20 Dec 2022 18:43:48 +0100 Subject: [PATCH] [7.x] Resolve artifacts from dra (#92478) * WIP - Dra plugin * Fix pattern for non classifier dependencies in dra artifacts * Cleanup ml build script * Fix spotless --- build-tools-internal/build.gradle | 4 + .../fixtures/LocalRepositoryFixture.groovy | 27 ++-- .../dra/DraResolvePluginFuncTest.groovy | 130 ++++++++++++++++++ .../gradle/internal/dra/DraResolvePlugin.java | 101 ++++++++++++++ x-pack/plugin/ml/build.gradle | 54 ++++---- 5 files changed, 279 insertions(+), 37 deletions(-) create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 21c339ad2a28..2bd3cc8c9da7 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -52,6 +52,10 @@ gradlePlugin { id = 'elasticsearch.docs-test' implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin' } + draArtifacts { + id = 'elasticsearch.dra-artifacts' + implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin' + } globalBuildInfo { id = 'elasticsearch.global-build-info' implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy index 823c9e4c878b..387a80a73737 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy @@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource { private TemporaryFolder temporaryFolder - LocalRepositoryFixture(){ + LocalRepositoryFixture() { this.temporaryFolder = new TemporaryFolder() } @@ -33,22 +33,27 @@ class LocalRepositoryFixture extends ExternalResource { temporaryFolder.after() } - void generateJar(String group, String module, String version, String... clazzNames){ + void generateJar(String group, String module, String version, String... clazzNames) { def baseGroupFolderPath = group.replace('.', '/') def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version") targetFolder.mkdirs() def jarFile = new File(targetFolder, "${module}-${version}.jar") - clazzNames.each {clazzName -> - DynamicType.Unloaded dynamicType = new ByteBuddy().subclass(Object.class) - .name(clazzName) - .make() - if(jarFile.exists()) { - dynamicType.inject(jarFile); - }else { - dynamicType.toJar(jarFile); + if (clazzNames.size() == 0) { + jarFile.write("blubb") + } else { + clazzNames.each { clazzName -> + DynamicType.Unloaded dynamicType = new ByteBuddy().subclass(Object.class) + .name(clazzName) + .make() + if (jarFile.exists()) { + dynamicType.inject(jarFile); + } else { + dynamicType.toJar(jarFile); + } } } + } void configureBuild(File buildFile) { @@ -68,4 +73,4 @@ class LocalRepositoryFixture extends ExternalResource { File getRepoDir() { new File(temporaryFolder.root, 'local-repo') } -} +} \ No newline at end of file diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy new file mode 100644 index 000000000000..77a0a8e053a4 --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.dra + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture +import org.elasticsearch.gradle.fixtures.WiremockFixture +import org.gradle.testkit.runner.TaskOutcome +import org.junit.ClassRule +import spock.lang.Shared + +class DraResolvePluginFuncTest extends AbstractGradleFuncTest { + + @Shared + @ClassRule + public LocalRepositoryFixture repository = new LocalRepositoryFixture() + + def setup() { + configurationCacheCompatible = false + + buildFile << """ + plugins { + id 'elasticsearch.dra-artifacts' + } + + repositories.all { + // for supporting http testing repos here + allowInsecureProtocol = true + } + """ + } + + def "provides flag indicating dra usage"() { + setup: + repository.generateJar("org.acme", "ml-cpp", "8.6.0-SNAPSHOT") + buildFile << """ + if(useDra == false) { + repositories { + maven { + name = "local-test" + url = "${repository.getRepoDir().toURI()}" + metadataSources { + artifact() + } + } + } + } + """ + + buildFile << """ + configurations { + someConfig + } + + dependencies { + someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT" + } + + tasks.register('resolveArtifacts') { + doLast { + configurations.someConfig.files.each { println it } + } + } + """ + + when: + def result = gradleRunner("resolveArtifacts").build() + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("resolveArtifacts", "-Ddra.artifacts=true", "-Ddra.workflow=SNAPSHOT").buildAndFail() + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.FAILED + result.output.contains("Cannot resolve external dependency org.acme:ml-cpp:8.6.0-SNAPSHOT because no repositories are defined.") + } + + def "configures repositories to resolve #draKey like dra #workflow artifacts"() { + setup: + repository.generateJar("some.group", "bar", "1.0.0") + repository.generateJar("some.group", "baz", "1.0.0-SNAPSHOT") + repository.configureBuild(buildFile) + buildFile << """ + configurations { + someConfig + } + + dependencies { + someConfig "some.group:bar:1.0.0" + someConfig "some.group:baz:1.0.0-SNAPSHOT" + someConfig "org.acme:$draArtifact:$draVersion@zip" + } + + tasks.register('resolveArtifacts') { + doLast { + configurations.someConfig.files.each { println it } + } + } + """ + + when: + def result = WiremockFixture.withWireMock(expectedRequest, "content".getBytes('UTF-8')) { server -> + gradleRunner("resolveArtifacts", + '-Ddra.artifacts=true', + "-Ddra.workflow=$workflow", + "-Ddra.artifacts.dependency.${draKey}=$buildId", + "-Ddra.artifacts.url.repo.prefix=${server.baseUrl()}").build() + } + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS + + where: + workflow | buildId | draVersion | draKey | draArtifact | expectedRequest + "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip" + "staging" | '8.6.0-f633b1d7' | "8.6.0" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip" + "release" | '8.6.0-f633b1d7' | "8.6.0" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip" + "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip" + "staging" | '8.6.0-f633b1d7' | "8.6.0" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip" + "release" | '8.6.0-f633b1d7' | "8.6.0" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip" + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java new file mode 100644 index 000000000000..180f68b99fe9 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.dra; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; + +import java.util.Map; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import static java.util.Map.Entry; + +public class DraResolvePlugin implements Plugin { + + public static final String USE_DRA_ARTIFACTS_FLAG = "dra.artifacts"; + public static final String DRA_WORKFLOW = "dra.workflow"; + public static final String DRA_ARTIFACTS_DEPENDENCY_PREFIX = "dra.artifacts.dependency"; + private final ProviderFactory providerFactory; + + private final Provider repositoryPrefix; + + @Inject + public DraResolvePlugin(ProviderFactory providerFactory) { + this.providerFactory = providerFactory; + this.repositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.prefix"); + } + + @Override + public void apply(Project project) { + boolean useDra = providerFactory.systemProperty(USE_DRA_ARTIFACTS_FLAG).map(Boolean::parseBoolean).getOrElse(false); + project.getExtensions().getExtraProperties().set("useDra", useDra); + if (useDra) { + DraWorkflow workflow = providerFactory.systemProperty(DRA_WORKFLOW).map(String::toUpperCase).map(DraWorkflow::valueOf).get(); + resolveBuildIdProperties().get().forEach((key, buildId) -> { + configureDraRepository( + project, + "dra-" + workflow.name().toLowerCase() + "-artifacts-" + key, + key, + buildId, + repositoryPrefix.orElse(workflow.repository), + workflow.versionRegex + ); + }); + } + } + + private void configureDraRepository( + Project project, + String repositoryName, + String draKey, + String buildId, + Provider repoPrefix, + String includeVersionRegex + ) { + project.getRepositories().ivy(repo -> { + repo.setName(repositoryName); + repo.setUrl(repoPrefix.get()); + repo.patternLayout(patternLayout -> { + patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]-[revision].[ext]", draKey, buildId, draKey)); + patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]/[module]-[revision].[ext]", draKey, buildId, draKey)); + }); + repo.metadataSources(metadataSources -> metadataSources.artifact()); + repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex)); + }); + } + + private Provider> resolveBuildIdProperties() { + return providerFactory.systemPropertiesPrefixedBy(DRA_ARTIFACTS_DEPENDENCY_PREFIX) + .map( + stringStringMap -> stringStringMap.entrySet() + .stream() + .collect( + Collectors.toMap(entry -> entry.getKey().substring(DRA_ARTIFACTS_DEPENDENCY_PREFIX.length() + 1), Entry::getValue) + ) + ); + } + + enum DraWorkflow { + SNAPSHOT("https://artifacts-snapshot.elastic.co/", ".*SNAPSHOT"), + STAGING("https://artifacts-staging.elastic.co/", "^(.(?!SNAPSHOT))*$"), + RELEASE("https://artifacts.elastic.co/", "^(.(?!SNAPSHOT))*$"); + + private final String repository; + public String versionRegex; + + DraWorkflow(String repository, String versionRegex) { + this.repository = repository; + this.versionRegex = versionRegex; + } + } +} diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index 5078d946c129..c13d178c3b27 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -3,6 +3,7 @@ import org.elasticsearch.gradle.VersionProperties apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.dra-artifacts' esplugin { name 'x-pack-ml' @@ -14,32 +15,34 @@ esplugin { def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull -repositories { - exclusiveContent { - filter { - includeGroup 'org.elasticsearch.ml' - } - forRepository { - ivy { - name "ml-cpp" - metadataSources { - // no repository metadata, look directly for the artifact - artifact() - } - if (localRepo) { - url localRepo - patternLayout { - artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" +if (useDra == false) { + repositories { + exclusiveContent { + filter { + includeGroup 'org.elasticsearch.ml' + } + forRepository { + ivy { + name "ml-cpp" + metadataSources { + // no repository metadata, look directly for the artifact + artifact() } - } else { - url "https://artifacts-snapshot.elastic.co/" - patternLayout { - if (VersionProperties.isElasticsearchSnapshot()) { - artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision](-[classifier]).[ext]' - } else { - // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. - // Release builds are always done with a local repo. - artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT(-[classifier]).[ext]' + if (localRepo) { + url localRepo + patternLayout { + artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" + } + } else { + url "https://artifacts-snapshot.elastic.co/" + patternLayout { + if (VersionProperties.isElasticsearchSnapshot()) { + artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision](-[classifier]).[ext]' + } else { + // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. + // Release builds are always done with a local repo. + artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT(-[classifier]).[ext]' + } } } } @@ -47,7 +50,6 @@ repositories { } } } - configurations { nativeBundle { resolutionStrategy.dependencySubstitution {