mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
[7.x] Resolve artifacts from dra (#92478)
* WIP - Dra plugin * Fix pattern for non classifier dependencies in dra artifacts * Cleanup ml build script * Fix spotless
This commit is contained in:
Rene Groeschke
GitHub
parent
bc781f305e
commit
1aa7237781
5 changed files with 279 additions and 37 deletions
|
|
@ -52,6 +52,10 @@ gradlePlugin {
|
|||
id = 'elasticsearch.docs-test'
|
||||
implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
|
||||
}
|
||||
draArtifacts {
|
||||
id = 'elasticsearch.dra-artifacts'
|
||||
implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin'
|
||||
}
|
||||
globalBuildInfo {
|
||||
id = 'elasticsearch.global-build-info'
|
||||
implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource {
|
|||
|
||||
private TemporaryFolder temporaryFolder
|
||||
|
||||
LocalRepositoryFixture(){
|
||||
LocalRepositoryFixture() {
|
||||
this.temporaryFolder = new TemporaryFolder()
|
||||
}
|
||||
|
||||
|
|
@ -33,22 +33,27 @@ class LocalRepositoryFixture extends ExternalResource {
|
|||
temporaryFolder.after()
|
||||
}
|
||||
|
||||
void generateJar(String group, String module, String version, String... clazzNames){
|
||||
void generateJar(String group, String module, String version, String... clazzNames) {
|
||||
def baseGroupFolderPath = group.replace('.', '/')
|
||||
def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version")
|
||||
targetFolder.mkdirs()
|
||||
|
||||
def jarFile = new File(targetFolder, "${module}-${version}.jar")
|
||||
clazzNames.each {clazzName ->
|
||||
DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
|
||||
.name(clazzName)
|
||||
.make()
|
||||
if(jarFile.exists()) {
|
||||
dynamicType.inject(jarFile);
|
||||
}else {
|
||||
dynamicType.toJar(jarFile);
|
||||
if (clazzNames.size() == 0) {
|
||||
jarFile.write("blubb")
|
||||
} else {
|
||||
clazzNames.each { clazzName ->
|
||||
DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
|
||||
.name(clazzName)
|
||||
.make()
|
||||
if (jarFile.exists()) {
|
||||
dynamicType.inject(jarFile);
|
||||
} else {
|
||||
dynamicType.toJar(jarFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void configureBuild(File buildFile) {
|
||||
|
|
@ -68,4 +73,4 @@ class LocalRepositoryFixture extends ExternalResource {
|
|||
File getRepoDir() {
|
||||
new File(temporaryFolder.root, 'local-repo')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal.dra
|
||||
|
||||
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
|
||||
import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture
|
||||
import org.elasticsearch.gradle.fixtures.WiremockFixture
|
||||
import org.gradle.testkit.runner.TaskOutcome
|
||||
import org.junit.ClassRule
|
||||
import spock.lang.Shared
|
||||
|
||||
class DraResolvePluginFuncTest extends AbstractGradleFuncTest {
|
||||
|
||||
@Shared
|
||||
@ClassRule
|
||||
public LocalRepositoryFixture repository = new LocalRepositoryFixture()
|
||||
|
||||
def setup() {
|
||||
configurationCacheCompatible = false
|
||||
|
||||
buildFile << """
|
||||
plugins {
|
||||
id 'elasticsearch.dra-artifacts'
|
||||
}
|
||||
|
||||
repositories.all {
|
||||
// for supporting http testing repos here
|
||||
allowInsecureProtocol = true
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
def "provides flag indicating dra usage"() {
|
||||
setup:
|
||||
repository.generateJar("org.acme", "ml-cpp", "8.6.0-SNAPSHOT")
|
||||
buildFile << """
|
||||
if(useDra == false) {
|
||||
repositories {
|
||||
maven {
|
||||
name = "local-test"
|
||||
url = "${repository.getRepoDir().toURI()}"
|
||||
metadataSources {
|
||||
artifact()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
buildFile << """
|
||||
configurations {
|
||||
someConfig
|
||||
}
|
||||
|
||||
dependencies {
|
||||
someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT"
|
||||
}
|
||||
|
||||
tasks.register('resolveArtifacts') {
|
||||
doLast {
|
||||
configurations.someConfig.files.each { println it }
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = gradleRunner("resolveArtifacts").build()
|
||||
|
||||
then:
|
||||
result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
|
||||
|
||||
when:
|
||||
result = gradleRunner("resolveArtifacts", "-Ddra.artifacts=true", "-Ddra.workflow=SNAPSHOT").buildAndFail()
|
||||
|
||||
then:
|
||||
result.task(":resolveArtifacts").outcome == TaskOutcome.FAILED
|
||||
result.output.contains("Cannot resolve external dependency org.acme:ml-cpp:8.6.0-SNAPSHOT because no repositories are defined.")
|
||||
}
|
||||
|
||||
def "configures repositories to resolve #draKey like dra #workflow artifacts"() {
|
||||
setup:
|
||||
repository.generateJar("some.group", "bar", "1.0.0")
|
||||
repository.generateJar("some.group", "baz", "1.0.0-SNAPSHOT")
|
||||
repository.configureBuild(buildFile)
|
||||
buildFile << """
|
||||
configurations {
|
||||
someConfig
|
||||
}
|
||||
|
||||
dependencies {
|
||||
someConfig "some.group:bar:1.0.0"
|
||||
someConfig "some.group:baz:1.0.0-SNAPSHOT"
|
||||
someConfig "org.acme:$draArtifact:$draVersion@zip"
|
||||
}
|
||||
|
||||
tasks.register('resolveArtifacts') {
|
||||
doLast {
|
||||
configurations.someConfig.files.each { println it }
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(expectedRequest, "content".getBytes('UTF-8')) { server ->
|
||||
gradleRunner("resolveArtifacts",
|
||||
'-Ddra.artifacts=true',
|
||||
"-Ddra.workflow=$workflow",
|
||||
"-Ddra.artifacts.dependency.${draKey}=$buildId",
|
||||
"-Ddra.artifacts.url.repo.prefix=${server.baseUrl()}").build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
|
||||
|
||||
where:
|
||||
workflow | buildId | draVersion | draKey | draArtifact | expectedRequest
|
||||
"snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
"staging" | '8.6.0-f633b1d7' | "8.6.0" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
"release" | '8.6.0-f633b1d7' | "8.6.0" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
"snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
"staging" | '8.6.0-f633b1d7' | "8.6.0" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
"release" | '8.6.0-f633b1d7' | "8.6.0" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal.dra;
|
||||
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.provider.ProviderFactory;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import static java.util.Map.Entry;
|
||||
|
||||
public class DraResolvePlugin implements Plugin<Project> {
|
||||
|
||||
public static final String USE_DRA_ARTIFACTS_FLAG = "dra.artifacts";
|
||||
public static final String DRA_WORKFLOW = "dra.workflow";
|
||||
public static final String DRA_ARTIFACTS_DEPENDENCY_PREFIX = "dra.artifacts.dependency";
|
||||
private final ProviderFactory providerFactory;
|
||||
|
||||
private final Provider<String> repositoryPrefix;
|
||||
|
||||
@Inject
|
||||
public DraResolvePlugin(ProviderFactory providerFactory) {
|
||||
this.providerFactory = providerFactory;
|
||||
this.repositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.prefix");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
boolean useDra = providerFactory.systemProperty(USE_DRA_ARTIFACTS_FLAG).map(Boolean::parseBoolean).getOrElse(false);
|
||||
project.getExtensions().getExtraProperties().set("useDra", useDra);
|
||||
if (useDra) {
|
||||
DraWorkflow workflow = providerFactory.systemProperty(DRA_WORKFLOW).map(String::toUpperCase).map(DraWorkflow::valueOf).get();
|
||||
resolveBuildIdProperties().get().forEach((key, buildId) -> {
|
||||
configureDraRepository(
|
||||
project,
|
||||
"dra-" + workflow.name().toLowerCase() + "-artifacts-" + key,
|
||||
key,
|
||||
buildId,
|
||||
repositoryPrefix.orElse(workflow.repository),
|
||||
workflow.versionRegex
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void configureDraRepository(
|
||||
Project project,
|
||||
String repositoryName,
|
||||
String draKey,
|
||||
String buildId,
|
||||
Provider<String> repoPrefix,
|
||||
String includeVersionRegex
|
||||
) {
|
||||
project.getRepositories().ivy(repo -> {
|
||||
repo.setName(repositoryName);
|
||||
repo.setUrl(repoPrefix.get());
|
||||
repo.patternLayout(patternLayout -> {
|
||||
patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]-[revision].[ext]", draKey, buildId, draKey));
|
||||
patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]/[module]-[revision].[ext]", draKey, buildId, draKey));
|
||||
});
|
||||
repo.metadataSources(metadataSources -> metadataSources.artifact());
|
||||
repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex));
|
||||
});
|
||||
}
|
||||
|
||||
private Provider<Map<String, String>> resolveBuildIdProperties() {
|
||||
return providerFactory.systemPropertiesPrefixedBy(DRA_ARTIFACTS_DEPENDENCY_PREFIX)
|
||||
.map(
|
||||
stringStringMap -> stringStringMap.entrySet()
|
||||
.stream()
|
||||
.collect(
|
||||
Collectors.toMap(entry -> entry.getKey().substring(DRA_ARTIFACTS_DEPENDENCY_PREFIX.length() + 1), Entry::getValue)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
enum DraWorkflow {
|
||||
SNAPSHOT("https://artifacts-snapshot.elastic.co/", ".*SNAPSHOT"),
|
||||
STAGING("https://artifacts-staging.elastic.co/", "^(.(?!SNAPSHOT))*$"),
|
||||
RELEASE("https://artifacts.elastic.co/", "^(.(?!SNAPSHOT))*$");
|
||||
|
||||
private final String repository;
|
||||
public String versionRegex;
|
||||
|
||||
DraWorkflow(String repository, String versionRegex) {
|
||||
this.repository = repository;
|
||||
this.versionRegex = versionRegex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ import org.elasticsearch.gradle.VersionProperties
|
|||
apply plugin: 'elasticsearch.internal-es-plugin'
|
||||
apply plugin: 'elasticsearch.internal-cluster-test'
|
||||
apply plugin: 'elasticsearch.internal-test-artifact'
|
||||
apply plugin: 'elasticsearch.dra-artifacts'
|
||||
|
||||
esplugin {
|
||||
name 'x-pack-ml'
|
||||
|
|
@ -14,32 +15,34 @@ esplugin {
|
|||
|
||||
def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull
|
||||
|
||||
repositories {
|
||||
exclusiveContent {
|
||||
filter {
|
||||
includeGroup 'org.elasticsearch.ml'
|
||||
}
|
||||
forRepository {
|
||||
ivy {
|
||||
name "ml-cpp"
|
||||
metadataSources {
|
||||
// no repository metadata, look directly for the artifact
|
||||
artifact()
|
||||
}
|
||||
if (localRepo) {
|
||||
url localRepo
|
||||
patternLayout {
|
||||
artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
|
||||
if (useDra == false) {
|
||||
repositories {
|
||||
exclusiveContent {
|
||||
filter {
|
||||
includeGroup 'org.elasticsearch.ml'
|
||||
}
|
||||
forRepository {
|
||||
ivy {
|
||||
name "ml-cpp"
|
||||
metadataSources {
|
||||
// no repository metadata, look directly for the artifact
|
||||
artifact()
|
||||
}
|
||||
} else {
|
||||
url "https://artifacts-snapshot.elastic.co/"
|
||||
patternLayout {
|
||||
if (VersionProperties.isElasticsearchSnapshot()) {
|
||||
artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision](-[classifier]).[ext]'
|
||||
} else {
|
||||
// When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
|
||||
// Release builds are always done with a local repo.
|
||||
artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT(-[classifier]).[ext]'
|
||||
if (localRepo) {
|
||||
url localRepo
|
||||
patternLayout {
|
||||
artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
|
||||
}
|
||||
} else {
|
||||
url "https://artifacts-snapshot.elastic.co/"
|
||||
patternLayout {
|
||||
if (VersionProperties.isElasticsearchSnapshot()) {
|
||||
artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision](-[classifier]).[ext]'
|
||||
} else {
|
||||
// When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
|
||||
// Release builds are always done with a local repo.
|
||||
artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT(-[classifier]).[ext]'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -47,7 +50,6 @@ repositories {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
configurations {
|
||||
nativeBundle {
|
||||
resolutionStrategy.dependencySubstitution {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue