[7.x] Resolve artifacts from dra (#92478)

* WIP - Dra plugin

* Fix pattern for non classifier dependencies in dra artifacts

* Cleanup ml build script

* Fix spotless

build-tools-internal/build.gradle

@@ -52,6 +52,10 @@ gradlePlugin {
       id = 'elasticsearch.docs-test'
       implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
     }
+    draArtifacts {
+      id = 'elasticsearch.dra-artifacts'
+      implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin'
+    }
     globalBuildInfo {
       id = 'elasticsearch.global-build-info'
       implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy

@@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource {
 
     private TemporaryFolder temporaryFolder
 
-    LocalRepositoryFixture(){
+    LocalRepositoryFixture() {
         this.temporaryFolder = new TemporaryFolder()
     }
 
@@ -33,24 +33,29 @@ class LocalRepositoryFixture extends ExternalResource {
         temporaryFolder.after()
     }
 
-    void generateJar(String group, String module, String version, String... clazzNames){
+    void generateJar(String group, String module, String version, String... clazzNames) {
         def baseGroupFolderPath = group.replace('.', '/')
         def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version")
         targetFolder.mkdirs()
 
         def jarFile = new File(targetFolder, "${module}-${version}.jar")
-        clazzNames.each {clazzName ->
+        if (clazzNames.size() == 0) {
+            jarFile.write("blubb")
+        } else {
+            clazzNames.each { clazzName ->
                 DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
                         .name(clazzName)
                         .make()
-            if(jarFile.exists()) {
+                if (jarFile.exists()) {
                     dynamicType.inject(jarFile);
-            }else {
+                } else {
                     dynamicType.toJar(jarFile);
                 }
             }
         }
 
+    }
+
     void configureBuild(File buildFile) {
         buildFile << """
         repositories {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.dra
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture
+import org.elasticsearch.gradle.fixtures.WiremockFixture
+import org.gradle.testkit.runner.TaskOutcome
+import org.junit.ClassRule
+import spock.lang.Shared
+
+class DraResolvePluginFuncTest extends AbstractGradleFuncTest {
+
+    @Shared
+    @ClassRule
+    public LocalRepositoryFixture repository = new LocalRepositoryFixture()
+
+    def setup() {
+        configurationCacheCompatible = false
+
+        buildFile << """
+        plugins {
+            id 'elasticsearch.dra-artifacts'
+        }
+        
+        repositories.all {
+            // for supporting http testing repos here
+            allowInsecureProtocol = true
+        }
+        """
+    }
+
+    def "provides flag indicating dra usage"() {
+        setup:
+        repository.generateJar("org.acme", "ml-cpp", "8.6.0-SNAPSHOT")
+        buildFile << """
+        if(useDra == false) {
+            repositories {
+              maven {
+                name = "local-test"
+                url = "${repository.getRepoDir().toURI()}"
+                metadataSources {
+                  artifact()
+                }
+              }
+            }  
+        }
+        """
+
+        buildFile << """
+        configurations {
+            someConfig
+        }
+        
+        dependencies {
+            someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT"
+        }
+        
+        tasks.register('resolveArtifacts') {
+            doLast {
+                configurations.someConfig.files.each { println it }
+            }
+        }
+        """
+
+        when:
+        def result = gradleRunner("resolveArtifacts").build()
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
+
+        when:
+        result = gradleRunner("resolveArtifacts", "-Ddra.artifacts=true", "-Ddra.workflow=SNAPSHOT").buildAndFail()
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.FAILED
+        result.output.contains("Cannot resolve external dependency org.acme:ml-cpp:8.6.0-SNAPSHOT because no repositories are defined.")
+    }
+
+    def "configures repositories to resolve #draKey like dra #workflow artifacts"() {
+        setup:
+        repository.generateJar("some.group", "bar", "1.0.0")
+        repository.generateJar("some.group", "baz", "1.0.0-SNAPSHOT")
+        repository.configureBuild(buildFile)
+        buildFile << """
+        configurations {
+            someConfig
+        }
+        
+        dependencies {
+            someConfig "some.group:bar:1.0.0"
+            someConfig "some.group:baz:1.0.0-SNAPSHOT"
+            someConfig "org.acme:$draArtifact:$draVersion@zip"
+        }
+        
+        tasks.register('resolveArtifacts') {
+            doLast {
+                configurations.someConfig.files.each { println it }
+            }
+        }
+        """
+
+        when:
+        def result = WiremockFixture.withWireMock(expectedRequest, "content".getBytes('UTF-8')) { server ->
+            gradleRunner("resolveArtifacts",
+                    '-Ddra.artifacts=true',
+                    "-Ddra.workflow=$workflow",
+                    "-Ddra.artifacts.dependency.${draKey}=$buildId",
+                    "-Ddra.artifacts.url.repo.prefix=${server.baseUrl()}").build()
+        }
+
+        then:
+        result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS
+
+        where:
+        workflow   | buildId          | draVersion       | draKey   | draArtifact  | expectedRequest
+        "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "ml-cpp" | "ml-cpp"     | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
+        "staging"  | '8.6.0-f633b1d7' | "8.6.0"          | "ml-cpp" | "ml-cpp"     | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
+        "release"  | '8.6.0-f633b1d7' | "8.6.0"          | "ml-cpp" | "ml-cpp"     | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}.zip"
+        "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "beats"  | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
+        "staging"  | '8.6.0-f633b1d7' | "8.6.0"          | "beats"  | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
+        "release"  | '8.6.0-f633b1d7' | "8.6.0"          | "beats"  | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}.zip"
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.dra;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+import static java.util.Map.Entry;
+
+public class DraResolvePlugin implements Plugin<Project> {
+
+    public static final String USE_DRA_ARTIFACTS_FLAG = "dra.artifacts";
+    public static final String DRA_WORKFLOW = "dra.workflow";
+    public static final String DRA_ARTIFACTS_DEPENDENCY_PREFIX = "dra.artifacts.dependency";
+    private final ProviderFactory providerFactory;
+
+    private final Provider<String> repositoryPrefix;
+
+    @Inject
+    public DraResolvePlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+        this.repositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.prefix");
+    }
+
+    @Override
+    public void apply(Project project) {
+        boolean useDra = providerFactory.systemProperty(USE_DRA_ARTIFACTS_FLAG).map(Boolean::parseBoolean).getOrElse(false);
+        project.getExtensions().getExtraProperties().set("useDra", useDra);
+        if (useDra) {
+            DraWorkflow workflow = providerFactory.systemProperty(DRA_WORKFLOW).map(String::toUpperCase).map(DraWorkflow::valueOf).get();
+            resolveBuildIdProperties().get().forEach((key, buildId) -> {
+                configureDraRepository(
+                    project,
+                    "dra-" + workflow.name().toLowerCase() + "-artifacts-" + key,
+                    key,
+                    buildId,
+                    repositoryPrefix.orElse(workflow.repository),
+                    workflow.versionRegex
+                );
+            });
+        }
+    }
+
+    private void configureDraRepository(
+        Project project,
+        String repositoryName,
+        String draKey,
+        String buildId,
+        Provider<String> repoPrefix,
+        String includeVersionRegex
+    ) {
+        project.getRepositories().ivy(repo -> {
+            repo.setName(repositoryName);
+            repo.setUrl(repoPrefix.get());
+            repo.patternLayout(patternLayout -> {
+                patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]-[revision].[ext]", draKey, buildId, draKey));
+                patternLayout.artifact(String.format("/%s/%s/downloads/%s/[module]/[module]-[revision].[ext]", draKey, buildId, draKey));
+            });
+            repo.metadataSources(metadataSources -> metadataSources.artifact());
+            repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex));
+        });
+    }
+
+    private Provider<Map<String, String>> resolveBuildIdProperties() {
+        return providerFactory.systemPropertiesPrefixedBy(DRA_ARTIFACTS_DEPENDENCY_PREFIX)
+            .map(
+                stringStringMap -> stringStringMap.entrySet()
+                    .stream()
+                    .collect(
+                        Collectors.toMap(entry -> entry.getKey().substring(DRA_ARTIFACTS_DEPENDENCY_PREFIX.length() + 1), Entry::getValue)
+                    )
+            );
+    }
+
+    enum DraWorkflow {
+        SNAPSHOT("https://artifacts-snapshot.elastic.co/", ".*SNAPSHOT"),
+        STAGING("https://artifacts-staging.elastic.co/", "^(.(?!SNAPSHOT))*$"),
+        RELEASE("https://artifacts.elastic.co/", "^(.(?!SNAPSHOT))*$");
+
+        private final String repository;
+        public String versionRegex;
+
+        DraWorkflow(String repository, String versionRegex) {
+            this.repository = repository;
+            this.versionRegex = versionRegex;
+        }
+    }
+}

x-pack/plugin/ml/build.gradle

@@ -3,6 +3,7 @@ import org.elasticsearch.gradle.VersionProperties
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.dra-artifacts'
 
 esplugin {
   name 'x-pack-ml'
@@ -14,7 +15,8 @@ esplugin {
 
 def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull
 
-repositories {
+if (useDra == false) {
+  repositories {
     exclusiveContent {
       filter {
         includeGroup 'org.elasticsearch.ml'
@@ -46,8 +48,8 @@ repositories {
         }
       }
     }
+  }
 }
-
 configurations {
   nativeBundle {
     resolutionStrategy.dependencySubstitution {