Port repository-url to new TestFramework (#102588)

removes docker-compose dependency from url-fixture

modules/repository-url/build.gradle

@@ -8,12 +8,9 @@
 
 import org.elasticsearch.gradle.PropertyNormalization
 
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.test.fixtures'
-
-final Project fixture = project(':test:fixtures:url-fixture')
 
 esplugin {
   description 'Module for URL repository'
@@ -32,6 +29,8 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  yamlRestTestImplementation project(':test:fixtures:url-fixture')
+  internalClusterTestImplementation project(':test:fixtures:url-fixture')
 }
 
 tasks.named("thirdPartyAudit").configure {
@@ -45,15 +44,7 @@ tasks.named("thirdPartyAudit").configure {
   )
 }
 
-testFixtures.useFixture(fixture.path, 'url-fixture')
+//File repositoryDir = fixture.fsRepositoryDir as File
-
-def fixtureAddress = { fixtureName ->
-  int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
-  assert ephemeralPort > 0
-  'http://127.0.0.1:' + ephemeralPort
-}
-
-File repositoryDir = fixture.fsRepositoryDir as File
 
 testClusters.configureEach {
   // repositoryDir is used by a FS repository to create snapshots

modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.repositories.url;
 
+import fixture.url.URLFixture;
+
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
@@ -22,11 +24,15 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -42,6 +48,22 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    public static final URLFixture urlFixture = new URLFixture();
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-url")
+        .setting("path.repo", urlFixture::getRepositoryDir)
+        .setting("repositories.url.allowed_urls", () -> "http://snapshot.test*, " + urlFixture.getAddress())
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(urlFixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }

test/fixtures/url-fixture/Dockerfile

@@ -1,14 +0,0 @@
-FROM openjdk:17.0.2
-
-ARG port
-ARG workingDir
-ARG repositoryDir
-
-ENV URL_FIXTURE_PORT=${port}
-ENV URL_FIXTURE_WORKING_DIR=${workingDir}
-ENV URL_FIXTURE_REPO_DIR=${repositoryDir}
-
-ENTRYPOINT exec java -classpath "/fixture/shared/*" \
-    fixture.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR"
-
-EXPOSE $port

test/fixtures/url-fixture/build.gradle

@@ -6,30 +6,9 @@
  * Side Public License, v 1.
  */
 apply plugin: 'elasticsearch.java'
-apply plugin: 'elasticsearch.test.fixtures'
-
 description = 'Fixture for URL external service'
-tasks.named("test").configure { enabled = false }
 
 dependencies {
   api project(':server')
   api project(':test:framework')
 }
-
-// These directories are shared between the URL repository and the FS repository in integration tests
-project.ext {
-  fsRepositoryDir = file("${testFixturesDir}/fs-repository")
-}
-
-tasks.named("preProcessFixture").configure {
-  dependsOn "jar", configurations.runtimeClasspath
-  doLast {
-    file("${testFixturesDir}/shared").mkdirs()
-    project.copy {
-      from jar
-      from configurations.runtimeClasspath
-      into "${testFixturesDir}/shared"
-    }
-    project.fsRepositoryDir.mkdirs()
-  }
-}

test/fixtures/url-fixture/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3'
-services:
-  url-fixture:
-    build:
-      context: .
-      args:
-        port: 80
-        workingDir: "/fixture/work"
-        repositoryDir: "/fixture/repo"
-    volumes:
-      - ./testfixtures_shared/shared:/fixture/shared
-      - ./testfixtures_shared/fs-repository:/fixture/repo
-      - ./testfixtures_shared/work:/fixture/work
-    ports:
-      - "80"

test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java

@@ -7,15 +7,17 @@
  */
 package fixture.url;
 
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.fixture.AbstractHttpFixture;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -25,32 +27,17 @@ import java.util.regex.Pattern;
  * This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
  * integration tests to expose a directory created by a regular FS repository.
  */
-public class URLFixture extends AbstractHttpFixture {
+public class URLFixture extends AbstractHttpFixture implements TestRule {
     private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)$");
-    private final Path repositoryDir;
+    private final TemporaryFolder temporaryFolder;
+    private Path repositoryDir;
 
     /**
      * Creates a {@link URLFixture}
      */
-    private URLFixture(final int port, final String workingDir, final String repositoryDir) {
+    public URLFixture() {
-        super(workingDir, port);
+        super();
-        this.repositoryDir = dir(repositoryDir);
+        this.temporaryFolder = new TemporaryFolder();
-    }
-
-    public static void main(String[] args) throws Exception {
-        if (args == null || args.length != 3) {
-            throw new IllegalArgumentException("URLFixture <port> <working directory> <repository directory>");
-        }
-        String workingDirectory = args[1];
-        if (Files.exists(dir(workingDirectory)) == false) {
-            throw new IllegalArgumentException("Configured working directory " + workingDirectory + " does not exist");
-        }
-        String repositoryDirectory = args[2];
-        if (Files.exists(dir(repositoryDirectory)) == false) {
-            throw new IllegalArgumentException("Configured repository directory " + repositoryDirectory + " does not exist");
-        }
-        final URLFixture fixture = new URLFixture(Integer.parseInt(args[0]), workingDirectory, repositoryDirectory);
-        fixture.listen(InetAddress.getByName("0.0.0.0"), false);
     }
 
     @Override
@@ -107,8 +94,32 @@ public class URLFixture extends AbstractHttpFixture {
         }
     }
 
-    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    @Override
-    private static Path dir(final String dir) {
+    protected void before() throws Throwable {
-        return Paths.get(dir);
+        this.temporaryFolder.create();
+        this.repositoryDir = temporaryFolder.newFolder("repoDir").toPath();
+        InetSocketAddress inetSocketAddress = resolveAddress("0.0.0.0", 0);
+        listen(inetSocketAddress, false);
+    }
+
+    public String getRepositoryDir() {
+        if (repositoryDir == null) {
+            throw new IllegalStateException("Rule has not been started yet");
+        }
+        return repositoryDir.toFile().getAbsolutePath();
+    }
+
+    private static InetSocketAddress resolveAddress(String address, int port) {
+        try {
+            return new InetSocketAddress(InetAddress.getByName(address), port);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected void after() {
+        super.stop();
+        this.temporaryFolder.delete();
     }
 }

test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java

@@ -12,6 +12,7 @@ import com.sun.net.httpserver.HttpServer;
 
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
+import org.junit.rules.ExternalResource;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -40,7 +41,7 @@ import static java.util.Collections.singletonMap;
  * Base class for test fixtures that requires a {@link HttpServer} to work.
  */
 @SuppressForbidden(reason = "uses httpserver by design")
-public abstract class AbstractHttpFixture {
+public abstract class AbstractHttpFixture extends ExternalResource {
 
     protected static final Map<String, String> TEXT_PLAIN_CONTENT_TYPE = contentType("text/plain; charset=utf-8");
     protected static final Map<String, String> JSON_CONTENT_TYPE = contentType("application/json; charset=utf-8");
@@ -51,8 +52,9 @@ public abstract class AbstractHttpFixture {
     private final AtomicLong requests = new AtomicLong(0);
 
     /** Current working directory of the fixture **/
-    private final Path workingDirectory;
+    private Path workingDirectory;
-    private final int port;
+    private int port;
+    private HttpServer httpServer;
 
     protected AbstractHttpFixture(final String workingDir) {
         this(workingDir, 0);
@@ -63,6 +65,8 @@ public abstract class AbstractHttpFixture {
         this.workingDirectory = PathUtils.get(Objects.requireNonNull(workingDir));
     }
 
+    public AbstractHttpFixture() {}
+
     /**
      * Opens a {@link HttpServer} and start listening on a provided or random port.
      */
@@ -75,85 +79,100 @@ public abstract class AbstractHttpFixture {
      */
     public final void listen(InetAddress inetAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
         final InetSocketAddress socketAddress = new InetSocketAddress(inetAddress, port);
-        final HttpServer httpServer = HttpServer.create(socketAddress, 0);
+        listenAndWait(socketAddress, exposePidAndPort);
+    }
 
+    public final void listenAndWait(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
         try {
-            if (exposePidAndPort) {
+            listen(socketAddress, exposePidAndPort);
-                /// Writes the PID of the current Java process in a `pid` file located in the working directory
-                writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
-
-                final String addressAndPort = addressToString(httpServer.getAddress());
-                // Writes the address and port of the http server in a `ports` file located in the working directory
-                writeFile(workingDirectory, "ports", addressAndPort);
-            }
-
-            httpServer.createContext("/", exchange -> {
-                try {
-                    Response response;
-
-                    // Check if this is a request made by the AntFixture
-                    final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
-                    if (userAgent != null
-                        && userAgent.startsWith("Apache Ant")
-                        && "GET".equals(exchange.getRequestMethod())
-                        && "/".equals(exchange.getRequestURI().getPath())) {
-                        response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8));
-
-                    } else {
-                        try {
-                            final long requestId = requests.getAndIncrement();
-                            final String method = exchange.getRequestMethod();
-
-                            final Map<String, String> headers = new HashMap<>();
-                            for (Map.Entry<String, List<String>> header : exchange.getRequestHeaders().entrySet()) {
-                                headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey()));
-                            }
-
-                            final ByteArrayOutputStream body = new ByteArrayOutputStream();
-                            try (InputStream requestBody = exchange.getRequestBody()) {
-                                final byte[] buffer = new byte[1024];
-                                int i;
-                                while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) {
-                                    body.write(buffer, 0, i);
-                                }
-                                body.flush();
-                            }
-
-                            final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray());
-                            response = handle(request);
-
-                        } catch (Exception e) {
-                            final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request";
-                            response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8));
-                        }
-                    }
-
-                    if (response == null) {
-                        response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
-                    }
-
-                    response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v)));
-                    if (response.body.length > 0) {
-                        exchange.sendResponseHeaders(response.status, response.body.length);
-                        exchange.getResponseBody().write(response.body);
-                    } else {
-                        exchange.sendResponseHeaders(response.status, -1);
-                    }
-                } finally {
-                    exchange.close();
-                }
-            });
-            httpServer.start();
-
             // Wait to be killed
             Thread.sleep(Long.MAX_VALUE);
-
         } finally {
+            stop();
+        }
+    }
+
+    public final void listen(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
+        httpServer = HttpServer.create(socketAddress, 0);
+        if (exposePidAndPort) {
+            /// Writes the PID of the current Java process in a `pid` file located in the working directory
+            writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
+
+            final String addressAndPort = addressToString(httpServer.getAddress());
+            // Writes the address and port of the http server in a `ports` file located in the working directory
+            writeFile(workingDirectory, "ports", addressAndPort);
+        }
+
+        httpServer.createContext("/", exchange -> {
+            try {
+                Response response;
+
+                // Check if this is a request made by the AntFixture
+                final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
+                if (userAgent != null
+                    && userAgent.startsWith("Apache Ant")
+                    && "GET".equals(exchange.getRequestMethod())
+                    && "/".equals(exchange.getRequestURI().getPath())) {
+                    response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8));
+
+                } else {
+                    try {
+                        final long requestId = requests.getAndIncrement();
+                        final String method = exchange.getRequestMethod();
+
+                        final Map<String, String> headers = new HashMap<>();
+                        for (Map.Entry<String, List<String>> header : exchange.getRequestHeaders().entrySet()) {
+                            headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey()));
+                        }
+
+                        final ByteArrayOutputStream body = new ByteArrayOutputStream();
+                        try (InputStream requestBody = exchange.getRequestBody()) {
+                            final byte[] buffer = new byte[1024];
+                            int i;
+                            while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) {
+                                body.write(buffer, 0, i);
+                            }
+                            body.flush();
+                        }
+
+                        final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray());
+                        response = handle(request);
+
+                    } catch (Exception e) {
+                        final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request";
+                        response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8));
+                    }
+                }
+
+                if (response == null) {
+                    response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
+                }
+
+                response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v)));
+                if (response.body.length > 0) {
+                    exchange.sendResponseHeaders(response.status, response.body.length);
+                    exchange.getResponseBody().write(response.body);
+                } else {
+                    exchange.sendResponseHeaders(response.status, -1);
+                }
+            } finally {
+                exchange.close();
+            }
+        });
+        httpServer.start();
+    }
+
+    protected abstract Response handle(Request request) throws IOException;
+
+    protected void stop() {
+        if (httpServer != null) {
             httpServer.stop(0);
         }
     }
 
-    protected abstract Response handle(Request request) throws IOException;
+    public String getAddress() {
+        return "http://127.0.0.1:" + httpServer.getAddress().getPort();
+    }
 
     @FunctionalInterface
     public interface RequestHandler {