From bc17acdada7b67dfbc9cd2c9b10648d8bd188425 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 1 Dec 2023 11:23:53 +0100 Subject: [PATCH] Port repository-url to new TestFramework (#102588) removes docker-compose dependency from url-fixture --- modules/repository-url/build.gradle | 19 +- .../RepositoryURLClientYamlTestSuiteIT.java | 22 +++ test/fixtures/url-fixture/Dockerfile | 14 -- test/fixtures/url-fixture/build.gradle | 21 --- test/fixtures/url-fixture/docker-compose.yml | 15 -- .../src/main/java/fixture/url/URLFixture.java | 63 ++++--- .../test/fixture/AbstractHttpFixture.java | 165 ++++++++++-------- 7 files changed, 156 insertions(+), 163 deletions(-) delete mode 100644 test/fixtures/url-fixture/Dockerfile delete mode 100644 test/fixtures/url-fixture/docker-compose.yml diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 7b671802f3a2..2850aee68a2f 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -8,12 +8,9 @@ import org.elasticsearch.gradle.PropertyNormalization -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.test.fixtures' - -final Project fixture = project(':test:fixtures:url-fixture') esplugin { description 'Module for URL repository' @@ -32,6 +29,8 @@ dependencies { api "commons-logging:commons-logging:${versions.commonslogging}" api "commons-codec:commons-codec:${versions.commonscodec}" api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" + yamlRestTestImplementation project(':test:fixtures:url-fixture') + internalClusterTestImplementation project(':test:fixtures:url-fixture') } tasks.named("thirdPartyAudit").configure { @@ -45,15 +44,7 @@ tasks.named("thirdPartyAudit").configure { ) } -testFixtures.useFixture(fixture.path, 'url-fixture') - -def fixtureAddress = { fixtureName -> - int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80" - assert ephemeralPort > 0 - 'http://127.0.0.1:' + ephemeralPort -} - -File repositoryDir = fixture.fsRepositoryDir as File +//File repositoryDir = fixture.fsRepositoryDir as File testClusters.configureEach { // repositoryDir is used by a FS repository to create snapshots diff --git a/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index 0958276656a8..a5b1a48f94ac 100644 --- a/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.repositories.url; +import fixture.url.URLFixture; + import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -22,11 +24,15 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.PathUtils; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; import java.io.IOException; import java.net.InetAddress; @@ -42,6 +48,22 @@ import static org.hamcrest.Matchers.notNullValue; public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + public static final URLFixture urlFixture = new URLFixture(); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-url") + .setting("path.repo", urlFixture::getRepositoryDir) + .setting("repositories.url.allowed_urls", () -> "http://snapshot.test*, " + urlFixture.getAddress()) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(urlFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/test/fixtures/url-fixture/Dockerfile b/test/fixtures/url-fixture/Dockerfile deleted file mode 100644 index d6c1443fa1f8..000000000000 --- a/test/fixtures/url-fixture/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM openjdk:17.0.2 - -ARG port -ARG workingDir -ARG repositoryDir - -ENV URL_FIXTURE_PORT=${port} -ENV URL_FIXTURE_WORKING_DIR=${workingDir} -ENV URL_FIXTURE_REPO_DIR=${repositoryDir} - -ENTRYPOINT exec java -classpath "/fixture/shared/*" \ - fixture.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR" - -EXPOSE $port diff --git a/test/fixtures/url-fixture/build.gradle b/test/fixtures/url-fixture/build.gradle index d7d9fd2964c1..d8bcce6ce821 100644 --- a/test/fixtures/url-fixture/build.gradle +++ b/test/fixtures/url-fixture/build.gradle @@ -6,30 +6,9 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.java' -apply plugin: 'elasticsearch.test.fixtures' - description = 'Fixture for URL external service' -tasks.named("test").configure { enabled = false } dependencies { api project(':server') api project(':test:framework') } - -// These directories are shared between the URL repository and the FS repository in integration tests -project.ext { - fsRepositoryDir = file("${testFixturesDir}/fs-repository") -} - -tasks.named("preProcessFixture").configure { - dependsOn "jar", configurations.runtimeClasspath - doLast { - file("${testFixturesDir}/shared").mkdirs() - project.copy { - from jar - from configurations.runtimeClasspath - into "${testFixturesDir}/shared" - } - project.fsRepositoryDir.mkdirs() - } -} diff --git a/test/fixtures/url-fixture/docker-compose.yml b/test/fixtures/url-fixture/docker-compose.yml deleted file mode 100644 index edfc879b1cec..000000000000 --- a/test/fixtures/url-fixture/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - url-fixture: - build: - context: . - args: - port: 80 - workingDir: "/fixture/work" - repositoryDir: "/fixture/repo" - volumes: - - ./testfixtures_shared/shared:/fixture/shared - - ./testfixtures_shared/fs-repository:/fixture/repo - - ./testfixtures_shared/work:/fixture/work - ports: - - "80" diff --git a/test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java b/test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java index 3f6eed903765..5192140f1af4 100644 --- a/test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java +++ b/test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java @@ -7,15 +7,17 @@ */ package fixture.url; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.fixture.AbstractHttpFixture; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; import java.io.IOException; import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -25,32 +27,17 @@ import java.util.regex.Pattern; * This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url * integration tests to expose a directory created by a regular FS repository. */ -public class URLFixture extends AbstractHttpFixture { +public class URLFixture extends AbstractHttpFixture implements TestRule { private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)$"); - private final Path repositoryDir; + private final TemporaryFolder temporaryFolder; + private Path repositoryDir; /** * Creates a {@link URLFixture} */ - private URLFixture(final int port, final String workingDir, final String repositoryDir) { - super(workingDir, port); - this.repositoryDir = dir(repositoryDir); - } - - public static void main(String[] args) throws Exception { - if (args == null || args.length != 3) { - throw new IllegalArgumentException("URLFixture "); - } - String workingDirectory = args[1]; - if (Files.exists(dir(workingDirectory)) == false) { - throw new IllegalArgumentException("Configured working directory " + workingDirectory + " does not exist"); - } - String repositoryDirectory = args[2]; - if (Files.exists(dir(repositoryDirectory)) == false) { - throw new IllegalArgumentException("Configured repository directory " + repositoryDirectory + " does not exist"); - } - final URLFixture fixture = new URLFixture(Integer.parseInt(args[0]), workingDirectory, repositoryDirectory); - fixture.listen(InetAddress.getByName("0.0.0.0"), false); + public URLFixture() { + super(); + this.temporaryFolder = new TemporaryFolder(); } @Override @@ -107,8 +94,32 @@ public class URLFixture extends AbstractHttpFixture { } } - @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here") - private static Path dir(final String dir) { - return Paths.get(dir); + @Override + protected void before() throws Throwable { + this.temporaryFolder.create(); + this.repositoryDir = temporaryFolder.newFolder("repoDir").toPath(); + InetSocketAddress inetSocketAddress = resolveAddress("0.0.0.0", 0); + listen(inetSocketAddress, false); + } + + public String getRepositoryDir() { + if (repositoryDir == null) { + throw new IllegalStateException("Rule has not been started yet"); + } + return repositoryDir.toFile().getAbsolutePath(); + } + + private static InetSocketAddress resolveAddress(String address, int port) { + try { + return new InetSocketAddress(InetAddress.getByName(address), port); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + + @Override + protected void after() { + super.stop(); + this.temporaryFolder.delete(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java b/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java index 87b8f5f89ffa..8e7fae85e57f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java +++ b/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java @@ -12,6 +12,7 @@ import com.sun.net.httpserver.HttpServer; import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.SuppressForbidden; +import org.junit.rules.ExternalResource; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,7 +41,7 @@ import static java.util.Collections.singletonMap; * Base class for test fixtures that requires a {@link HttpServer} to work. */ @SuppressForbidden(reason = "uses httpserver by design") -public abstract class AbstractHttpFixture { +public abstract class AbstractHttpFixture extends ExternalResource { protected static final Map TEXT_PLAIN_CONTENT_TYPE = contentType("text/plain; charset=utf-8"); protected static final Map JSON_CONTENT_TYPE = contentType("application/json; charset=utf-8"); @@ -51,8 +52,9 @@ public abstract class AbstractHttpFixture { private final AtomicLong requests = new AtomicLong(0); /** Current working directory of the fixture **/ - private final Path workingDirectory; - private final int port; + private Path workingDirectory; + private int port; + private HttpServer httpServer; protected AbstractHttpFixture(final String workingDir) { this(workingDir, 0); @@ -63,6 +65,8 @@ public abstract class AbstractHttpFixture { this.workingDirectory = PathUtils.get(Objects.requireNonNull(workingDir)); } + public AbstractHttpFixture() {} + /** * Opens a {@link HttpServer} and start listening on a provided or random port. */ @@ -75,85 +79,100 @@ public abstract class AbstractHttpFixture { */ public final void listen(InetAddress inetAddress, boolean exposePidAndPort) throws IOException, InterruptedException { final InetSocketAddress socketAddress = new InetSocketAddress(inetAddress, port); - final HttpServer httpServer = HttpServer.create(socketAddress, 0); + listenAndWait(socketAddress, exposePidAndPort); + } + public final void listenAndWait(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException { try { - if (exposePidAndPort) { - /// Writes the PID of the current Java process in a `pid` file located in the working directory - writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]); - - final String addressAndPort = addressToString(httpServer.getAddress()); - // Writes the address and port of the http server in a `ports` file located in the working directory - writeFile(workingDirectory, "ports", addressAndPort); - } - - httpServer.createContext("/", exchange -> { - try { - Response response; - - // Check if this is a request made by the AntFixture - final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); - if (userAgent != null - && userAgent.startsWith("Apache Ant") - && "GET".equals(exchange.getRequestMethod()) - && "/".equals(exchange.getRequestURI().getPath())) { - response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8)); - - } else { - try { - final long requestId = requests.getAndIncrement(); - final String method = exchange.getRequestMethod(); - - final Map headers = new HashMap<>(); - for (Map.Entry> header : exchange.getRequestHeaders().entrySet()) { - headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey())); - } - - final ByteArrayOutputStream body = new ByteArrayOutputStream(); - try (InputStream requestBody = exchange.getRequestBody()) { - final byte[] buffer = new byte[1024]; - int i; - while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) { - body.write(buffer, 0, i); - } - body.flush(); - } - - final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray()); - response = handle(request); - - } catch (Exception e) { - final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request"; - response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8)); - } - } - - if (response == null) { - response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); - } - - response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v))); - if (response.body.length > 0) { - exchange.sendResponseHeaders(response.status, response.body.length); - exchange.getResponseBody().write(response.body); - } else { - exchange.sendResponseHeaders(response.status, -1); - } - } finally { - exchange.close(); - } - }); - httpServer.start(); - + listen(socketAddress, exposePidAndPort); // Wait to be killed Thread.sleep(Long.MAX_VALUE); - } finally { + stop(); + } + } + + public final void listen(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException { + httpServer = HttpServer.create(socketAddress, 0); + if (exposePidAndPort) { + /// Writes the PID of the current Java process in a `pid` file located in the working directory + writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]); + + final String addressAndPort = addressToString(httpServer.getAddress()); + // Writes the address and port of the http server in a `ports` file located in the working directory + writeFile(workingDirectory, "ports", addressAndPort); + } + + httpServer.createContext("/", exchange -> { + try { + Response response; + + // Check if this is a request made by the AntFixture + final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null + && userAgent.startsWith("Apache Ant") + && "GET".equals(exchange.getRequestMethod()) + && "/".equals(exchange.getRequestURI().getPath())) { + response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8)); + + } else { + try { + final long requestId = requests.getAndIncrement(); + final String method = exchange.getRequestMethod(); + + final Map headers = new HashMap<>(); + for (Map.Entry> header : exchange.getRequestHeaders().entrySet()) { + headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey())); + } + + final ByteArrayOutputStream body = new ByteArrayOutputStream(); + try (InputStream requestBody = exchange.getRequestBody()) { + final byte[] buffer = new byte[1024]; + int i; + while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) { + body.write(buffer, 0, i); + } + body.flush(); + } + + final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray()); + response = handle(request); + + } catch (Exception e) { + final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request"; + response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8)); + } + } + + if (response == null) { + response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); + } + + response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v))); + if (response.body.length > 0) { + exchange.sendResponseHeaders(response.status, response.body.length); + exchange.getResponseBody().write(response.body); + } else { + exchange.sendResponseHeaders(response.status, -1); + } + } finally { + exchange.close(); + } + }); + httpServer.start(); + } + + protected abstract Response handle(Request request) throws IOException; + + protected void stop() { + if (httpServer != null) { httpServer.stop(0); } } - protected abstract Response handle(Request request) throws IOException; + public String getAddress() { + return "http://127.0.0.1:" + httpServer.getAddress().getPort(); + } @FunctionalInterface public interface RequestHandler {