github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Port repository-url to new TestFramework (#102588)

Browse source 
removes docker-compose dependency from url-fixture
This commit is contained in:
Rene Groeschke 2023-12-01 11:23:53 +01:00 • committed by GitHub
parent bc0751d392
commit bc17acdada
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 156 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

19
modules/repository-url/build.gradle
View file

@ -8,12 +8,9 @@
import org.elasticsearch.gradle.PropertyNormalization
apply plugin: 'elasticsearch.legacy-yaml-rest-test'
apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
apply plugin: 'elasticsearch.internal-yaml-rest-test'
apply plugin: 'elasticsearch.yaml-rest-compat-test'
apply plugin: 'elasticsearch.internal-cluster-test'
apply plugin: 'elasticsearch.test.fixtures'
final Project fixture = project(':test:fixtures:url-fixture')
esplugin {
description 'Module for URL repository'
@ -32,6 +29,8 @@ dependencies {
api "commons-logging:commons-logging:${versions.commonslogging}"
api "commons-codec:commons-codec:${versions.commonscodec}"
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
yamlRestTestImplementation project(':test:fixtures:url-fixture')
internalClusterTestImplementation project(':test:fixtures:url-fixture')
}
tasks.named("thirdPartyAudit").configure {
@ -45,15 +44,7 @@ tasks.named("thirdPartyAudit").configure {
)
}
testFixtures.useFixture(fixture.path, 'url-fixture')
def fixtureAddress = { fixtureName ->
int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
File repositoryDir = fixture.fsRepositoryDir as File
//File repositoryDir = fixture.fsRepositoryDir as File
testClusters.configureEach {
// repositoryDir is used by a FS repository to create snapshots

22
modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java
View file

@ -8,6 +8,8 @@
package org.elasticsearch.repositories.url;
import fixture.url.URLFixture;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@ -22,11 +24,15 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.core.PathUtils;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import java.io.IOException;
import java.net.InetAddress;
@ -42,6 +48,22 @@ import static org.hamcrest.Matchers.notNullValue;
public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
public static final URLFixture urlFixture = new URLFixture();
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.module("repository-url")
.setting("path.repo", urlFixture::getRepositoryDir)
.setting("repositories.url.allowed_urls", () -> "http://snapshot.test*, " + urlFixture.getAddress())
.build();
@ClassRule
public static TestRule ruleChain = RuleChain.outerRule(urlFixture).around(cluster);
@Override
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}
public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}

14
test/fixtures/url-fixture/Dockerfile vendored
View file

@ -1,14 +0,0 @@
FROM openjdk:17.0.2
ARG port
ARG workingDir
ARG repositoryDir
ENV URL_FIXTURE_PORT=${port}
ENV URL_FIXTURE_WORKING_DIR=${workingDir}
ENV URL_FIXTURE_REPO_DIR=${repositoryDir}
ENTRYPOINT exec java -classpath "/fixture/shared/*" \
fixture.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR"
EXPOSE $port

21
test/fixtures/url-fixture/build.gradle vendored
View file

@ -6,30 +6,9 @@
* Side Public License, v 1.
*/
apply plugin: 'elasticsearch.java'
apply plugin: 'elasticsearch.test.fixtures'
description = 'Fixture for URL external service'
tasks.named("test").configure { enabled = false }
dependencies {
api project(':server')
api project(':test:framework')
}
// These directories are shared between the URL repository and the FS repository in integration tests
project.ext {
fsRepositoryDir = file("${testFixturesDir}/fs-repository")
}
tasks.named("preProcessFixture").configure {
dependsOn "jar", configurations.runtimeClasspath
doLast {
file("${testFixturesDir}/shared").mkdirs()
project.copy {
from jar
from configurations.runtimeClasspath
into "${testFixturesDir}/shared"
}
project.fsRepositoryDir.mkdirs()
}
}

15
test/fixtures/url-fixture/docker-compose.yml vendored
View file

@ -1,15 +0,0 @@
version: '3'
services:
url-fixture:
build:
context: .
args:
port: 80
workingDir: "/fixture/work"
repositoryDir: "/fixture/repo"
volumes:
- ./testfixtures_shared/shared:/fixture/shared
- ./testfixtures_shared/fs-repository:/fixture/repo
- ./testfixtures_shared/work:/fixture/work
ports:
- "80"

63
test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java vendored
View file

@ -7,15 +7,17 @@
*/
package fixture.url;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.fixture.AbstractHttpFixture;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
@ -25,32 +27,17 @@ import java.util.regex.Pattern;
* This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
* integration tests to expose a directory created by a regular FS repository.
*/
public class URLFixture extends AbstractHttpFixture {
public class URLFixture extends AbstractHttpFixture implements TestRule {
private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)$");
private final Path repositoryDir;
private final TemporaryFolder temporaryFolder;
private Path repositoryDir;
/**
* Creates a {@link URLFixture}
*/
private URLFixture(final int port, final String workingDir, final String repositoryDir) {
super(workingDir, port);
this.repositoryDir = dir(repositoryDir);
}
public static void main(String[] args) throws Exception {
if (args == null || args.length != 3) {
throw new IllegalArgumentException("URLFixture <port> <working directory> <repository directory>");
}
String workingDirectory = args[1];
if (Files.exists(dir(workingDirectory)) == false) {
throw new IllegalArgumentException("Configured working directory " + workingDirectory + " does not exist");
}
String repositoryDirectory = args[2];
if (Files.exists(dir(repositoryDirectory)) == false) {
throw new IllegalArgumentException("Configured repository directory " + repositoryDirectory + " does not exist");
}
final URLFixture fixture = new URLFixture(Integer.parseInt(args[0]), workingDirectory, repositoryDirectory);
fixture.listen(InetAddress.getByName("0.0.0.0"), false);
public URLFixture() {
super();
this.temporaryFolder = new TemporaryFolder();
}
@Override
@ -107,8 +94,32 @@ public class URLFixture extends AbstractHttpFixture {
}
}
@SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
private static Path dir(final String dir) {
return Paths.get(dir);
@Override
protected void before() throws Throwable {
this.temporaryFolder.create();
this.repositoryDir = temporaryFolder.newFolder("repoDir").toPath();
InetSocketAddress inetSocketAddress = resolveAddress("0.0.0.0", 0);
listen(inetSocketAddress, false);
}
public String getRepositoryDir() {
if (repositoryDir == null) {
throw new IllegalStateException("Rule has not been started yet");
}
return repositoryDir.toFile().getAbsolutePath();
}
private static InetSocketAddress resolveAddress(String address, int port) {
try {
return new InetSocketAddress(InetAddress.getByName(address), port);
} catch (UnknownHostException e) {
throw new RuntimeException(e);
}
}
@Override
protected void after() {
super.stop();
this.temporaryFolder.delete();
}
}

165
test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java
View file

@ -12,6 +12,7 @@ import com.sun.net.httpserver.HttpServer;
import org.elasticsearch.core.PathUtils;
import org.elasticsearch.core.SuppressForbidden;
import org.junit.rules.ExternalResource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -40,7 +41,7 @@ import static java.util.Collections.singletonMap;
* Base class for test fixtures that requires a {@link HttpServer} to work.
*/
@SuppressForbidden(reason = "uses httpserver by design")
public abstract class AbstractHttpFixture {
public abstract class AbstractHttpFixture extends ExternalResource {
protected static final Map<String, String> TEXT_PLAIN_CONTENT_TYPE = contentType("text/plain; charset=utf-8");
protected static final Map<String, String> JSON_CONTENT_TYPE = contentType("application/json; charset=utf-8");
@ -51,8 +52,9 @@ public abstract class AbstractHttpFixture {
private final AtomicLong requests = new AtomicLong(0);
/** Current working directory of the fixture **/
private final Path workingDirectory;
private final int port;
private Path workingDirectory;
private int port;
private HttpServer httpServer;
protected AbstractHttpFixture(final String workingDir) {
this(workingDir, 0);
@ -63,6 +65,8 @@ public abstract class AbstractHttpFixture {
this.workingDirectory = PathUtils.get(Objects.requireNonNull(workingDir));
}
public AbstractHttpFixture() {}
/**
* Opens a {@link HttpServer} and start listening on a provided or random port.
*/
@ -75,85 +79,100 @@ public abstract class AbstractHttpFixture {
*/
public final void listen(InetAddress inetAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
final InetSocketAddress socketAddress = new InetSocketAddress(inetAddress, port);
final HttpServer httpServer = HttpServer.create(socketAddress, 0);
listenAndWait(socketAddress, exposePidAndPort);
}
public final void listenAndWait(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
try {
if (exposePidAndPort) {
/// Writes the PID of the current Java process in a `pid` file located in the working directory
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
final String addressAndPort = addressToString(httpServer.getAddress());
// Writes the address and port of the http server in a `ports` file located in the working directory
writeFile(workingDirectory, "ports", addressAndPort);
}
httpServer.createContext("/", exchange -> {
try {
Response response;
// Check if this is a request made by the AntFixture
final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
if (userAgent != null
&& userAgent.startsWith("Apache Ant")
&& "GET".equals(exchange.getRequestMethod())
&& "/".equals(exchange.getRequestURI().getPath())) {
response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8));
} else {
try {
final long requestId = requests.getAndIncrement();
final String method = exchange.getRequestMethod();
final Map<String, String> headers = new HashMap<>();
for (Map.Entry<String, List<String>> header : exchange.getRequestHeaders().entrySet()) {
headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey()));
}
final ByteArrayOutputStream body = new ByteArrayOutputStream();
try (InputStream requestBody = exchange.getRequestBody()) {
final byte[] buffer = new byte[1024];
int i;
while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) {
body.write(buffer, 0, i);
}
body.flush();
}
final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray());
response = handle(request);
} catch (Exception e) {
final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request";
response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8));
}
}
if (response == null) {
response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
}
response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v)));
if (response.body.length > 0) {
exchange.sendResponseHeaders(response.status, response.body.length);
exchange.getResponseBody().write(response.body);
} else {
exchange.sendResponseHeaders(response.status, -1);
}
} finally {
exchange.close();
}
});
httpServer.start();
listen(socketAddress, exposePidAndPort);
// Wait to be killed
Thread.sleep(Long.MAX_VALUE);
} finally {
stop();
}
}
public final void listen(InetSocketAddress socketAddress, boolean exposePidAndPort) throws IOException, InterruptedException {
httpServer = HttpServer.create(socketAddress, 0);
if (exposePidAndPort) {
/// Writes the PID of the current Java process in a `pid` file located in the working directory
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
final String addressAndPort = addressToString(httpServer.getAddress());
// Writes the address and port of the http server in a `ports` file located in the working directory
writeFile(workingDirectory, "ports", addressAndPort);
}
httpServer.createContext("/", exchange -> {
try {
Response response;
// Check if this is a request made by the AntFixture
final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
if (userAgent != null
&& userAgent.startsWith("Apache Ant")
&& "GET".equals(exchange.getRequestMethod())
&& "/".equals(exchange.getRequestURI().getPath())) {
response = new Response(200, TEXT_PLAIN_CONTENT_TYPE, "OK".getBytes(UTF_8));
} else {
try {
final long requestId = requests.getAndIncrement();
final String method = exchange.getRequestMethod();
final Map<String, String> headers = new HashMap<>();
for (Map.Entry<String, List<String>> header : exchange.getRequestHeaders().entrySet()) {
headers.put(header.getKey(), exchange.getRequestHeaders().getFirst(header.getKey()));
}
final ByteArrayOutputStream body = new ByteArrayOutputStream();
try (InputStream requestBody = exchange.getRequestBody()) {
final byte[] buffer = new byte[1024];
int i;
while ((i = requestBody.read(buffer, 0, buffer.length)) != -1) {
body.write(buffer, 0, i);
}
body.flush();
}
final Request request = new Request(requestId, method, exchange.getRequestURI(), headers, body.toByteArray());
response = handle(request);
} catch (Exception e) {
final String error = e.getMessage() != null ? e.getMessage() : "Exception when processing the request";
response = new Response(500, singletonMap("Content-Type", "text/plain; charset=utf-8"), error.getBytes(UTF_8));
}
}
if (response == null) {
response = new Response(400, TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
}
response.headers.forEach((k, v) -> exchange.getResponseHeaders().put(k, singletonList(v)));
if (response.body.length > 0) {
exchange.sendResponseHeaders(response.status, response.body.length);
exchange.getResponseBody().write(response.body);
} else {
exchange.sendResponseHeaders(response.status, -1);
}
} finally {
exchange.close();
}
});
httpServer.start();
}
protected abstract Response handle(Request request) throws IOException;
protected void stop() {
if (httpServer != null) {
httpServer.stop(0);
}
}
protected abstract Response handle(Request request) throws IOException;
public String getAddress() {
return "http://127.0.0.1:" + httpServer.getAddress().getPort();
}
@FunctionalInterface
public interface RequestHandler {