github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Build] Make test cluster plugin configuration cache compatible (#116890)

Browse source 
This adds infrastructure to make the legacy test cluster plugin and
the legacy test cluster based test plugins generally configuration cache compatible.
This commit is contained in:
Rene Groeschke 2024-11-21 08:00:05 +01:00 • committed by GitHub
parent ea4b41fca8
commit 8c20ac5884
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 561 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

17
build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy
View file

@ -15,16 +15,12 @@ import org.elasticsearch.gradle.internal.test.AntFixture
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.file.ProjectLayout
import org.gradle.api.provider.ProviderFactory
import org.gradle.api.tasks.Internal
import org.gradle.process.ExecOperations
import javax.inject.Inject
abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
@Internal
AntFixture fixture
@Inject
AntFixtureStop(ProjectLayout projectLayout,
ExecOperations execOperations,
@ -34,12 +30,12 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
}
void setFixture(AntFixture fixture) {
assert this.fixture == null
this.fixture = fixture;
final Object pid = "${-> this.fixture.pid}"
onlyIf("pidFile exists") { fixture.pidFile.exists() }
def pidFile = fixture.pidFile
def fixtureName = fixture.name
final Object pid = "${-> Integer.parseInt(pidFile.getText('UTF-8').trim())}"
onlyIf("pidFile exists") { pidFile.exists() }
doFirst {
logger.info("Shutting down ${fixture.name} with pid ${pid}")
logger.info("Shutting down ${fixtureName} with pid ${pid}")
}
if (OS.current() == OS.WINDOWS) {
@ -51,9 +47,8 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
}
doLast {
fileSystemOperations.delete {
it.delete(fixture.pidFile)
it.delete(pidFile)
}
}
this.fixture = fixture
}
}

10
build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy
View file

@ -29,11 +29,6 @@ import java.nio.charset.Charset
*/
public abstract class AntTask extends DefaultTask {
/**
* A buffer that will contain the output of the ant code run,
* if the output was not already written directly to stdout.
*/
public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
@Inject
protected FileSystemOperations getFileSystemOperations() {
@ -57,6 +52,11 @@ public abstract class AntTask extends DefaultTask {
// otherwise groovy replaces System.out, and you have no chance to debug
// ant.saveStreams = false
/**
* A buffer that will contain the output of the ant code run,
* if the output was not already written directly to stdout.
*/
ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())

60
build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy
View file

@ -10,22 +10,37 @@
package org.elasticsearch.gradle.internal.test
import org.elasticsearch.gradle.OS
import org.elasticsearch.gradle.internal.AntFixtureStop
import org.elasticsearch.gradle.internal.AntTask
import org.elasticsearch.gradle.testclusters.TestClusterInfo
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.gradle.api.GradleException
import org.gradle.api.file.ProjectLayout
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.provider.ProviderFactory
import org.gradle.api.provider.ValueSource
import org.gradle.api.provider.ValueSourceParameters
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskProvider
import javax.inject.Inject
/**
* A fixture for integration tests which runs in a separate process launched by Ant.
*/
class AntFixture extends AntTask implements Fixture {
class AntFixture extends AntTask {
/** The path to the executable that starts the fixture. */
@Internal
String executable
private final List<Object> arguments = new ArrayList<>()
private ProjectLayout projectLayout
private final ProviderFactory providerFactory
void args(Object... args) {
arguments.addAll(args)
@ -69,19 +84,14 @@ class AntFixture extends AntTask implements Fixture {
return tmpFile.exists()
}
private final TaskProvider<AntFixtureStop> stopTask
AntFixture() {
stopTask = createStopTask()
@Inject
AntFixture(ProjectLayout projectLayout, ProviderFactory providerFactory) {
this.providerFactory = providerFactory
this.projectLayout = projectLayout;
TaskProvider<AntFixtureStop> stopTask = createStopTask()
finalizedBy(stopTask)
}
@Override
@Internal
TaskProvider<AntFixtureStop> getStopTask() {
return stopTask
}
@Override
protected void runAnt(AntBuilder ant) {
// reset everything
@ -231,7 +241,7 @@ class AntFixture extends AntTask implements Fixture {
*/
@Internal
protected File getBaseDir() {
return new File(project.buildDir, "fixtures/${name}")
return new File(projectLayout.getBuildDirectory().getAsFile().get(), "fixtures/${name}")
}
/** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
@ -242,7 +252,7 @@ class AntFixture extends AntTask implements Fixture {
/** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
@Internal
protected File getPidFile() {
File getPidFile() {
return new File(baseDir, 'pid')
}
@ -264,6 +274,12 @@ class AntFixture extends AntTask implements Fixture {
return portsFile.readLines("UTF-8").get(0)
}
@Internal
Provider<String> getAddressAndPortProvider() {
File thePortFile = portsFile
return providerFactory.provider(() -> thePortFile.readLines("UTF-8").get(0))
}
/** Returns a file that wraps around the actual command when {@code spawn == true}. */
@Internal
protected File getWrapperScript() {
@ -281,4 +297,22 @@ class AntFixture extends AntTask implements Fixture {
protected File getRunLog() {
return new File(cwd, 'run.log')
}
@Internal
Provider<AntFixtureValueSource> getAddressAndPortSource() {
return providerFactory.of(AntFixtureValueSource.class, spec -> {
spec.getParameters().getPortFile().set(portsFile);
});
}
static abstract class AntFixtureValueSource implements ValueSource<String, AntFixtureValueSource.Parameters> {
@Override
String obtain() {
return getParameters().getPortFile().map { it.readLines("UTF-8").get(0) }.get()
}
interface Parameters extends ValueSourceParameters {
Property<File> getPortFile();
}
}
}

21
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java
View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.gradle.internal.test;
/**
* Any object that can produce an accompanying stop task, meant to tear down
* a previously instantiated service.
*/
public interface Fixture {
/** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */
Object getStopTask();
}

4
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
View file

@ -26,6 +26,7 @@ import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.Directory;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.file.RelativePath;
import org.gradle.api.internal.file.FileOperations;
@ -244,10 +245,11 @@ public abstract class AbstractYamlRestCompatTestPlugin implements Plugin<Project
yamlRestCompatTestTask.configure(testTask -> {
testTask.systemProperty("tests.restCompat", true);
// Use test runner and classpath from "normal" yaml source set
FileCollection outputFileCollection = yamlCompatTestSourceSet.getOutput();
testTask.setTestClassesDirs(
yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
);
testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false);
testTask.onlyIf("Compatibility tests are available", t -> outputFileCollection.isEmpty() == false);
testTask.setClasspath(
yamlCompatTestSourceSet.getRuntimeClasspath()
// remove the "normal" api and tests

15
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
View file

@ -76,6 +76,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
private final transient Project project;
private final Provider<ReaperService> reaper;
private final Provider<TestClustersRegistry> testClustersRegistryProvider;
private final FileSystemOperations fileSystemOperations;
private final ArchiveOperations archiveOperations;
private final ExecOperations execOperations;
@ -87,11 +88,14 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
private boolean shared = false;
private int claims = 0;
public ElasticsearchCluster(
String path,
String clusterName,
Project project,
Provider<ReaperService> reaper,
Provider<TestClustersRegistry> testClustersRegistryProvider,
FileSystemOperations fileSystemOperations,
ArchiveOperations archiveOperations,
ExecOperations execOperations,
@ -104,6 +108,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
this.clusterName = clusterName;
this.project = project;
this.reaper = reaper;
this.testClustersRegistryProvider = testClustersRegistryProvider;
this.fileSystemOperations = fileSystemOperations;
this.archiveOperations = archiveOperations;
this.execOperations = execOperations;
@ -120,6 +125,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
clusterName + "-0",
project,
reaper,
testClustersRegistryProvider,
fileSystemOperations,
archiveOperations,
execOperations,
@ -177,6 +183,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
clusterName + "-" + i,
project,
reaper,
testClustersRegistryProvider,
fileSystemOperations,
archiveOperations,
execOperations,
@ -408,6 +415,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
public void freeze() {
nodes.forEach(ElasticsearchNode::freeze);
configurationFrozen.set(true);
nodes.whenObjectAdded(node -> { throw new IllegalStateException("Cannot add nodes to test cluster after is has been frozen"); });
}
private void checkFrozen() {
@ -663,4 +671,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
return "cluster{" + path + ":" + clusterName + "}";
}
int addClaim() {
return ++this.claims;
}
int removeClaim() {
return --this.claims;
}
}

15
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View file

@ -124,6 +124,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final String name;
transient private final Project project;
private final Provider<ReaperService> reaperServiceProvider;
private final Provider<TestClustersRegistry> testClustersRegistryProvider;
private final FileSystemOperations fileSystemOperations;
private final ArchiveOperations archiveOperations;
private final ExecOperations execOperations;
@ -164,7 +166,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final List<ElasticsearchDistribution> distributions = new ArrayList<>();
private int currentDistro = 0;
private TestDistribution testDistribution;
private volatile Process esProcess;
private Function<String, String> nameCustomization = s -> s;
private boolean isWorkingDirConfigured = false;
private String httpPort = "0";
@ -179,6 +180,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
String name,
Project project,
Provider<ReaperService> reaperServiceProvider,
Provider<TestClustersRegistry> testClustersRegistryProvider,
FileSystemOperations fileSystemOperations,
ArchiveOperations archiveOperations,
ExecOperations execOperations,
@ -191,6 +193,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
this.name = name;
this.project = project;
this.reaperServiceProvider = reaperServiceProvider;
this.testClustersRegistryProvider = testClustersRegistryProvider;
this.fileSystemOperations = fileSystemOperations;
this.archiveOperations = archiveOperations;
this.execOperations = execOperations;
@ -892,11 +895,13 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
}
LOGGER.info("Running `{}` in `{}` for {} env: {}", command, workingDir, this, environment);
Process esProcess;
try {
esProcess = processBuilder.start();
} catch (IOException e) {
throw new TestClustersException("Failed to start ES process for " + this, e);
}
testClustersRegistryProvider.get().storeProcess(id(), esProcess);
reaperServiceProvider.get().registerPid(toString(), esProcess.pid());
}
@ -982,6 +987,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Process esProcess = testClustersRegistryProvider.get().getProcess(id());
if (esProcess == null && tailLogs) {
// This is a special case. If start() throws an exception the plugin will still call stop
// Another exception here would eat the orriginal.
@ -1574,6 +1580,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
@Override
@Internal
public boolean isProcessAlive() {
Process esProcess = testClustersRegistryProvider.get().getProcess(id());
requireNonNull(esProcess, "Can't wait for `" + this + "` as it's not started. Does the task have `useCluster` ?");
return esProcess.isAlive();
}
@ -1602,6 +1609,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
@Override
public String toString() {
return id() + " (" + System.identityHashCode(this) + ")";
}
private String id() {
return "node{" + path + ":" + name + "}";
}
@ -1702,7 +1713,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
}
private record FeatureFlag(String feature, Version from, Version until) {
public record FeatureFlag(String feature, Version from, Version until) {
@Input
public String getFeature() {

36
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterInfo.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.gradle.testclusters;
import java.io.File;
import java.util.List;
public class TestClusterInfo {
private final List<String> allHttpSocketURI;
private final List<String> allTransportPortURI;
private final List<File> auditLogs;
public TestClusterInfo(List<String> allHttpSocketURI, List<String> allTransportPortURI, List<File> auditLogs) {
this.allHttpSocketURI = allHttpSocketURI;
this.allTransportPortURI = allTransportPortURI;
this.auditLogs = auditLogs;
}
public List<String> getAllHttpSocketURI() {
return allHttpSocketURI;
}
public List<String> getAllTransportPortURI() {
return allTransportPortURI;
}
public List<File> getAuditLogs() {
return auditLogs;
}
}

34
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterValueSource.java Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.gradle.testclusters;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.ValueSource;
import org.gradle.api.provider.ValueSourceParameters;
import org.jetbrains.annotations.Nullable;
public abstract class TestClusterValueSource implements ValueSource<TestClusterInfo, TestClusterValueSource.Parameters> {
@Nullable
@Override
public TestClusterInfo obtain() {
String clusterName = getParameters().getClusterName().get();
String path = getParameters().getPath().get();
return getParameters().getService().get().getClusterDetails(path, clusterName);
}
interface Parameters extends ValueSourceParameters {
Property<String> getClusterName();
Property<String> getPath();
Property<TestClustersRegistry> getService();
}
}

12
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java
View file

@ -8,6 +8,7 @@
*/
package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.gradle.ElasticsearchDistribution;
import org.gradle.api.Task;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
@ -34,10 +35,15 @@ public interface TestClustersAware extends Task {
if (cluster.getPath().equals(getProject().getPath()) == false) {
throw new TestClustersException("Task " + getPath() + " can't use test cluster from" + " another project " + cluster);
}
cluster.getNodes()
.all(node -> node.getDistributions().forEach(distro -> dependsOn(getProject().provider(() -> distro.maybeFreeze()))));
if (cluster.getName().equals(getName())) {
for (ElasticsearchNode node : cluster.getNodes()) {
for (ElasticsearchDistribution distro : node.getDistributions()) {
ElasticsearchDistribution frozenDistro = distro.maybeFreeze();
dependsOn(frozenDistro);
}
}
dependsOn(cluster.getPluginAndModuleConfigurations());
}
getClusters().add(cluster);
}

47
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
View file

@ -26,6 +26,7 @@ import org.gradle.api.internal.file.FileOperations;
import org.gradle.api.invocation.Gradle;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.services.BuildService;
@ -106,15 +107,22 @@ public class TestClustersPlugin implements Plugin<Project> {
runtimeJavaProvider = providerFactory.provider(
() -> System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME"))
);
// register cluster registry as a global build service
Provider<TestClustersRegistry> testClustersRegistryProvider = project.getGradle()
.getSharedServices()
.registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, noop());
// enable the DSL to describe clusters
NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project, reaperServiceProvider);
NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(
project,
testClustersRegistryProvider,
reaperServiceProvider
);
// provide a task to be able to list defined clusters.
createListClustersTask(project, container);
// register cluster registry as a global build service
project.getGradle().getSharedServices().registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, noop());
// register throttle so we only run at most max-workers/2 nodes concurrently
Provider<TestClustersThrottle> testClustersThrottleProvider = project.getGradle()
.getSharedServices()
@ -145,6 +153,7 @@ public class TestClustersPlugin implements Plugin<Project> {
private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(
Project project,
Provider<TestClustersRegistry> testClustersRegistryProvider,
Provider<ReaperService> reaper
) {
// Create an extensions that allows describing clusters
@ -155,6 +164,7 @@ public class TestClustersPlugin implements Plugin<Project> {
name,
project,
reaper,
testClustersRegistryProvider,
getFileSystemOperations(),
getArchiveOperations(),
getExecOperations(),
@ -199,7 +209,9 @@ public class TestClustersPlugin implements Plugin<Project> {
Provider<TaskEventsService> testClusterTasksService = project.getGradle()
.getSharedServices()
.registerIfAbsent(TEST_CLUSTER_TASKS_SERVICE, TaskEventsService.class, spec -> {});
.registerIfAbsent(TEST_CLUSTER_TASKS_SERVICE, TaskEventsService.class, spec -> {
spec.getParameters().getRegistry().set(registryProvider);
});
TestClustersRegistry registry = registryProvider.get();
// When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
@ -209,7 +221,7 @@ public class TestClustersPlugin implements Plugin<Project> {
configureClaimClustersHook(project.getGradle(), registry);
// Before each task, we determine if a cluster needs to be started for that task.
configureStartClustersHook(project.getGradle(), registry, testClusterTasksService);
configureStartClustersHook(project.getGradle());
// After each task we determine if there are clusters that are no longer needed.
getEventsListenerRegistry().onTaskCompletion(testClusterTasksService);
@ -228,12 +240,7 @@ public class TestClustersPlugin implements Plugin<Project> {
});
}
private void configureStartClustersHook(
Gradle gradle,
TestClustersRegistry registry,
Provider<TaskEventsService> testClusterTasksService
) {
testClusterTasksService.get().registry(registry);
private void configureStartClustersHook(Gradle gradle) {
gradle.getTaskGraph().whenReady(taskExecutionGraph -> {
taskExecutionGraph.getAllTasks()
.stream()
@ -249,19 +256,14 @@ public class TestClustersPlugin implements Plugin<Project> {
}
}
static public abstract class TaskEventsService implements BuildService<BuildServiceParameters.None>, OperationCompletionListener {
static public abstract class TaskEventsService implements BuildService<TaskEventsService.Params>, OperationCompletionListener {
Map<String, TestClustersAware> tasksMap = new HashMap<>();
private TestClustersRegistry registryProvider;
public void register(TestClustersAware task) {
tasksMap.put(task.getPath(), task);
}
public void registry(TestClustersRegistry registry) {
this.registryProvider = registry;
}
@Override
public void onFinish(FinishEvent finishEvent) {
if (finishEvent instanceof TaskFinishEvent taskFinishEvent) {
@ -273,11 +275,18 @@ public class TestClustersPlugin implements Plugin<Project> {
if (task.getDidWork()) {
task.getClusters()
.forEach(
cluster -> registryProvider.stopCluster(cluster, taskFinishEvent.getResult() instanceof TaskFailureResult)
cluster -> getParameters().getRegistry()
.get()
.stopCluster(cluster, taskFinishEvent.getResult() instanceof TaskFailureResult)
);
}
}
}
}
// Some parameters for the web server
interface Params extends BuildServiceParameters {
Property<TestClustersRegistry> getRegistry();
}
}
}

56
build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java
View file

@ -10,6 +10,8 @@ package org.elasticsearch.gradle.testclusters;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.services.BuildService;
import org.gradle.api.services.BuildServiceParameters;
@ -17,20 +19,23 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
public abstract class TestClustersRegistry implements BuildService<BuildServiceParameters.None> {
private static final Logger logger = Logging.getLogger(TestClustersRegistry.class);
private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure";
private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false"));
private final Map<ElasticsearchCluster, Integer> claimsInventory = new HashMap<>();
private final Set<ElasticsearchCluster> runningClusters = new HashSet<>();
private final Map<String, Process> nodeProcesses = new HashMap<>();
@Inject
public abstract ProviderFactory getProviderFactory();
public void claimCluster(ElasticsearchCluster cluster) {
cluster.freeze();
int claim = claimsInventory.getOrDefault(cluster, 0) + 1;
claimsInventory.put(cluster, claim);
if (claim > 1) {
int claims = cluster.addClaim();
if (claims > 1) {
cluster.setShared(true);
}
}
@ -43,6 +48,13 @@ public abstract class TestClustersRegistry implements BuildService<BuildServiceP
cluster.start();
}
public Provider<TestClusterInfo> getClusterInfo(String clusterName) {
return getProviderFactory().of(TestClusterValueSource.class, spec -> {
spec.getParameters().getService().set(TestClustersRegistry.this);
spec.getParameters().getClusterName().set(clusterName);
});
}
public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) {
if (taskFailed) {
// If the task fails, and other tasks use this cluster, the other task will likely never be
@ -67,8 +79,7 @@ public abstract class TestClustersRegistry implements BuildService<BuildServiceP
runningClusters.remove(cluster);
}
} else {
int currentClaims = claimsInventory.getOrDefault(cluster, 0) - 1;
claimsInventory.put(cluster, currentClaims);
int currentClaims = cluster.removeClaim();
if (currentClaims <= 0 && runningClusters.contains(cluster)) {
cluster.stop(false);
runningClusters.remove(cluster);
@ -76,4 +87,33 @@ public abstract class TestClustersRegistry implements BuildService<BuildServiceP
}
}
public TestClusterInfo getClusterDetails(String path, String clusterName) {
ElasticsearchCluster cluster = runningClusters.stream()
.filter(c -> c.getPath().equals(path))
.filter(c -> c.getName().equals(clusterName))
.findFirst()
.orElseThrow();
return new TestClusterInfo(
cluster.getAllHttpSocketURI(),
cluster.getAllTransportPortURI(),
cluster.getNodes().stream().map(n -> n.getAuditLog()).collect(Collectors.toList())
);
}
public void restart(String path, String clusterName) {
ElasticsearchCluster cluster = runningClusters.stream()
.filter(c -> c.getPath().equals(path))
.filter(c -> c.getName().equals(clusterName))
.findFirst()
.orElseThrow();
cluster.restart();
}
public void storeProcess(String id, Process esProcess) {
nodeProcesses.put(id, esProcess);
}
public Process getProcess(String id) {
return nodeProcesses.get(id);
}
}

22
plugins/discovery-ec2/qa/amazon-ec2/build.gradle
View file

@ -8,7 +8,6 @@
*/
import org.apache.tools.ant.filters.ReplaceTokens
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.test.AntFixture
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
@ -55,7 +54,8 @@ tasks.named("yamlRestTest").configure { enabled = false }
['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action ->
TaskProvider<AntFixture> fixture = tasks.register("ec2Fixture${action}", AntFixture) {
dependsOn project.sourceSets.yamlRestTest.runtimeClasspath
env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}"
FileCollection cp = project.sourceSets.yamlRestTest.runtimeClasspath
env 'CLASSPATH', "${-> cp.asPath}"
executable = "${buildParams.runtimeJavaHome.get() }/bin/java"
args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt"
}
@ -68,9 +68,18 @@ tasks.named("yamlRestTest").configure { enabled = false }
classpath = yamlRestTestSourceSet.getRuntimeClasspath()
}
if(action == 'ContainerCredentials') {
def addressAndPortSource = fixture.get().addressAndPortSource
testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach {
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
() -> addressAndPortSource.map{ addr -> "http://${addr}/ecs_credentials_endpoint" }.get(), IGNORE_VALUE
}
}
tasks.named("check").configure {
dependsOn(yamlRestTestTask)
}
def addressAndPortSource = fixture.get().addressAndPortSource
testClusters.matching { it.name == yamlRestTestTask.name}.configureEach {
numberOfNodes = ec2NumberOfNodes
@ -78,9 +87,9 @@ tasks.named("yamlRestTest").configure { enabled = false }
setting 'discovery.seed_providers', 'ec2'
setting 'network.host', '_ec2_'
setting 'discovery.ec2.endpoint', { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
setting 'discovery.ec2.endpoint', { "http://${-> addressAndPortSource.get()}" }, IGNORE_VALUE
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> addressAndPortSource.get()}" }, IGNORE_VALUE
}
}
@ -107,11 +116,6 @@ tasks.named("ec2FixtureContainerCredentials").configure {
env 'ACTIVATE_CONTAINER_CREDENTIALS', true
}
testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach {
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
{ "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
// Extra config for InstanceProfile
tasks.named("ec2FixtureInstanceProfile").configure {
env 'ACTIVATE_INSTANCE_PROFILE', true

42
qa/mixed-cluster/build.gradle
View file

@ -11,6 +11,10 @@
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
apply plugin: 'elasticsearch.internal-testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
@ -63,6 +67,8 @@ excludeList.add('indices.resolve_index/20_resolve_system_index/*')
// Excluded because the error has changed
excludeList.add('aggregations/percentiles_hdr_metric/Negative values test')
def clusterPath = getPath()
buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
if (bwcVersion != VersionProperties.getElasticsearchVersion()) {
/* This project runs the core REST tests against a 4 node cluster where two of
@ -84,18 +90,42 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
useCluster baseCluster
mustRunAfter("precommit")
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def baseInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set(baseName)
it.parameters.service = serviceProvider
}.map { it.getAllHttpSocketURI() }
def baseInfoAfterOneNodeUpdate = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set(baseName)
it.parameters.service = serviceProvider
}.map { it.getAllHttpSocketURI() }
def baseInfoAfterTwoNodesUpdate = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set(baseName)
it.parameters.service = serviceProvider
}.map { it.getAllHttpSocketURI() }
def nonInputProps = nonInputProperties
def sharedRepoFolder = new File(buildDir, "cluster/shared/repo/${baseName}")
doFirst {
delete("${buildDir}/cluster/shared/repo/${baseName}")
delete(sharedRepoFolder)
// Getting the endpoints causes a wait for the cluster
println "Test cluster endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
println "Test cluster endpoints are: ${-> baseInfo.get().join(",")}"
println "Upgrading one node to create a mixed cluster"
baseCluster.get().nextNodeToNextVersion()
// Getting the endpoints causes a wait for the cluster
println "Upgrade complete, endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
println "Upgrade complete, endpoints are: ${-> baseInfoAfterOneNodeUpdate.get().join(",")}"
println "Upgrading another node to create a mixed cluster"
baseCluster.get().nextNodeToNextVersion()
nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
nonInputProperties.systemProperty('tests.clustername', baseName)
nonInputProps.systemProperty('tests.rest.cluster', baseInfoAfterTwoNodesUpdate.map(c -> c.join(",")))
nonInputProps.systemProperty('tests.clustername', baseName)
if (excludeList.isEmpty() == false) {
systemProperty 'tests.rest.blacklist', excludeList.join(',')
}
@ -103,7 +133,7 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
systemProperty 'tests.bwc_nodes_version', bwcVersion.toString().replace('-SNAPSHOT', '')
systemProperty 'tests.new_nodes_version', project.version.toString().replace('-SNAPSHOT', '')
onlyIf("BWC tests disabled") { project.bwc_tests_enabled }
// onlyIf("BWC tests disabled") { project.bwc_tests_enabled }
}
tasks.register(bwcTaskName(bwcVersion)) {

43
x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle
View file

@ -1,5 +1,9 @@
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.internal-testclusters'
@ -11,6 +15,8 @@ dependencies {
testImplementation project(':x-pack:plugin:ccr:qa')
}
def clusterPath = getPath()
def leaderCluster = testClusters.register("leader-cluster") {
testDistribution = 'DEFAULT'
setting 'xpack.license.self_generated.type', 'trial'
@ -24,7 +30,19 @@ def followCluster = testClusters.register("follow-cluster") {
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
user username: 'admin', password: 'admin-password', role: 'superuser'
setting 'cluster.remote.leader_cluster.seeds', { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}
def leaderUris = leaderInfo.map { it.getAllTransportPortURI() }
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE
}
tasks.register("leader-cluster", RestIntegTestTask) {
@ -41,7 +59,7 @@ tasks.register("writeJavaPolicy") {
policyFile.write(
[
"grant {",
" permission java.io.FilePermission \"${-> testClusters."follow-cluster".getFirstNode().getServerLog()}\", \"read\";",
" permission java.io.FilePermission \"${-> followCluster.map { it.getFirstNode().getServerLog() }.get()}\", \"read\";",
"};"
].join("\n")
)
@ -53,7 +71,24 @@ tasks.register("follow-cluster", RestIntegTestTask) {
useCluster leaderCluster
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'java.security.policy', "file://${policyFile}"
nonInputProperties.systemProperty 'tests.leader_host', leaderCluster.map(c -> c.allHttpSocketURI.get(0))
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}
def followInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("follow-cluster")
it.parameters.service = serviceProvider
}
def leaderUri = leaderInfo.map { it.getAllHttpSocketURI().get(0) }
def followerUri = followInfo.map { it.getAllHttpSocketURI().get(0) }
nonInputProperties.systemProperty 'tests.leader_host', leaderUri
nonInputProperties.systemProperty 'log', followCluster.map(c -> c.getFirstNode().getServerLog())
}

76
x-pack/plugin/ccr/qa/multi-cluster/build.gradle
View file

@ -1,6 +1,10 @@
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.internal-testclusters'
@ -12,6 +16,7 @@ dependencies {
testImplementation project(':x-pack:plugin:ccr:qa')
}
def clusterPath = getPath()
def leaderCluster = testClusters.register('leader-cluster') {
testDistribution = 'DEFAULT'
setting 'xpack.license.self_generated.type', 'trial'
@ -25,8 +30,19 @@ def middleCluster = testClusters.register('middle-cluster') {
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
user username: 'admin', password: 'admin-password', role: 'superuser'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}
def leaderUris = leaderInfo.map { it.getAllTransportPortURI() }
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE
{ "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE
}
tasks.register("leader-cluster", RestIntegTestTask) {
@ -40,9 +56,19 @@ tasks.register("middle-cluster", RestIntegTestTask) {
useCluster testClusters.named("leader-cluster")
systemProperty 'tests.target_cluster', 'middle'
systemProperty 'tests.leader_cluster_repository_path', "${buildDir}/cluster/shared/repo/leader-cluster"
nonInputProperties.systemProperty 'tests.leader_host',leaderCluster.map(c -> c.allHttpSocketURI.get(0))
}
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
nonInputProperties.systemProperty 'tests.leader_host', leaderUri
}
tasks.register('follow-cluster', RestIntegTestTask) {
dependsOn "leader-cluster", "middle-cluster"
@ -50,8 +76,25 @@ tasks.register('follow-cluster', RestIntegTestTask) {
useCluster middleCluster
systemProperty 'tests.target_cluster', 'follow'
systemProperty 'tests.leader_cluster_repository_path', "${buildDir}/cluster/shared/repo/leader-cluster"
nonInputProperties.systemProperty 'tests.leader_host', leaderCluster.map(c -> c.allHttpSocketURI.get(0))
nonInputProperties.systemProperty 'tests.middle_host', middleCluster.map(c -> c.allHttpSocketURI.get(0))
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
def middleUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("middle-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
nonInputProperties.systemProperty 'tests.leader_host', leaderUri
nonInputProperties.systemProperty 'tests.middle_host', middleUri
}
testClusters.matching { it.name == "follow-cluster" }.configureEach {
@ -60,10 +103,27 @@ testClusters.matching {it.name == "follow-cluster" }.configureEach {
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
user username: 'admin', password: 'admin-password', role: 'superuser'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUris = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.getAllTransportPortURI() }
def middleUris = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("middle-cluster")
it.parameters.service = serviceProvider
}.map { it.getAllTransportPortURI() }
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE
{ "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE
setting 'cluster.remote.middle_cluster.seeds',
{ "\"${middleCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE
{ "\"${middleUris.get().join(",")}\"" }, IGNORE_VALUE
}

34
x-pack/plugin/ccr/qa/non-compliant-license/build.gradle
View file

@ -1,5 +1,9 @@
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.internal-testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
@ -10,6 +14,8 @@ dependencies {
testImplementation project(':x-pack:plugin:ccr:qa:')
}
def clusterPath = getPath()
def leaderCluster = testClusters.register('leader-cluster') {
testDistribution = 'DEFAULT'
setting 'xpack.security.enabled', 'true'
@ -21,8 +27,20 @@ def followerCluster = testClusters.register('follow-cluster') {
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
user username: 'admin', password: 'admin-password', role: 'superuser'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}
def leaderUris = leaderInfo.map { it.getAllTransportPortURI() }
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE
{ "\"${leaderUris.get().join(",")}\"" }, IGNORE_VALUE
}
tasks.register('leader-cluster', RestIntegTestTask) {
@ -34,7 +52,19 @@ tasks.register('follow-cluster', RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster leaderCluster
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', followerCluster.map(c -> c.allHttpSocketURI.get(0))
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def followInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("follow-cluster")
it.parameters.service = serviceProvider
}
def followUri = followInfo.map { it.allHttpSocketURI.get(0) }
nonInputProperties.systemProperty 'tests.leader_host', followUri
}
tasks.named("check").configure { dependsOn "follow-cluster" }

57
x-pack/plugin/ccr/qa/restart/build.gradle
View file

@ -1,6 +1,10 @@
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.internal-testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
@ -9,6 +13,8 @@ dependencies {
testImplementation project(':x-pack:plugin:ccr:qa')
}
def clusterPath = getPath()
def leaderCluster = testClusters.register('leader-cluster') {
testDistribution = 'DEFAULT'
setting 'xpack.license.self_generated.type', 'trial'
@ -22,12 +28,23 @@ def followCluster = testClusters.register('follow-cluster') {
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
user username: 'admin', password: 'admin-password', role: 'superuser'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderInfo = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}
def leaderUri = leaderInfo.map { it.getAllTransportPortURI().get(0) }
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${leaderCluster.get().getAllTransportPortURI().get(0)}\"" }, IGNORE_VALUE
{ "\"${leaderUri.get()}\"" }, IGNORE_VALUE
nameCustomization = { 'follow' }
}
tasks.register('leader-cluster', RestIntegTestTask) {
mustRunAfter("precommit")
systemProperty 'tests.target_cluster', 'leader'
@ -37,8 +54,19 @@ tasks.register('follow-cluster', RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster leaderCluster
systemProperty 'tests.target_cluster', 'follow'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
nonInputProperties.systemProperty 'tests.leader_host',
"${-> leaderCluster.get().getAllHttpSocketURI().get(0)}"
"${-> leaderUri.get() }"
}
tasks.register("followClusterRestartTest", StandaloneRestIntegTestTask) {
@ -48,10 +76,27 @@ tasks.register("followClusterRestartTest", StandaloneRestIntegTestTask) {
systemProperty 'tests.rest.load_packaged', 'false'
systemProperty 'tests.target_cluster', 'follow-restart'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
def followUris = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("follow-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.join(",") }
nonInputProperties.systemProperty 'tests.leader_host', leaderUri
nonInputProperties.systemProperty 'tests.rest.cluster', followUris
doFirst {
followCluster.get().restart()
nonInputProperties.systemProperty 'tests.leader_host', leaderCluster.map(c-> c.getAllHttpSocketURI().get(0))
nonInputProperties.systemProperty 'tests.rest.cluster', followCluster.map(c -> c.getAllHttpSocketURI().join(","))
serviceProvider.get().restart(clusterPath, "follow-cluster")
}
}

31
x-pack/plugin/ccr/qa/security/build.gradle
View file

@ -1,4 +1,9 @@
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
apply plugin: 'elasticsearch.internal-testclusters'
@ -10,6 +15,8 @@ dependencies {
testImplementation project(':x-pack:plugin:ccr:qa')
}
def clusterPath = getPath()
def leadCluster = testClusters.register('leader-cluster') {
testDistribution = 'DEFAULT'
setting 'xpack.license.self_generated.type', 'trial'
@ -21,8 +28,18 @@ def leadCluster = testClusters.register('leader-cluster') {
testClusters.register('follow-cluster') {
testDistribution = 'DEFAULT'
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUris = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.AllTransportPortURI }
setting 'cluster.remote.leader_cluster.seeds', {
"\"${leadCluster.get().getAllTransportPortURI().join(",")}\""
"\"${leaderUris.get().join(",")}\""
}, IGNORE_VALUE
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
@ -41,7 +58,17 @@ def followerClusterTestTask = tasks.register('follow-cluster', RestIntegTestTask
dependsOn 'leader-cluster'
useCluster leadCluster
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', leadCluster.map(c-> c.getAllHttpSocketURI().get(0))
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
def leaderUri = project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("leader-cluster")
it.parameters.service = serviceProvider
}.map { it.allHttpSocketURI.get(0) }
nonInputProperties.systemProperty 'tests.leader_host', leaderUri
}
tasks.named("check").configure { dependsOn(followerClusterTestTask) }

1
x-pack/plugin/esql/build.gradle
View file

@ -83,7 +83,6 @@ tasks.named("test").configure {
}
}
File functionsFolder = file("build/testrun/test/temp/esql/functions")
File signatureFolder = file("build/testrun/test/temp/esql/functions/signature")
File typesFolder = file("build/testrun/test/temp/esql/functions/types")
def functionsDocFolder = file("${rootDir}/docs/reference/esql/functions")
def effectiveProjectDir = projectDir

18
x-pack/plugin/sql/qa/jdbc/security/build.gradle
View file

@ -1,4 +1,8 @@
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
import org.elasticsearch.gradle.testclusters.TestClusterValueSource
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestClustersRegistry
import org.elasticsearch.gradle.util.GradleUtils
apply plugin: 'elasticsearch.internal-test-artifact'
@ -11,7 +15,10 @@ dependencies {
Project mainProject = project
subprojects {
def clusterPath = getPath()
// Use tests from the root security qa project in subprojects
configurations.create('testArtifacts').transitive(false)
@ -46,6 +53,17 @@ subprojects {
dependsOn copyTestClasses
classpath += configurations.testArtifacts
testClassesDirs = project.files(testArtifactsDir)
Provider<TestClustersRegistry> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
TestClustersPlugin.REGISTRY_SERVICE_NAME
)
project.getProviders().of(TestClusterValueSource.class) {
it.parameters.path.set(clusterPath)
it.parameters.clusterName.set("javaRestTest")
it.parameters.service = serviceProvider
}
nonInputProperties.systemProperty 'tests.audit.logfile',
"${-> testClusters.javaRestTest.singleNode().getAuditLog()}"
nonInputProperties.systemProperty 'tests.audit.yesterday.logfile',

1
x-pack/qa/repository-old-versions/build.gradle
View file

@ -9,7 +9,6 @@ import org.elasticsearch.gradle.Architecture
import org.elasticsearch.gradle.OS
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.internal.BwcVersions
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.test.AntFixture
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.elasticsearch.gradle.transform.UnzipTransform