Make AntFixture handling task provider api compliant (#65832) (#66224)

This tweaks the AntFixture handling to make it compliant with the task avoidance api.
Tasks of type StandaloneRestTestTask are now generally finalised by using the typed ant stop task
which allows us to remove of errorprone dependsOn overrides in StandaloneRestTestTask. As a result
we also ported more task definitions in the build to task avoidance api.

Next work item regarding AntFixture handling is porting AntFixture to a plain Gradle task and remove
Groovy AntBuilder will allow us to port more build logic from Groovy to Java but is out of the scope of
This PR.
This commit is contained in:
Rene Groeschke 2020-12-12 21:26:25 +01:00 committed by GitHub
parent a779f61531
commit 8e5f7365e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 182 additions and 123 deletions

View file

@ -0,0 +1,66 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.test.AntFixture
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.tasks.Internal
import javax.inject.Inject
class AntFixtureStop extends LoggedExec implements FixtureStop {
@Internal
AntFixture fixture
@Internal
FileSystemOperations fileSystemOperations
@Inject
AntFixtureStop(FileSystemOperations fileSystemOperations) {
super(fileSystemOperations)
this.fileSystemOperations = fileSystemOperations
}
void setFixture(AntFixture fixture) {
assert this.fixture == null
this.fixture = fixture;
final Object pid = "${ -> this.fixture.pid }"
onlyIf { fixture.pidFile.exists() }
doFirst {
logger.info("Shutting down ${fixture.name} with pid ${pid}")
}
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable = 'Taskkill'
args('/PID', pid, '/F')
} else {
executable = 'kill'
args('-9', pid)
}
doLast {
fileSystemOperations.delete {
it.delete(fixture.pidFile)
}
}
this.fixture = fixture
}
}

View file

@ -20,11 +20,9 @@
package org.elasticsearch.gradle.test package org.elasticsearch.gradle.test
import org.apache.tools.ant.taskdefs.condition.Os import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.AntFixtureStop
import org.elasticsearch.gradle.AntTask import org.elasticsearch.gradle.AntTask
import org.elasticsearch.gradle.LoggedExec
import org.gradle.api.GradleException import org.gradle.api.GradleException
import org.gradle.api.Task
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskProvider import org.gradle.api.tasks.TaskProvider
@ -81,7 +79,7 @@ class AntFixture extends AntTask implements Fixture {
return tmpFile.exists() return tmpFile.exists()
} }
private final TaskProvider stopTask private final TaskProvider<AntFixtureStop> stopTask
AntFixture() { AntFixture() {
stopTask = createStopTask() stopTask = createStopTask()
@ -90,7 +88,7 @@ class AntFixture extends AntTask implements Fixture {
@Override @Override
@Internal @Internal
TaskProvider getStopTask() { TaskProvider<AntFixtureStop> getStopTask() {
return stopTask return stopTask
} }
@ -227,31 +225,12 @@ class AntFixture extends AntTask implements Fixture {
} }
/** Adds a task to kill an elasticsearch node with the given pidfile */ /** Adds a task to kill an elasticsearch node with the given pidfile */
private TaskProvider createStopTask() { private TaskProvider<AntFixtureStop> createStopTask() {
final AntFixture fixture = this final AntFixture fixture = this
final Object pid = "${ -> fixture.pid }" TaskProvider<AntFixtureStop> stop = project.tasks.register("${name}#stop", AntFixtureStop)
TaskProvider<Exec> stop = project.tasks.register("${name}#stop", LoggedExec)
stop.configure { stop.configure {
onlyIf { fixture.pidFile.exists() } it.fixture = fixture
doFirst {
logger.info("Shutting down ${fixture.name} with pid ${pid}")
} }
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable = 'Taskkill'
args('/PID', pid, '/F')
} else {
executable = 'kill'
args('-9', pid)
}
doLast {
getFileSystemOperations().delete {
it.delete(fixture.pidFile)
}
}
}
return stop return stop
} }

View file

@ -0,0 +1,24 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;
import org.gradle.api.Task;
public interface FixtureStop extends Task {}

View file

@ -20,8 +20,10 @@
package org.elasticsearch.gradle.test; package org.elasticsearch.gradle.test;
import org.elasticsearch.gradle.ElasticsearchTestBasePlugin; import org.elasticsearch.gradle.ElasticsearchTestBasePlugin;
import org.elasticsearch.gradle.FixtureStop;
import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider;
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin; import org.gradle.api.Plugin;
@ -64,5 +66,8 @@ public class RestTestBasePlugin implements Plugin<Project> {
} }
} }
}); });
project.getTasks()
.withType(StandaloneRestIntegTestTask.class)
.configureEach(t -> t.finalizedBy(project.getTasks().withType(FixtureStop.class)));
} }
} }

View file

@ -19,9 +19,7 @@
package org.elasticsearch.gradle.testclusters; package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.gradle.FileSystemOperationsAware; import org.elasticsearch.gradle.FileSystemOperationsAware;
import org.elasticsearch.gradle.test.Fixture;
import org.elasticsearch.gradle.util.GradleUtils; import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.Task;
import org.gradle.api.provider.Provider; import org.gradle.api.provider.Provider;
import org.gradle.api.services.internal.BuildServiceRegistryInternal; import org.gradle.api.services.internal.BuildServiceRegistryInternal;
import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.CacheableTask;
@ -98,31 +96,9 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
if (nodeCount > 0) { if (nodeCount > 0) {
locks.add(resource.getResourceLock(Math.min(nodeCount, resource.getMaxUsages()))); locks.add(resource.getResourceLock(Math.min(nodeCount, resource.getMaxUsages())));
} }
return Collections.unmodifiableList(locks); return Collections.unmodifiableList(locks);
} }
@Override
public Task dependsOn(Object... dependencies) {
super.dependsOn(dependencies);
for (Object dependency : dependencies) {
if (dependency instanceof Fixture) {
finalizedBy(((Fixture) dependency).getStopTask());
}
}
return this;
}
@Override
public void setDependsOn(Iterable<?> dependencies) {
super.setDependsOn(dependencies);
for (Object dependency : dependencies) {
if (dependency instanceof Fixture) {
finalizedBy(((Fixture) dependency).getStopTask());
}
}
}
public WorkResult delete(Object... objects) { public WorkResult delete(Object... objects) {
return getFileSystemOperations().delete(d -> d.delete(objects)); return getFileSystemOperations().delete(d -> d.delete(objects));
} }

View file

@ -130,6 +130,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
versions = ['2', '1'] versions = ['2', '1']
} }
for (String version : versions) { for (String version : versions) {
// TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
TaskProvider<Sync> unzip = tasks.register("unzipEs${version}", Sync) { TaskProvider<Sync> unzip = tasks.register("unzipEs${version}", Sync) {
Configuration oldEsDependency = configurations['es' + version] Configuration oldEsDependency = configurations['es' + version]
dependsOn oldEsDependency dependsOn oldEsDependency

View file

@ -40,7 +40,7 @@ restResources {
File repositoryDir = new File(project.buildDir, "shared-repository") File repositoryDir = new File(project.buildDir, "shared-repository")
/** A task to start the URLFixture which exposes the repositoryDir over HTTP **/ /** A task to start the URLFixture which exposes the repositoryDir over HTTP **/
task urlFixture(type: AntFixture) { def urlFixtureTaskProvider = tasks.register("urlFixture", AntFixture) {
dependsOn testClasses dependsOn testClasses
doFirst { doFirst {
repositoryDir.mkdirs() repositoryDir.mkdirs()
@ -49,12 +49,13 @@ task urlFixture(type: AntFixture) {
executable = "${BuildParams.runtimeJavaHome}/bin/java" executable = "${BuildParams.runtimeJavaHome}/bin/java"
args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}" args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}"
} }
tasks.named("yamlRestTest").configure { tasks.named("yamlRestTest").configure {
dependsOn urlFixture dependsOn urlFixtureTaskProvider
} }
tasks.named("internalClusterTest").configure { tasks.named("internalClusterTest").configure {
dependsOn urlFixture dependsOn urlFixtureTaskProvider
} }
testClusters.all { testClusters.all {

View file

@ -44,13 +44,15 @@ Map<String, Object> expansions = [
'expected_nodes': ec2NumberOfNodes 'expected_nodes': ec2NumberOfNodes
] ]
processYamlRestTestResources { tasks.named("processYamlRestTestResources").configure {
inputs.properties(expansions) inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions) MavenFilteringHack.filter(it, expansions)
} }
// disable default yamlRestTest task, use spezialized ones below // disable default yamlRestTest task, use spezialized ones below
yamlRestTest.enabled = false tasks.named("yamlRestTest").configure {
enabled = false
}
/* /*
* Test using various credential providers (see also https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html): * Test using various credential providers (see also https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html):
@ -64,61 +66,66 @@ yamlRestTest.enabled = false
* custom Java security policy to work. * custom Java security policy to work.
*/ */
['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action -> ['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action ->
AntFixture fixture = tasks.create(name: "ec2Fixture${action}", type: AntFixture) { TaskProvider<AntFixture> fixture = tasks.register("ec2Fixture${action}", AntFixture) {
dependsOn project.sourceSets.yamlRestTest.runtimeClasspath dependsOn project.sourceSets.yamlRestTest.runtimeClasspath
env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}"
executable = "${BuildParams.runtimeJavaHome}/bin/java" executable = "${BuildParams.runtimeJavaHome}/bin/java"
args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt" args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt"
} }
tasks.create(name: "yamlRestTest${action}", type: RestIntegTestTask) { def yamlRestTestTask = tasks.register("yamlRestTest${action}", RestIntegTestTask) {
dependsOn fixture dependsOn fixture
}
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
"yamlRestTest${action}" { testClassesDirs = yamlRestTestSourceSet.getOutput().getClassesDirs()
setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) classpath = yamlRestTestSourceSet.getRuntimeClasspath()
setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
} }
check.dependsOn("yamlRestTest${action}")
testClusters."yamlRestTest${action}" { tasks.named("check").configure {
dependsOn(yamlRestTestTask)
}
testClusters.matching { it.name == yamlRestTestTask.name}.configureEach {
numberOfNodes = ec2NumberOfNodes numberOfNodes = ec2NumberOfNodes
plugin ':plugins:discovery-ec2' plugin ':plugins:discovery-ec2'
setting 'discovery.seed_providers', 'ec2' setting 'discovery.seed_providers', 'ec2'
setting 'network.host', '_ec2_' setting 'network.host', '_ec2_'
setting 'discovery.ec2.endpoint', { "http://${-> fixture.addressAndPort}" }, IGNORE_VALUE setting 'discovery.ec2.endpoint', { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.addressAndPort}" }, IGNORE_VALUE systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE
} }
} }
// Extra config for KeyStore // Extra config for KeyStore
testClusters.yamlRestTestKeyStore { testClusters.matching { it.name == "yamlRestTestKeyStore" }.configureEach {
keystore 'discovery.ec2.access_key', 'ec2_integration_test_access_key' keystore 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
keystore 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key' keystore 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
} }
// Extra config for EnvVariables // Extra config for EnvVariables
testClusters.yamlRestTestEnvVariables { testClusters.matching { it.name == "yamlRestTestEnvVariables" }.configureEach {
environment 'AWS_ACCESS_KEY_ID', 'ec2_integration_test_access_key' environment 'AWS_ACCESS_KEY_ID', 'ec2_integration_test_access_key'
environment 'AWS_SECRET_ACCESS_KEY', 'ec2_integration_test_secret_key' environment 'AWS_SECRET_ACCESS_KEY', 'ec2_integration_test_secret_key'
} }
// Extra config for SystemProperties // Extra config for SystemProperties
testClusters.yamlRestTestSystemProperties { testClusters.matching { it.name == "yamlRestTestSystemProperties" }.configureEach {
systemProperty 'aws.accessKeyId', 'ec2_integration_test_access_key' systemProperty 'aws.accessKeyId', 'ec2_integration_test_access_key'
systemProperty 'aws.secretKey', 'ec2_integration_test_secret_key' systemProperty 'aws.secretKey', 'ec2_integration_test_secret_key'
} }
// Extra config for ContainerCredentials // Extra config for ContainerCredentials
ec2FixtureContainerCredentials.env 'ACTIVATE_CONTAINER_CREDENTIALS', true tasks.named("ec2FixtureContainerCredentials").configure {
env 'ACTIVATE_CONTAINER_CREDENTIALS', true
}
testClusters.yamlRestTestContainerCredentials { testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach {
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
{ "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
} }
// Extra config for InstanceProfile // Extra config for InstanceProfile
ec2FixtureInstanceProfile.env 'ACTIVATE_INSTANCE_PROFILE', true tasks.named("ec2FixtureInstanceProfile").configure {
env 'ACTIVATE_INSTANCE_PROFILE', true
}

View file

@ -39,7 +39,7 @@ restResources {
} }
/** A task to start the GCEFixture which emulates a GCE service **/ /** A task to start the GCEFixture which emulates a GCE service **/
task gceFixture(type: AntFixture) { def gceFixtureProvider = tasks.register("gceFixture", AntFixture) {
dependsOn project.sourceSets.yamlRestTest.runtimeClasspath dependsOn project.sourceSets.yamlRestTest.runtimeClasspath
env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}"
executable = "${BuildParams.runtimeJavaHome}/bin/java" executable = "${BuildParams.runtimeJavaHome}/bin/java"
@ -55,21 +55,21 @@ tasks.named("processYamlRestTestResources").configure {
MavenFilteringHack.filter(it, expansions) MavenFilteringHack.filter(it, expansions)
} }
yamlRestTest { tasks.named("yamlRestTest").configure {
dependsOn gceFixture dependsOn gceFixtureProvider
} }
testClusters.yamlRestTest { testClusters.matching { it.name == "yamlRestTest" }.configureEach {
numberOfNodes = gceNumberOfNodes numberOfNodes = gceNumberOfNodes
plugin ':plugins:discovery-gce' plugin ':plugins:discovery-gce'
// use gce fixture for Auth calls instead of http://metadata.google.internal // use gce fixture for Auth calls instead of http://metadata.google.internal
environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE environment 'GCE_METADATA_HOST', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE
// allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`) // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`)
systemProperty 'es.allow_reroute_gce_settings', 'true' systemProperty 'es.allow_reroute_gce_settings', 'true'
setting 'discovery.seed_providers', 'gce' setting 'discovery.seed_providers', 'gce'
// use gce fixture for metadata server calls instead of http://metadata.google.internal // use gce fixture for metadata server calls instead of http://metadata.google.internal
setting 'cloud.gce.host', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE setting 'cloud.gce.host', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE
// use gce fixture for API calls instead of https://www.googleapis.com // use gce fixture for API calls instead of https://www.googleapis.com
setting 'cloud.gce.root_url', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE setting 'cloud.gce.root_url', { "http://${gceFixtureProvider.get().addressAndPort}" }, IGNORE_VALUE
} }

View file

@ -33,15 +33,14 @@ esplugin {
// No unit tests in this example // No unit tests in this example
tasks.named("test").configure { enabled = false } tasks.named("test").configure { enabled = false }
tasks.register("exampleFixture", org.elasticsearch.gradle.test.AntFixture) { def fixture = tasks.register("exampleFixture", org.elasticsearch.gradle.test.AntFixture) {
dependsOn sourceSets.javaRestTest.runtimeClasspath dependsOn sourceSets.javaRestTest.runtimeClasspath
env 'CLASSPATH', "${-> project.sourceSets.javaRestTest.runtimeClasspath.asPath}" env 'CLASSPATH', "${-> project.sourceSets.javaRestTest.runtimeClasspath.asPath}"
executable = "${BuildParams.runtimeJavaHome}/bin/java" executable = "${BuildParams.runtimeJavaHome}/bin/java"
args 'org.elasticsearch.example.resthandler.ExampleFixture', baseDir, 'TEST' args 'org.elasticsearch.example.resthandler.ExampleFixture', baseDir, 'TEST'
} }
javaRestTest { tasks.named("javaRestTest").configure {
dependsOn exampleFixture dependsOn fixture
nonInputProperties.systemProperty 'external.address', "${-> exampleFixture.addressAndPort}" nonInputProperties.systemProperty 'external.address', "${-> fixture.get().addressAndPort}"
} }

View file

@ -108,7 +108,7 @@ String krb5conf = project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")
// Create HDFS File System Testing Fixtures for HA/Secure combinations // Create HDFS File System Testing Fixtures for HA/Secure combinations
for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) { for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) {
def tsk = project.tasks.register(fixtureName, org.elasticsearch.gradle.test.AntFixture) { project.tasks.register(fixtureName, org.elasticsearch.gradle.test.AntFixture) {
dependsOn project.configurations.hdfsFixture, project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture dependsOn project.configurations.hdfsFixture, project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture
executable = "${BuildParams.runtimeJavaHome}/bin/java" executable = "${BuildParams.runtimeJavaHome}/bin/java"
env 'CLASSPATH', "${-> project.configurations.hdfsFixture.asPath}" env 'CLASSPATH', "${-> project.configurations.hdfsFixture.asPath}"
@ -122,14 +122,14 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
final List<String> miniHDFSArgs = [] final List<String> miniHDFSArgs = []
// If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options
if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { if (name.equals('secureHdfsFixture') || name.equals('secureHaHdfsFixture')) {
miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}"); miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}");
if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_1_9) { if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_1_9) {
miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED') miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED')
} }
} }
// If it's an HA fixture, set a nameservice to use in the JVM options // If it's an HA fixture, set a nameservice to use in the JVM options
if (fixtureName.equals('haHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { if (name.equals('haHdfsFixture') || name.equals('secureHaHdfsFixture')) {
miniHDFSArgs.add("-Dha-nameservice=ha-hdfs") miniHDFSArgs.add("-Dha-nameservice=ha-hdfs")
} }
@ -138,7 +138,7 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
miniHDFSArgs.add(baseDir) miniHDFSArgs.add(baseDir)
// If it's a secure fixture, then set the principal name and keytab locations to use for auth. // If it's a secure fixture, then set the principal name and keytab locations to use for auth.
if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { if (name.equals('secureHdfsFixture') || name.equals('secureHaHdfsFixture')) {
miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}") miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}")
miniHDFSArgs.add( miniHDFSArgs.add(
project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab")
@ -147,36 +147,32 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
args miniHDFSArgs.toArray() args miniHDFSArgs.toArray()
} }
// TODO: The task configuration block has side effects that require it currently to be always executed.
// Otherwise tests start failing. Therefore we enforce the task creation for now.
tsk.get()
} }
Set disabledIntegTestTaskNames = [] Set disabledIntegTestTaskNames = []
for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
task "${integTestTaskName}"(type: RestIntegTestTask) { def testTask = tasks.register(integTestTaskName, RestIntegTestTask) {
description = "Runs rest tests against an elasticsearch cluster with HDFS." description = "Runs rest tests against an elasticsearch cluster with HDFS."
dependsOn("bundlePlugin") dependsOn("bundlePlugin")
if (disabledIntegTestTaskNames.contains(integTestTaskName)) { if (disabledIntegTestTaskNames.contains(name)) {
enabled = false; enabled = false;
} }
if (integTestTaskName.contains("Secure")) { if (name.contains("Secure")) {
if (integTestTaskName.contains("Ha")) { if (name.contains("Ha")) {
dependsOn secureHaHdfsFixture dependsOn "secureHaHdfsFixture"
} else { } else {
dependsOn secureHdfsFixture dependsOn "secureHdfsFixture"
} }
} }
onlyIf { BuildParams.inFipsJvm == false } onlyIf { BuildParams.inFipsJvm == false }
if (integTestTaskName.contains("Ha")) { if (name.contains("Ha")) {
Path portsFile Path portsFile
File portsFileDir = file("${workingDir}/hdfsFixture") File portsFileDir = file("${workingDir}/hdfsFixture")
if (integTestTaskName.contains("Secure")) { if (name.contains("Secure")) {
portsFile = buildDir.toPath() portsFile = buildDir.toPath()
.resolve("fixtures") .resolve("fixtures")
.resolve("secureHaHdfsFixture") .resolve("secureHaHdfsFixture")
@ -199,8 +195,8 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
} }
} }
if (integTestTaskName.contains("Secure")) { if (name.contains("Secure")) {
if (disabledIntegTestTaskNames.contains(integTestTaskName) == false) { if (disabledIntegTestTaskNames.contains(name) == false) {
nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}"
nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}"
jvmArgs "-Djava.security.krb5.conf=${krb5conf}" jvmArgs "-Djava.security.krb5.conf=${krb5conf}"
@ -212,7 +208,7 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
} }
} }
testClusters."${integTestTaskName}" { testClusters.matching { it.name == testTask.name}.configureEach {
plugin(bundlePlugin.archiveFile) plugin(bundlePlugin.archiveFile)
if (integTestTaskName.contains("Secure")) { if (integTestTaskName.contains("Secure")) {
systemProperty "java.security.krb5.conf", krb5conf systemProperty "java.security.krb5.conf", krb5conf
@ -251,28 +247,31 @@ if (legalPath == false) {
// Always ignore HA integration tests in the normal integration test runner, they are included below as // Always ignore HA integration tests in the normal integration test runner, they are included below as
// part of their own HA-specific integration test tasks. // part of their own HA-specific integration test tasks.
integTest { tasks.named("integTest").configure {
onlyIf { BuildParams.inFipsJvm == false } onlyIf { BuildParams.inFipsJvm == false }
exclude('**/Ha*TestSuiteIT.class') exclude('**/Ha*TestSuiteIT.class')
} }
if (fixtureSupported) { if (fixtureSupported) {
// Check depends on the HA test. Already depends on the standard test. // Check depends on the HA test. Already depends on the standard test.
project.check.dependsOn(integTestHa) tasks.named("check").configure {
dependsOn("integTestHa")
}
// Both standard and HA tests depend on their respective HDFS fixtures // Both standard and HA tests depend on their respective HDFS fixtures
integTest.dependsOn hdfsFixture tasks.named("integTest").configure {
integTestHa.dependsOn haHdfsFixture dependsOn "hdfsFixture"
// The normal test runner only runs the standard hdfs rest tests // The normal test runner only runs the standard hdfs rest tests
integTest {
systemProperty 'tests.rest.suite', 'hdfs_repository' systemProperty 'tests.rest.suite', 'hdfs_repository'
} }
tasks.named("integTestHa").configure {
dependsOn "haHdfsFixture"
// Only include the HA integration tests for the HA test task // Only include the HA integration tests for the HA test task
integTestHa {
setIncludes(['**/Ha*TestSuiteIT.class']) setIncludes(['**/Ha*TestSuiteIT.class'])
} }
} else { } else {
if (legalPath) { if (legalPath) {
logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
@ -281,29 +280,31 @@ if (fixtureSupported) {
} }
// The normal integration test runner will just test that the plugin loads // The normal integration test runner will just test that the plugin loads
integTest { tasks.named("integTest").configure {
systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
} }
// HA fixture is unsupported. Don't run them. // HA fixture is unsupported. Don't run them.
integTestHa.setEnabled(false) tasks.named("integTestHa").configure {
setEnabled(false)
}
} }
check.dependsOn(integTestSecure, integTestSecureHa) tasks.named("check").configure {
dependsOn("integTestSecure", "integTestSecureHa")
}
// Run just the secure hdfs rest test suite. // Run just the secure hdfs rest test suite.
integTestSecure { tasks.named("integTestSecure").configure {
systemProperty 'tests.rest.suite', 'secure_hdfs_repository' systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
} // Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner.
// Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner.
integTestSecure {
exclude('**/Ha*TestSuiteIT.class') exclude('**/Ha*TestSuiteIT.class')
} }
// Only include the HA integration tests for the HA test task // Only include the HA integration tests for the HA test task
integTestSecureHa { tasks.named("integTestSecureHa").configure {
setIncludes(['**/Ha*TestSuiteIT.class']) setIncludes(['**/Ha*TestSuiteIT.class'])
} }
thirdPartyAudit { tasks.named("thirdPartyAudit").configure {
ignoreMissingClasses() ignoreMissingClasses()
ignoreViolations( ignoreViolations(
// internal java api: sun.misc.Unsafe // internal java api: sun.misc.Unsafe