mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-19 04:45:07 -04:00
13c8aaeffa
Static fields dont do well in Gradle with configuration cache enabled. - Use buildParams extension in build scripts - Keep BuildParams.ci for now for easy serverless migration - Tweak testing doc
244 lines
10 KiB
Groovy
244 lines
10 KiB
Groovy
import org.apache.tools.ant.filters.ReplaceTokens
|
|
import org.elasticsearch.gradle.internal.info.BuildParams
|
|
import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
|
|
|
|
/*
|
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
* or more contributor license agreements. Licensed under the "Elastic License
|
|
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
|
* Public License v 1"; you may not use this file except in compliance with, at
|
|
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
|
* License v3.0 only", or the "Server Side Public License, v 1".
|
|
*/
|
|
apply plugin: 'elasticsearch.internal-yaml-rest-test'
|
|
apply plugin: 'elasticsearch.internal-cluster-test'
|
|
apply plugin: 'elasticsearch.internal-java-rest-test'
|
|
|
|
esplugin {
|
|
description 'The S3 repository plugin adds S3 repositories'
|
|
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
|
|
}
|
|
|
|
versions << [
|
|
'aws': '1.12.270'
|
|
]
|
|
|
|
dependencies {
|
|
api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
|
|
api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
|
|
api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
|
|
api "com.amazonaws:jmespath-java:${versions.aws}"
|
|
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
|
|
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
|
|
api "commons-logging:commons-logging:${versions.commonslogging}"
|
|
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
api "commons-codec:commons-codec:${versions.commonscodec}"
|
|
api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
|
|
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
|
|
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
|
api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
|
|
api "joda-time:joda-time:2.10.14"
|
|
|
|
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
|
|
// and whitelist this hack in JarHell
|
|
api 'javax.xml.bind:jaxb-api:2.2.2'
|
|
|
|
testImplementation project(':test:fixtures:s3-fixture')
|
|
yamlRestTestImplementation project(":test:framework")
|
|
yamlRestTestImplementation project(':test:fixtures:s3-fixture')
|
|
yamlRestTestImplementation project(':test:fixtures:minio-fixture')
|
|
internalClusterTestImplementation project(':test:fixtures:minio-fixture')
|
|
|
|
javaRestTestImplementation project(":test:framework")
|
|
javaRestTestImplementation project(':test:fixtures:s3-fixture')
|
|
javaRestTestImplementation project(':modules:repository-s3')
|
|
|
|
yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
|
|
internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
|
|
}
|
|
|
|
restResources {
|
|
restApi {
|
|
include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
|
|
}
|
|
}
|
|
|
|
tasks.named("dependencyLicenses").configure {
|
|
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
|
|
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
}
|
|
|
|
esplugin.bundleSpec.from('config/repository-s3') {
|
|
into 'config'
|
|
}
|
|
|
|
def testRepositoryCreds = tasks.register("testRepositoryCreds", Test) {
|
|
include '**/RepositoryCredentialsTests.class'
|
|
systemProperty 'es.allow_insecure_settings', 'true'
|
|
classpath = sourceSets.test.runtimeClasspath
|
|
testClassesDirs = sourceSets.test.output.classesDirs
|
|
}
|
|
|
|
tasks.named('check').configure {
|
|
dependsOn(testRepositoryCreds)
|
|
}
|
|
|
|
tasks.named('test').configure {
|
|
// this is tested explicitly in separate test tasks
|
|
exclude '**/RepositoryCredentialsTests.class'
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
|
|
// the other which uses three-part "temporary" or "session" credentials.
|
|
|
|
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
|
|
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
|
|
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
|
|
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
|
|
|
|
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
|
|
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
|
|
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
|
|
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
|
|
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
|
|
|
|
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
|
|
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
|
|
|
|
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
|
|
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
|
|
|
|
String s3STSBucket = System.getenv("amazon_s3_bucket_sts")
|
|
String s3STSBasePath = System.getenv("amazon_s3_base_path_sts")
|
|
|
|
boolean s3DisableChunkedEncoding = buildParams.random.nextBoolean()
|
|
|
|
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
|
|
// credentials hard-coded in.
|
|
|
|
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
|
|
s3PermanentAccessKey = 's3_test_access_key'
|
|
s3PermanentSecretKey = 's3_test_secret_key'
|
|
s3PermanentBucket = 'bucket'
|
|
s3PermanentBasePath = 'base_path'
|
|
useFixture = true
|
|
}
|
|
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
|
s3TemporaryAccessKey = 'session_token_access_key'
|
|
s3TemporarySecretKey = 'session_token_secret_key'
|
|
s3TemporaryBucket = 'session_token_bucket'
|
|
s3TemporaryBasePath = 'session_token_base_path'
|
|
}
|
|
|
|
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
|
|
s3EC2Bucket = 'ec2_bucket'
|
|
s3EC2BasePath = 'ec2_base_path'
|
|
s3ECSBucket = 'ecs_bucket'
|
|
s3ECSBasePath = 'ecs_base_path'
|
|
}
|
|
|
|
if (!s3STSBucket && !s3STSBasePath) {
|
|
s3STSBucket = 'sts_bucket'
|
|
s3STSBasePath = 'sts_base_path'
|
|
}
|
|
|
|
tasks.named("processYamlRestTestResources").configure {
|
|
from("src/test/resources") {
|
|
include "aws-web-identity-token-file"
|
|
}
|
|
Map<String, Object> expansions = [
|
|
'permanent_bucket' : s3PermanentBucket,
|
|
'permanent_base_path' : s3PermanentBasePath + "_integration_tests",
|
|
'temporary_bucket' : s3TemporaryBucket,
|
|
'temporary_base_path' : s3TemporaryBasePath + "_integration_tests",
|
|
'ec2_bucket' : s3EC2Bucket,
|
|
'ec2_base_path' : s3EC2BasePath,
|
|
'ecs_bucket' : s3ECSBucket,
|
|
'ecs_base_path' : s3ECSBasePath,
|
|
'sts_bucket' : s3STSBucket,
|
|
'sts_base_path' : s3STSBasePath,
|
|
'disable_chunked_encoding': s3DisableChunkedEncoding
|
|
]
|
|
inputs.properties(expansions)
|
|
filter("tokens" : expansions.collectEntries {k, v -> [k, v.toString()]} /* must be a map of strings */, ReplaceTokens.class)
|
|
}
|
|
|
|
tasks.named("internalClusterTest").configure {
|
|
// this is tested explicitly in a separate test task
|
|
exclude '**/S3RepositoryThirdPartyTests.class'
|
|
// TODO: remove once https://github.com/elastic/elasticsearch/issues/101608 is fixed
|
|
systemProperty 'es.insecure_network_trace_enabled', 'true'
|
|
}
|
|
|
|
tasks.named("yamlRestTest").configure {
|
|
systemProperty("s3PermanentAccessKey", s3PermanentAccessKey)
|
|
systemProperty("s3PermanentSecretKey", s3PermanentSecretKey)
|
|
systemProperty("s3TemporaryAccessKey", s3TemporaryAccessKey)
|
|
systemProperty("s3TemporarySecretKey", s3TemporarySecretKey)
|
|
systemProperty("s3EC2AccessKey", s3PermanentAccessKey)
|
|
|
|
// ideally we could resolve an env path in cluster config as resource similar to configuring a config file
|
|
// not sure how common this is, but it would be nice to support
|
|
File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file')
|
|
// The web identity token can be read only from the plugin config directory because of security restrictions
|
|
// Ideally we would create a symlink, but extraConfigFile doesn't support it
|
|
nonInputProperties.systemProperty("awsWebIdentityTokenExternalLocation", awsWebIdentityTokenExternalLocation.getAbsolutePath())
|
|
}
|
|
|
|
// 3rd Party Tests
|
|
tasks.register("s3ThirdPartyTest", Test) {
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(internalTestSourceSet.getRuntimeClasspath())
|
|
include '**/S3RepositoryThirdPartyTests.class'
|
|
systemProperty("tests.use.fixture", Boolean.toString(useFixture))
|
|
|
|
// test container accesses ~/.testcontainers.properties read
|
|
systemProperty "tests.security.manager", "false"
|
|
systemProperty 'test.s3.account', s3PermanentAccessKey
|
|
systemProperty 'test.s3.key', s3PermanentSecretKey
|
|
systemProperty 'test.s3.bucket', s3PermanentBucket
|
|
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + buildParams.testSeed
|
|
}
|
|
|
|
tasks.named("thirdPartyAudit").configure {
|
|
ignoreMissingClasses(
|
|
// classes are missing
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
'javax.jms.Message',
|
|
'software.amazon.ion.IonReader',
|
|
'software.amazon.ion.IonSystem',
|
|
'software.amazon.ion.IonType',
|
|
'software.amazon.ion.IonWriter',
|
|
'software.amazon.ion.Timestamp',
|
|
'software.amazon.ion.system.IonBinaryWriterBuilder',
|
|
'software.amazon.ion.system.IonSystemBuilder',
|
|
'software.amazon.ion.system.IonTextWriterBuilder',
|
|
'software.amazon.ion.system.IonWriterBuilder',
|
|
// We don't use the kms dependency
|
|
'com.amazonaws.services.kms.AWSKMS',
|
|
'com.amazonaws.services.kms.AWSKMSClient',
|
|
'com.amazonaws.services.kms.AWSKMSClientBuilder',
|
|
'com.amazonaws.services.kms.model.DecryptRequest',
|
|
'com.amazonaws.services.kms.model.DecryptResult',
|
|
'com.amazonaws.services.kms.model.EncryptRequest',
|
|
'com.amazonaws.services.kms.model.EncryptResult',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
|
|
'com.amazonaws.services.kms.model.GenerateDataKeyResult',
|
|
'javax.activation.DataHandler'
|
|
)
|
|
}
|
|
|
|
tasks.named("check").configure {
|
|
dependsOn(tasks.withType(Test))
|
|
}
|
|
|