mirror of
https://github.com/elastic/logstash.git
synced 2025-04-19 20:27:23 -04:00
534c267622
- have `bootstrap` task do as little as possible: install gems in Gemfile.template that don't belong to groups - have test tasks depend on the `installTestGems` task instead of `bootstrap` - logstash es output is now a dependency because of license checking - fix out of memory problem in SharedHelpers.trap Also use release lockfile during installDefaultGems The release lockfile is only copied to Gemfile.lock if it doesn't exist. During the `installDefaultGems` task other plugin installation tasks already occurred, generating a lock file. This commit removes it before running the plugin installation. Fixes #10942
419 lines
15 KiB
Groovy
419 lines
15 KiB
Groovy
buildscript {
|
|
repositories {
|
|
mavenCentral()
|
|
maven {
|
|
url 'https://plugins.gradle.org/m2/'
|
|
}
|
|
}
|
|
dependencies {
|
|
classpath "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
|
|
}
|
|
}
|
|
|
|
plugins {
|
|
id "de.undercouch.download" version "3.2.0"
|
|
}
|
|
|
|
apply plugin: 'de.undercouch.download'
|
|
apply from: "rubyUtils.gradle"
|
|
|
|
|
|
import de.undercouch.gradle.tasks.download.Download
|
|
import groovy.json.JsonSlurper
|
|
|
|
allprojects {
|
|
group = 'org.logstash'
|
|
|
|
apply plugin: 'java'
|
|
apply plugin: 'idea'
|
|
|
|
project.sourceCompatibility = JavaVersion.VERSION_1_8
|
|
project.targetCompatibility = JavaVersion.VERSION_1_8
|
|
|
|
tasks.withType(JavaCompile).all {
|
|
options.compilerArgs.add("-Xlint:all")
|
|
options.compilerArgs.add("-Xlint:-processing")
|
|
options.compilerArgs.add("-Werror")
|
|
}
|
|
|
|
tasks.withType(Javadoc) {
|
|
options.addStringOption("Xwerror", "-quiet")
|
|
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_9) > 0) {
|
|
options.addBooleanOption("html5", true)
|
|
}
|
|
}
|
|
|
|
clean {
|
|
delete "${projectDir}/out/"
|
|
}
|
|
|
|
//https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
|
|
tasks.withType(Test) {
|
|
testLogging {
|
|
// set options for log level LIFECYCLE
|
|
events "passed", "skipped", "failed", "standardOut"
|
|
showExceptions true
|
|
exceptionFormat "full"
|
|
showCauses true
|
|
showStackTraces true
|
|
enableAssertions false
|
|
|
|
// set options for log level DEBUG and INFO
|
|
debug {
|
|
events "started", "passed", "skipped", "failed", "standardOut", "standardError"
|
|
exceptionFormat "full"
|
|
}
|
|
info.events = debug.events
|
|
info.exceptionFormat = debug.exceptionFormat
|
|
|
|
afterSuite { desc, result ->
|
|
if (!desc.parent) { // will match the outermost suite
|
|
def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)"
|
|
def startItem = '| ', endItem = ' |'
|
|
def repeatLength = startItem.length() + output.length() + endItem.length()
|
|
println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
subprojects {
|
|
repositories {
|
|
maven {
|
|
url 'https://plugins.gradle.org/m2/'
|
|
}
|
|
}
|
|
dependencies {
|
|
compile "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
|
|
}
|
|
|
|
apply plugin: 'com.github.jk1.dependency-license-report'
|
|
|
|
licenseReport {
|
|
renderer = new com.github.jk1.license.render.CsvReportRenderer()
|
|
configurations = ['compile', 'runtime']
|
|
}
|
|
}
|
|
|
|
// fetch version from Logstash's master versions.yml file
|
|
version = versionMap['logstash-core']
|
|
def versionQualifier = System.getenv('VERSION_QUALIFIER')
|
|
if (versionQualifier) {
|
|
version = "$version-$versionQualifier"
|
|
}
|
|
// a release build will try to download the exact version artifact and not append -SNAPSHOT to it see
|
|
// the downloadEs task below
|
|
def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
|
|
|
|
// Tasks
|
|
|
|
clean {
|
|
delete "${projectDir}/Gemfile"
|
|
delete "${projectDir}/Gemfile.lock"
|
|
delete "${projectDir}/vendor"
|
|
delete "${projectDir}/.bundle"
|
|
delete "${projectDir}/qa/integration/Gemfile.lock"
|
|
delete "${projectDir}/qa/integration/.bundle"
|
|
delete "${projectDir}/build/licenseReportFolders.txt"
|
|
delete "${projectDir}/build/rubyDependencies.csv"
|
|
}
|
|
|
|
task bootstrap {
|
|
doLast {
|
|
rake(projectDir, buildDir, 'plugin:install-base')
|
|
}
|
|
}
|
|
|
|
def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect {
|
|
it.tasks.findByName("assemble")
|
|
}
|
|
|
|
task installDefaultGems(dependsOn: assemblyDeps) {
|
|
doLast {
|
|
rake(projectDir, buildDir, 'plugin:install-default')
|
|
}
|
|
}
|
|
|
|
task installTestGems(dependsOn: assemblyDeps) {
|
|
doLast {
|
|
rake(projectDir, buildDir, 'plugin:install-development-dependencies')
|
|
}
|
|
}
|
|
|
|
task assembleTarDistribution(dependsOn: assemblyDeps) {
|
|
inputs.files fileTree("${projectDir}/rakelib")
|
|
inputs.files fileTree("${projectDir}/bin")
|
|
inputs.files fileTree("${projectDir}/config")
|
|
inputs.files fileTree("${projectDir}/lib")
|
|
inputs.files fileTree("${projectDir}/modules")
|
|
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
|
|
inputs.files fileTree("${projectDir}/logstash-core/lib")
|
|
inputs.files fileTree("${projectDir}/logstash-core/src")
|
|
inputs.files fileTree("${projectDir}/x-pack")
|
|
outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
|
|
doLast {
|
|
rake(projectDir, buildDir, 'artifact:tar')
|
|
}
|
|
}
|
|
|
|
task assembleOssTarDistribution(dependsOn: assemblyDeps) {
|
|
inputs.files fileTree("${projectDir}/rakelib")
|
|
inputs.files fileTree("${projectDir}/bin")
|
|
inputs.files fileTree("${projectDir}/config")
|
|
inputs.files fileTree("${projectDir}/lib")
|
|
inputs.files fileTree("${projectDir}/modules")
|
|
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
|
|
inputs.files fileTree("${projectDir}/logstash-core/lib")
|
|
inputs.files fileTree("${projectDir}/logstash-core/src")
|
|
doLast {
|
|
rake(projectDir, buildDir, 'artifact:tar_oss')
|
|
}
|
|
}
|
|
|
|
task assembleZipDistribution(dependsOn: assemblyDeps) {
|
|
inputs.files fileTree("${projectDir}/rakelib")
|
|
inputs.files fileTree("${projectDir}/bin")
|
|
inputs.files fileTree("${projectDir}/config")
|
|
inputs.files fileTree("${projectDir}/lib")
|
|
inputs.files fileTree("${projectDir}/modules")
|
|
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
|
|
inputs.files fileTree("${projectDir}/logstash-core/lib")
|
|
inputs.files fileTree("${projectDir}/logstash-core/src")
|
|
inputs.files fileTree("${projectDir}/x-pack")
|
|
outputs.files file("${buildDir}/logstash-${project.version}.zip")
|
|
doLast {
|
|
rake(projectDir, buildDir, 'artifact:zip')
|
|
}
|
|
}
|
|
|
|
task assembleOssZipDistribution(dependsOn: assemblyDeps) {
|
|
inputs.files fileTree("${projectDir}/rakelib")
|
|
inputs.files fileTree("${projectDir}/bin")
|
|
inputs.files fileTree("${projectDir}/config")
|
|
inputs.files fileTree("${projectDir}/lib")
|
|
inputs.files fileTree("${projectDir}/modules")
|
|
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
|
|
inputs.files fileTree("${projectDir}/logstash-core/lib")
|
|
inputs.files fileTree("${projectDir}/logstash-core/src")
|
|
outputs.files file("${buildDir}/logstash-${project.version}.zip")
|
|
doLast {
|
|
rake(projectDir, buildDir, 'artifact:zip_oss')
|
|
|
|
}
|
|
}
|
|
|
|
project(":logstash-core") {
|
|
["rubyTests", "test"].each { tsk ->
|
|
tasks.getByPath(":logstash-core:" + tsk).configure {
|
|
dependsOn installTestGems
|
|
}
|
|
}
|
|
}
|
|
|
|
def logstashBuildDir = "${buildDir}/logstash-${project.version}-SNAPSHOT"
|
|
|
|
task unpackTarDistribution(dependsOn: assembleTarDistribution, type: Copy) {
|
|
def tar = file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
|
|
inputs.files tar
|
|
outputs.files fileTree(logstashBuildDir)
|
|
from tarTree(tar)
|
|
into {buildDir}
|
|
}
|
|
|
|
def qaVendorPath = "${buildDir}/qa/integration/vendor"
|
|
def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0"
|
|
def qaBundleBin = "${qaBundledGemPath}/bin/bundle"
|
|
|
|
task installIntegrationTestBundler(dependsOn: unpackTarDistribution) {
|
|
outputs.files fileTree("${qaBundledGemPath}/gems/bundler-1.17.3")
|
|
doLast {
|
|
gem(projectDir, buildDir, "bundler", "1.17.3", qaBundledGemPath)
|
|
}
|
|
}
|
|
|
|
task installIntegrationTestGems(dependsOn: installIntegrationTestBundler) {
|
|
inputs.files file("${projectDir}/qa/integration/Gemfile")
|
|
inputs.files file("${projectDir}/qa/integration/integration_tests.gemspec")
|
|
inputs.files file("${logstashBuildDir}/Gemfile")
|
|
inputs.files file("${logstashBuildDir}/Gemfile.lock")
|
|
inputs.files file("${logstashBuildDir}/logstash-core/logstash-core.gemspec")
|
|
outputs.files fileTree("${qaVendorPath}")
|
|
outputs.files file("${projectDir}/qa/integration/Gemfile.lock")
|
|
doLast {
|
|
bundleWithEnv(
|
|
projectDir, buildDir,
|
|
"${projectDir}/qa/integration", qaBundleBin, ['install', '--path', qaVendorPath],
|
|
[LS_GEM_PATH: qaBundledGemPath, LS_GEM_HOME: qaBundledGemPath]
|
|
)
|
|
}
|
|
}
|
|
|
|
def rubyIntegrationSpecs = project.hasProperty("rubyIntegrationSpecs") ? ((String) project.property("rubyIntegrationSpecs")).split(/\s+/).join(",") : "specs/**/*_spec.rb"
|
|
def integrationTestPwd = "${projectDir}/qa/integration"
|
|
|
|
project(":logstash-integration-tests") {
|
|
tasks.getByPath(":logstash-integration-tests:integrationTests").configure {
|
|
systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs
|
|
environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG')
|
|
workingDir integrationTestPwd
|
|
dependsOn installIntegrationTestGems
|
|
}
|
|
}
|
|
|
|
task runIntegrationTests(dependsOn: [tasks.getByPath(":logstash-integration-tests:integrationTests")]) {}
|
|
|
|
task generateLicenseReport(type: JavaExec) {
|
|
dependsOn("generateLicenseReportInputs")
|
|
dependsOn(":dependencies-report:assemble")
|
|
|
|
def jarFile = project('dependencies-report').getBuildDir().toString() + "/libs/dependencies-report.jar"
|
|
|
|
String licenseReportInputCSV = project.hasProperty("licenseReportInputCSV") ? project.property("licenseReportInputCSV") : "build/dependencies.csv.ruby"
|
|
String licenseReportOutputCSV = project.hasProperty("licenseReportOutputCSV") ? project.property("licenseReportOutputCSV") : "build/dependencies.csv"
|
|
String noticePath = "NOTICE.txt"
|
|
|
|
classpath = project.files([jarFile])
|
|
main = "org.logstash.dependencies.Main"
|
|
args licenseReportInputCSV,
|
|
project.getBuildDir().toString() + "/licenseReportFolders.txt",
|
|
licenseReportOutputCSV, noticePath
|
|
}
|
|
|
|
task generateLicenseReportInputs() {
|
|
dependsOn subprojects.generateLicenseReport
|
|
|
|
// write location of all license reports for subprojects containing artifacts that are distributed to single file
|
|
StringBuilder licenseReportFolders = new StringBuilder()
|
|
subprojects.findAll { s1 -> !s1.hasProperty("isDistributedArtifact") || s1.property("isDistributedArtifact") == 'true'}.each { s ->
|
|
s.tasks.findAll { t2 -> t2.getName() == "generateLicenseReport" }.each { t3 ->
|
|
licenseReportFolders.append(t3.outputs.files.asPath + "\n")
|
|
}
|
|
}
|
|
|
|
if (gradle.startParameter.taskNames.contains("generateLicenseReport")) {
|
|
def licenseReportPath = project.getBuildDir().toString() + "/licenseReportFolders.txt"
|
|
def licenseReportFolder = new File(licenseReportPath)
|
|
licenseReportFolder.delete()
|
|
licenseReportFolder = new File(licenseReportPath)
|
|
licenseReportFolder.createNewFile()
|
|
if (licenseReportFolder.canWrite()) {
|
|
licenseReportFolder.text = licenseReportFolders.toString()
|
|
}
|
|
}
|
|
}
|
|
|
|
task generatePluginsVersion(dependsOn: installDefaultGems) {
|
|
doLast {
|
|
rake(projectDir, buildDir, 'generate_plugins_version')
|
|
}
|
|
}
|
|
|
|
bootstrap.dependsOn assemblyDeps
|
|
|
|
runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test")
|
|
check.dependsOn runIntegrationTests
|
|
|
|
String artifactsVersionApi = "https://artifacts-api.elastic.co/v1/versions/"
|
|
|
|
task downloadEs(type: Download) {
|
|
description "Download ES Snapshot for current branch version: ${version}"
|
|
|
|
doFirst {
|
|
if (!project.ext.versionFound) {
|
|
throw new GradleException("could not find the current artifact from the artifact-api ${artifactsVersionApi} for " + (isReleaseBuild ? "release" : "snapshot") + " version: ${version}")
|
|
}
|
|
}
|
|
|
|
String apiResponse = artifactsVersionApi.toURL().text
|
|
def dlVersions = new JsonSlurper().parseText(apiResponse)
|
|
// the version string can be either '7.0.0' or '7.0.0-alpha1', i.e. with the qualifier.
|
|
// in the normal PR type builds it is plain '7.0.0'
|
|
// in the build invoked by the release manager it is '7.0.0-alpha1' etc.
|
|
// the artifacts-api will return JSON like this: `{"versions":["5.6.13-SNAPSHOT","6.4.3-SNAPSHOT","6.5.0-SNAPSHOT","6.6.0-SNAPSHOT","7.0.0-alpha1-SNAPSHOT"]}`
|
|
|
|
String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
|
|
|
|
if (qualifiedVersion == null) {
|
|
// the version is not found in the versions API, for now just set dummy values so the
|
|
// task parameters like src and dest below sees these dummy values but also set
|
|
// versionFound to false so that we can fail the task in the doFirst closure.
|
|
// this is somewhat convoluted and there is certainly a better way to do this but
|
|
// it seems to be an acceptable solution for now.
|
|
project.ext.set("versionFound", false)
|
|
project.ext.set("elasticsearchSnapshotURL", "http://elastic.co/invalid")
|
|
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/invalid")
|
|
} else {
|
|
project.ext.set("versionFound", true)
|
|
|
|
String arch = "x86_64"
|
|
String osName = System.properties['os.name']
|
|
|
|
if (osName ==~ /Mac OS X/) {
|
|
osName = "darwin"
|
|
} else {
|
|
osName = "linux"
|
|
}
|
|
|
|
String architecture = "${osName}-${arch}"
|
|
|
|
String downloadedElasticsearchName = "elasticsearch-${qualifiedVersion}-${architecture}"
|
|
project.ext.set("unpackedElasticsearchName", "elasticsearch-${qualifiedVersion}")
|
|
|
|
// find latest reference to last build
|
|
String buildsListApi = "https://artifacts-api.elastic.co/v1/versions/${qualifiedVersion}/builds/"
|
|
apiResponse = buildsListApi.toURL().text
|
|
def dlBuilds = new JsonSlurper().parseText(apiResponse)
|
|
String build = dlBuilds["builds"][0]
|
|
|
|
// find url of build artifact
|
|
String artifactApiUrl = "https://artifacts-api.elastic.co/v1/versions/${qualifiedVersion}/builds/${build}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
|
apiResponse = artifactApiUrl.toURL().text
|
|
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
|
|
|
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
|
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
|
}
|
|
|
|
src project.ext.elasticsearchSnapshotURL
|
|
onlyIfNewer true
|
|
inputs.file("${projectDir}/versions.yml")
|
|
outputs.file(project.ext.elasticsearchDownloadLocation)
|
|
dest new File(project.ext.elasticsearchDownloadLocation)
|
|
|
|
doLast {
|
|
System.out.println "Downloaded to ${project.ext.elasticsearchDownloadLocation}"
|
|
}
|
|
}
|
|
|
|
task deleteLocalEs(type: Delete) {
|
|
delete ('./build/elasticsearch')
|
|
}
|
|
|
|
task copyEs(type: Copy, dependsOn: [downloadEs, deleteLocalEs]) {
|
|
from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation))
|
|
into "./build/"
|
|
|
|
doLast {
|
|
file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch')
|
|
System.out.println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch"
|
|
}
|
|
}
|
|
|
|
Boolean oss = System.getenv('OSS').equals('true')
|
|
|
|
if (!oss) {
|
|
project(":logstash-xpack") {
|
|
["rubyTests", "rubyIntegrationTests", "test"].each { tsk ->
|
|
tasks.getByPath(":logstash-xpack:" + tsk).configure {
|
|
dependsOn installTestGems
|
|
}
|
|
}
|
|
tasks.getByPath(":logstash-xpack:rubyIntegrationTests").configure {
|
|
dependsOn copyEs
|
|
}
|
|
}
|
|
|
|
task runXPackUnitTests(dependsOn: [tasks.getByPath(":logstash-xpack:rubyTests")]) {}
|
|
task runXPackIntegrationTests(dependsOn: [tasks.getByPath(":logstash-xpack:rubyIntegrationTests")]) {}
|
|
}
|