/* * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch B.V. licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ buildscript { repositories { mavenCentral() maven { url 'https://plugins.gradle.org/m2/' } } dependencies { classpath "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1" } } plugins { id "de.undercouch.download" version "4.0.4" } apply plugin: 'de.undercouch.download' apply from: "rubyUtils.gradle" import de.undercouch.gradle.tasks.download.Download import groovy.json.JsonSlurper allprojects { group = 'org.logstash' apply plugin: 'java' apply plugin: 'idea' apply plugin: 'java-library' project.sourceCompatibility = JavaVersion.VERSION_1_8 project.targetCompatibility = JavaVersion.VERSION_1_8 tasks.withType(JavaCompile).configureEach { options.compilerArgs.add("-Xlint:all") options.compilerArgs.add("-Xlint:-processing") options.compilerArgs.add("-Werror") } tasks.withType(Javadoc).configureEach { options.addStringOption("Xwerror", "-quiet") if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_9) > 0) { options.addBooleanOption("html5", true) } } tasks.withType(Copy).configureEach { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } clean { delete "${projectDir}/out/" } //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time tasks.withType(Test) { testLogging { // set options for log level LIFECYCLE events "passed", "skipped", "failed", "standardOut" showExceptions true exceptionFormat "full" showCauses true showStackTraces true enableAssertions false // set options for log level DEBUG and INFO debug { events "started", "passed", "skipped", "failed", "standardOut", "standardError" exceptionFormat "full" } info.events = debug.events info.exceptionFormat = debug.exceptionFormat afterSuite { desc, result -> if (!desc.parent) { // will match the outermost suite def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)" def startItem = '| ', endItem = ' |' def repeatLength = startItem.length() + output.length() + endItem.length() println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength)) } } } } } subprojects { repositories { mavenCentral() maven { url 'https://plugins.gradle.org/m2/' } } dependencies { implementation "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1" } apply plugin: 'com.github.jk1.dependency-license-report' licenseReport { renderer = new com.github.jk1.license.render.CsvReportRenderer() configurations = ['compile', 'runtime'] } } // fetch version from Logstash's master versions.yml file version = versionMap['logstash-core'] def versionQualifier = System.getenv('VERSION_QUALIFIER') if (versionQualifier) { version = "$version-$versionQualifier" } // a release build will try to download the exact version artifact and not append -SNAPSHOT to it see // the downloadEs task below def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier // Tasks clean { delete "${projectDir}/Gemfile" delete "${projectDir}/Gemfile.lock" delete "${projectDir}/vendor" delete "${projectDir}/.bundle" delete "${projectDir}/qa/integration/Gemfile.lock" delete "${projectDir}/qa/integration/.bundle" delete "${projectDir}/build/licenseReportFolders.txt" delete "${projectDir}/build/rubyDependencies.csv" } def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect { it.tasks.findByName("assemble") } task installBundler(dependsOn: assemblyDeps) { outputs.files file("${projectDir}/vendor/bundle/jruby/2.5.0/bin/bundle") doLast { gem(projectDir, buildDir, "bundler", "1.17.3", "${projectDir}/vendor/bundle/jruby/2.5.0") } } task bootstrap(dependsOn: installBundler) { doLast { setupJruby(projectDir, buildDir) } } task installDefaultGems(dependsOn: bootstrap) { doLast { rake(projectDir, buildDir, 'plugin:install-default') } } task installTestGems(dependsOn: bootstrap) { doLast { rake(projectDir, buildDir, 'plugin:install-development-dependencies') } } task compileGrammar(dependsOn: bootstrap) { doLast { rake(projectDir, buildDir, 'compile:grammar') } } task assembleTarDistribution(dependsOn: bootstrap) { inputs.files fileTree("${projectDir}/rakelib") inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") inputs.files fileTree("${projectDir}/x-pack") outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz") doLast { rake(projectDir, buildDir, 'artifact:tar') } } task assembleOssTarDistribution(dependsOn: bootstrap) { inputs.files fileTree("${projectDir}/rakelib") inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") doLast { rake(projectDir, buildDir, 'artifact:tar_oss') } } task assembleZipDistribution(dependsOn: bootstrap) { inputs.files fileTree("${projectDir}/rakelib") inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") inputs.files fileTree("${projectDir}/x-pack") outputs.files file("${buildDir}/logstash-${project.version}.zip") doLast { rake(projectDir, buildDir, 'artifact:zip') } } task assembleOssZipDistribution(dependsOn: bootstrap) { inputs.files fileTree("${projectDir}/rakelib") inputs.files fileTree("${projectDir}/bin") inputs.files fileTree("${projectDir}/config") inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/modules") inputs.files fileTree("${projectDir}/logstash-core-plugin-api") inputs.files fileTree("${projectDir}/logstash-core/lib") inputs.files fileTree("${projectDir}/logstash-core/src") outputs.files file("${buildDir}/logstash-${project.version}.zip") doLast { rake(projectDir, buildDir, 'artifact:zip_oss') } } project(":logstash-core") { ["rubyTests", "test"].each { tsk -> tasks.getByPath(":logstash-core:" + tsk).configure { dependsOn installTestGems } } } def logstashBuildDir = "${buildDir}/logstash-${project.version}-SNAPSHOT" task unpackTarDistribution(dependsOn: assembleTarDistribution, type: Copy) { def tar = file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz") inputs.files tar outputs.files fileTree(logstashBuildDir) from tarTree(tar) into {buildDir} } def qaVendorPath = "${buildDir}/qa/integration/vendor" def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0" def qaBundleBin = "${qaBundledGemPath}/bin/bundle" task installIntegrationTestBundler(dependsOn: unpackTarDistribution) { outputs.files fileTree("${qaBundledGemPath}/gems/bundler-1.17.3") doLast { gem(projectDir, buildDir, "bundler", "1.17.3", qaBundledGemPath) } } task installIntegrationTestGems(dependsOn: installIntegrationTestBundler) { inputs.files file("${projectDir}/qa/integration/Gemfile") inputs.files file("${projectDir}/qa/integration/integration_tests.gemspec") inputs.files file("${logstashBuildDir}/Gemfile") inputs.files file("${logstashBuildDir}/Gemfile.lock") inputs.files file("${logstashBuildDir}/logstash-core/logstash-core.gemspec") outputs.files fileTree("${qaVendorPath}") outputs.files file("${projectDir}/qa/integration/Gemfile.lock") doLast { bundleWithEnv( projectDir, buildDir, "${projectDir}/qa/integration", qaBundleBin, ['install', '--path', qaVendorPath], [LS_GEM_PATH: qaBundledGemPath, LS_GEM_HOME: qaBundledGemPath] ) } } def rubyIntegrationSpecs = project.hasProperty("rubyIntegrationSpecs") ? ((String) project.property("rubyIntegrationSpecs")).split(/\s+/).join(",") : "specs/**/*_spec.rb" def integrationTestPwd = "${projectDir}/qa/integration" project(":logstash-integration-tests") { tasks.getByPath(":logstash-integration-tests:integrationTests").configure { systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG') workingDir integrationTestPwd dependsOn installIntegrationTestGems } } task runIntegrationTests(dependsOn: [tasks.getByPath(":logstash-integration-tests:integrationTests")]) {} task generateLicenseReport(type: JavaExec) { dependsOn("generateLicenseReportInputs") dependsOn(":dependencies-report:assemble") def jarFile = project('dependencies-report').getBuildDir().toString() + "/libs/dependencies-report.jar" String licenseReportInputCSV = project.hasProperty("licenseReportInputCSV") ? project.property("licenseReportInputCSV") : "build/dependencies.csv.ruby" String licenseReportOutputCSV = project.hasProperty("licenseReportOutputCSV") ? project.property("licenseReportOutputCSV") : "build/dependencies.csv" String noticePath = "NOTICE.txt" classpath = project.files([jarFile]) main = "org.logstash.dependencies.Main" args licenseReportInputCSV, project.getBuildDir().toString() + "/licenseReportFolders.txt", licenseReportOutputCSV, noticePath } task generateLicenseReportInputs() { dependsOn subprojects.generateLicenseReport // write location of all license reports for subprojects containing artifacts that are distributed to single file StringBuilder licenseReportFolders = new StringBuilder() subprojects.findAll { s1 -> !s1.hasProperty("isDistributedArtifact") || s1.property("isDistributedArtifact") == 'true'}.each { s -> s.tasks.findAll { t2 -> t2.getName() == "generateLicenseReport" }.each { t3 -> licenseReportFolders.append(t3.outputs.files.asPath + "\n") } } if (gradle.startParameter.taskNames.contains("generateLicenseReport")) { def licenseReportPath = project.getBuildDir().toString() + "/licenseReportFolders.txt" def licenseReportFolder = new File(licenseReportPath) licenseReportFolder.delete() licenseReportFolder = new File(licenseReportPath) licenseReportFolder.createNewFile() if (licenseReportFolder.canWrite()) { licenseReportFolder.text = licenseReportFolders.toString() } } } task generatePluginsVersion(dependsOn: installDefaultGems) { doLast { rake(projectDir, buildDir, 'generate_plugins_version') } } bootstrap.dependsOn assemblyDeps runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test") check.dependsOn runIntegrationTests String artifactsVersionApi = "https://artifacts-api.elastic.co/v1/versions/" task downloadEs(type: Download) { description "Download ES Snapshot for current branch version: ${version}" doFirst { if (!project.ext.versionFound) { throw new GradleException("could not find the current artifact from the artifact-api ${artifactsVersionApi} for " + (isReleaseBuild ? "release" : "snapshot") + " version: ${version}") } } String apiResponse = artifactsVersionApi.toURL().text def dlVersions = new JsonSlurper().parseText(apiResponse) // the version string can be either '7.0.0' or '7.0.0-alpha1', i.e. with the qualifier. // in the normal PR type builds it is plain '7.0.0' // in the build invoked by the release manager it is '7.0.0-alpha1' etc. // the artifacts-api will return JSON like this: `{"versions":["5.6.13-SNAPSHOT","6.4.3-SNAPSHOT","6.5.0-SNAPSHOT","6.6.0-SNAPSHOT","7.0.0-alpha1-SNAPSHOT"]}` String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0] if (qualifiedVersion == null) { // the version is not found in the versions API, for now just set dummy values so the // task parameters like src and dest below sees these dummy values but also set // versionFound to false so that we can fail the task in the doFirst closure. // this is somewhat convoluted and there is certainly a better way to do this but // it seems to be an acceptable solution for now. project.ext.set("versionFound", false) project.ext.set("elasticsearchSnapshotURL", "http://elastic.co/invalid") project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/invalid") } else { project.ext.set("versionFound", true) String arch = "x86_64" String osName = System.properties['os.name'] if (osName ==~ /Mac OS X/) { osName = "darwin" } else { osName = "linux" } String architecture = "${osName}-${arch}" String downloadedElasticsearchName = "elasticsearch-${qualifiedVersion}-${architecture}" project.ext.set("unpackedElasticsearchName", "elasticsearch-${qualifiedVersion}") // find latest reference to last build String buildsListApi = "https://artifacts-api.elastic.co/v1/versions/${qualifiedVersion}/builds/" apiResponse = buildsListApi.toURL().text def dlBuilds = new JsonSlurper().parseText(apiResponse) String build = dlBuilds["builds"][0] // find url of build artifact String artifactApiUrl = "https://artifacts-api.elastic.co/v1/versions/${qualifiedVersion}/builds/${build}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz" apiResponse = artifactApiUrl.toURL().text def buildUrls = new JsonSlurper().parseText(apiResponse) project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"]) project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz") } src project.ext.elasticsearchSnapshotURL onlyIfNewer true inputs.file("${projectDir}/versions.yml") outputs.file(project.ext.elasticsearchDownloadLocation) dest new File(project.ext.elasticsearchDownloadLocation) doLast { System.out.println "Downloaded to ${project.ext.elasticsearchDownloadLocation}" } } task deleteLocalEs(type: Delete) { delete ('./build/elasticsearch') } task copyEs(type: Copy, dependsOn: [downloadEs, deleteLocalEs]) { from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation)) into "./build/" doLast { file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch') System.out.println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch" } } Boolean oss = System.getenv('OSS').equals('true') if (!oss) { project(":logstash-xpack") { ["rubyTests", "rubyIntegrationTests", "test"].each { tsk -> tasks.getByPath(":logstash-xpack:" + tsk).configure { dependsOn installTestGems } } tasks.getByPath(":logstash-xpack:rubyIntegrationTests").configure { dependsOn copyEs } } task runXPackUnitTests(dependsOn: [tasks.getByPath(":logstash-xpack:rubyTests")]) {} task runXPackIntegrationTests(dependsOn: [tasks.getByPath(":logstash-xpack:rubyIntegrationTests")]) {} }