mirror of
https://github.com/elastic/logstash.git
synced 2025-04-17 19:35:03 -04:00
Fix IT tests after version bumps (#15827)
This commit fixes IT failures that frequently occur after version bumps due to missing unified release snapshot builds for the new version. This commit uses project specific DRA snapshot URLs for ES and Filebeat in all cases apart from release builds. Closes #2825
This commit is contained in:
Dimitrios Liappis
GitHub
parent
15e19a96c2
commit
d74fea4b55
2 changed files with 81 additions and 24 deletions
77
build.gradle
77
build.gradle
|
|
@ -51,6 +51,7 @@ import org.logstash.gradle.tooling.ListProjectDependencies
|
|||
import org.logstash.gradle.tooling.ExtractBundledJdkVersion
|
||||
import org.logstash.gradle.tooling.SignAliasDefinitions
|
||||
import org.logstash.gradle.tooling.ToolingUtils
|
||||
import org.logstash.gradle.tooling.SnapshotArtifactURLs
|
||||
|
||||
allprojects {
|
||||
group = 'org.logstash'
|
||||
|
|
@ -174,12 +175,17 @@ tasks.register("configureArtifactInfo") {
|
|||
version = "$version-$versionQualifier"
|
||||
}
|
||||
|
||||
def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
|
||||
boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
|
||||
String apiResponse = artifactVersionsApi.toURL().text
|
||||
|
||||
def dlVersions = new JsonSlurper().parseText(apiResponse)
|
||||
String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
|
||||
if (qualifiedVersion == null) {
|
||||
if (!isReleaseBuild) {
|
||||
project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
|
||||
project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
|
||||
return
|
||||
}
|
||||
throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
|
||||
}
|
||||
// find latest reference to last build
|
||||
|
|
@ -190,6 +196,7 @@ tasks.register("configureArtifactInfo") {
|
|||
|
||||
project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
|
||||
project.ext.set("stackArtifactSuffix", qualifiedVersion)
|
||||
project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -432,16 +439,23 @@ tasks.register("downloadFilebeat") {
|
|||
|
||||
doLast {
|
||||
download {
|
||||
String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
|
||||
String beatVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
|
||||
project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
|
||||
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
|
||||
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
|
||||
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
|
||||
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
|
||||
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
|
||||
}
|
||||
|
||||
src project.ext.filebeatSnapshotUrl
|
||||
onlyIfNewer true
|
||||
|
|
@ -477,14 +491,20 @@ tasks.register("checkEsSHA") {
|
|||
description "Download ES version remote's fingerprint file"
|
||||
|
||||
doLast {
|
||||
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
|
||||
String esVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
|
||||
String remoteSHA
|
||||
|
||||
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String remoteSHA = buildUrls.package.sha_url.toURL().text
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
|
||||
remoteSHA = res.packageShaUrl
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
remoteSHA = buildUrls.package.sha_url.toURL().text
|
||||
}
|
||||
|
||||
def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
if (localESArchive.exists()) {
|
||||
|
|
@ -512,22 +532,31 @@ tasks.register("checkEsSHA") {
|
|||
|
||||
tasks.register("downloadEs") {
|
||||
dependsOn = [configureArtifactInfo, checkEsSHA]
|
||||
description "Download ES Snapshot for current branch version: ${version}"
|
||||
|
||||
description "Download ES Snapshot for current branch version: ${version}"
|
||||
inputs.file("${projectDir}/versions.yml")
|
||||
|
||||
doLast {
|
||||
download {
|
||||
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
|
||||
project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")
|
||||
String esVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
|
||||
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
|
||||
|
||||
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
|
||||
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
|
||||
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
|
||||
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
}
|
||||
|
||||
src project.ext.elasticsearchSnapshotURL
|
||||
onlyIfNewer true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
package org.logstash.gradle.tooling
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
/**
|
||||
* Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
|
||||
* We use it in all cases apart from release builds.
|
||||
* */
|
||||
class SnapshotArtifactURLs {
|
||||
/**
|
||||
* Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
|
||||
* */
|
||||
static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
|
||||
String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
|
||||
|
||||
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
|
||||
String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
|
||||
def artifactUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String manifestUrl = artifactUrls["manifest_url"]
|
||||
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
|
||||
apiResponse = manifestUrl.toURL().text
|
||||
def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
|
||||
String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
|
||||
|
||||
return ["packageUrl": packageUrl, "packageShaUrl": packageShaUrl]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue