mirror of
https://github.com/elastic/logstash.git
synced 2025-04-19 04:15:23 -04:00
Fix IT tests after version bumps (#15827)
This commit fixes IT failures that frequently occur after version bumps due to missing unified release snapshot builds for the new version. This commit uses project specific DRA snapshot URLs for ES and Filebeat in all cases apart from release builds. Closes #2825
This commit is contained in:
Dimitrios Liappis
GitHub
parent
15e19a96c2
commit
d74fea4b55
2 changed files with 81 additions and 24 deletions
45
build.gradle
45
build.gradle
|
|
@ -51,6 +51,7 @@ import org.logstash.gradle.tooling.ListProjectDependencies
|
|||
import org.logstash.gradle.tooling.ExtractBundledJdkVersion
|
||||
import org.logstash.gradle.tooling.SignAliasDefinitions
|
||||
import org.logstash.gradle.tooling.ToolingUtils
|
||||
import org.logstash.gradle.tooling.SnapshotArtifactURLs
|
||||
|
||||
allprojects {
|
||||
group = 'org.logstash'
|
||||
|
|
@ -174,12 +175,17 @@ tasks.register("configureArtifactInfo") {
|
|||
version = "$version-$versionQualifier"
|
||||
}
|
||||
|
||||
def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
|
||||
boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
|
||||
String apiResponse = artifactVersionsApi.toURL().text
|
||||
|
||||
def dlVersions = new JsonSlurper().parseText(apiResponse)
|
||||
String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
|
||||
if (qualifiedVersion == null) {
|
||||
if (!isReleaseBuild) {
|
||||
project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
|
||||
project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
|
||||
return
|
||||
}
|
||||
throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
|
||||
}
|
||||
// find latest reference to last build
|
||||
|
|
@ -190,6 +196,7 @@ tasks.register("configureArtifactInfo") {
|
|||
|
||||
project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
|
||||
project.ext.set("stackArtifactSuffix", qualifiedVersion)
|
||||
project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -432,9 +439,15 @@ tasks.register("downloadFilebeat") {
|
|||
|
||||
doLast {
|
||||
download {
|
||||
String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
|
||||
String beatVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
|
||||
project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
|
||||
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
|
||||
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
|
||||
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
|
|
@ -442,6 +455,7 @@ tasks.register("downloadFilebeat") {
|
|||
|
||||
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
|
||||
}
|
||||
|
||||
src project.ext.filebeatSnapshotUrl
|
||||
onlyIfNewer true
|
||||
|
|
@ -477,14 +491,20 @@ tasks.register("checkEsSHA") {
|
|||
description "Download ES version remote's fingerprint file"
|
||||
|
||||
doLast {
|
||||
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
|
||||
|
||||
String esVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
|
||||
String remoteSHA
|
||||
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
|
||||
remoteSHA = res.packageShaUrl
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String remoteSHA = buildUrls.package.sha_url.toURL().text
|
||||
remoteSHA = buildUrls.package.sha_url.toURL().text
|
||||
}
|
||||
|
||||
def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
if (localESArchive.exists()) {
|
||||
|
|
@ -512,22 +532,31 @@ tasks.register("checkEsSHA") {
|
|||
|
||||
tasks.register("downloadEs") {
|
||||
dependsOn = [configureArtifactInfo, checkEsSHA]
|
||||
description "Download ES Snapshot for current branch version: ${version}"
|
||||
|
||||
description "Download ES Snapshot for current branch version: ${version}"
|
||||
inputs.file("${projectDir}/versions.yml")
|
||||
|
||||
doLast {
|
||||
download {
|
||||
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
|
||||
project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")
|
||||
String esVersion = project.ext.get("stackArtifactSuffix")
|
||||
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
|
||||
|
||||
project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
|
||||
|
||||
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
|
||||
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
|
||||
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
|
||||
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
} else {
|
||||
// find url of build artifact
|
||||
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
|
||||
String apiResponse = artifactApiUrl.toURL().text
|
||||
|
||||
def buildUrls = new JsonSlurper().parseText(apiResponse)
|
||||
|
||||
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
|
||||
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
|
||||
}
|
||||
|
||||
src project.ext.elasticsearchSnapshotURL
|
||||
onlyIfNewer true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
package org.logstash.gradle.tooling
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
/**
|
||||
* Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
|
||||
* We use it in all cases apart from release builds.
|
||||
* */
|
||||
class SnapshotArtifactURLs {
|
||||
/**
|
||||
* Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
|
||||
* */
|
||||
static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
|
||||
String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
|
||||
|
||||
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
|
||||
String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
|
||||
def artifactUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String manifestUrl = artifactUrls["manifest_url"]
|
||||
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
|
||||
apiResponse = manifestUrl.toURL().text
|
||||
def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
|
||||
String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
|
||||
String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
|
||||
|
||||
return ["packageUrl": packageUrl, "packageShaUrl": packageShaUrl]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue