github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-19 04:15:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fix IT tests after version bumps (#15827)

Browse source 
This commit fixes IT failures that frequently occur after
version bumps due to missing unified release snapshot builds for
the new version.

This commit uses project specific DRA snapshot URLs for ES and Filebeat
in all cases apart from release builds.

Closes #2825
This commit is contained in:
Dimitrios Liappis 2024-01-23 15:13:51 +02:00 • committed by GitHub
parent 15e19a96c2
commit d74fea4b55
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 81 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
build.gradle
View file

@ -51,6 +51,7 @@ import org.logstash.gradle.tooling.ListProjectDependencies
import org.logstash.gradle.tooling.ExtractBundledJdkVersion import org.logstash.gradle.tooling.ExtractBundledJdkVersion
import org.logstash.gradle.tooling.SignAliasDefinitions import org.logstash.gradle.tooling.SignAliasDefinitions
import org.logstash.gradle.tooling.ToolingUtils import org.logstash.gradle.tooling.ToolingUtils
import org.logstash.gradle.tooling.SnapshotArtifactURLs
allprojects { allprojects {
group = 'org.logstash' group = 'org.logstash'
@ -174,12 +175,17 @@ tasks.register("configureArtifactInfo") {
version = "$version-$versionQualifier" version = "$version-$versionQualifier"
} }
def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
String apiResponse = artifactVersionsApi.toURL().text String apiResponse = artifactVersionsApi.toURL().text
def dlVersions = new JsonSlurper().parseText(apiResponse) def dlVersions = new JsonSlurper().parseText(apiResponse)
String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0] String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
if (qualifiedVersion == null) { if (qualifiedVersion == null) {
if (!isReleaseBuild) {
project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
return
}
throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}") throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
} }
// find latest reference to last build // find latest reference to last build
@ -190,6 +196,7 @@ tasks.register("configureArtifactInfo") {
project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}") project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
project.ext.set("stackArtifactSuffix", qualifiedVersion) project.ext.set("stackArtifactSuffix", qualifiedVersion)
project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
} }
} }
@ -432,16 +439,23 @@ tasks.register("downloadFilebeat") {
doLast { doLast {
download { download {
String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}" String beatVersion = project.ext.get("stackArtifactSuffix")
String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
project.ext.set("unpackedFilebeatName", downloadedFilebeatName) project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
// find url of build artifact if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz" def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
String apiResponse = artifactApiUrl.toURL().text project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
def buildUrls = new JsonSlurper().parseText(apiResponse) project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
} else {
// find url of build artifact
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
String apiResponse = artifactApiUrl.toURL().text
def buildUrls = new JsonSlurper().parseText(apiResponse)
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"]) project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz") project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
}
src project.ext.filebeatSnapshotUrl src project.ext.filebeatSnapshotUrl
onlyIfNewer true onlyIfNewer true
@ -477,14 +491,20 @@ tasks.register("checkEsSHA") {
description "Download ES version remote's fingerprint file" description "Download ES version remote's fingerprint file"
doLast { doLast {
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}" String esVersion = project.ext.get("stackArtifactSuffix")
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
String remoteSHA
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
// find url of build artifact def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz" remoteSHA = res.packageShaUrl
String apiResponse = artifactApiUrl.toURL().text } else {
def buildUrls = new JsonSlurper().parseText(apiResponse) // find url of build artifact
String remoteSHA = buildUrls.package.sha_url.toURL().text String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
String apiResponse = artifactApiUrl.toURL().text
def buildUrls = new JsonSlurper().parseText(apiResponse)
remoteSHA = buildUrls.package.sha_url.toURL().text
}
def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz") def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
if (localESArchive.exists()) { if (localESArchive.exists()) {
@ -512,22 +532,31 @@ tasks.register("checkEsSHA") {
tasks.register("downloadEs") { tasks.register("downloadEs") {
dependsOn = [configureArtifactInfo, checkEsSHA] dependsOn = [configureArtifactInfo, checkEsSHA]
description "Download ES Snapshot for current branch version: ${version}"
description "Download ES Snapshot for current branch version: ${version}"
inputs.file("${projectDir}/versions.yml") inputs.file("${projectDir}/versions.yml")
doLast { doLast {
download { download {
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}" String esVersion = project.ext.get("stackArtifactSuffix")
project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}") String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
// find url of build artifact project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
String apiResponse = artifactApiUrl.toURL().text
def buildUrls = new JsonSlurper().parseText(apiResponse)
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"]) if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz") def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
} else {
// find url of build artifact
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
String apiResponse = artifactApiUrl.toURL().text
def buildUrls = new JsonSlurper().parseText(apiResponse)
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
}
src project.ext.elasticsearchSnapshotURL src project.ext.elasticsearchSnapshotURL
onlyIfNewer true onlyIfNewer true

28
buildSrc/src/main/groovy/org/logstash/gradle/tooling/SnapshotArtifactURLs.groovy Normal file
View file

@ -0,0 +1,28 @@
package org.logstash.gradle.tooling
import groovy.json.JsonSlurper
/**
* Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
* We use it in all cases apart from release builds.
* */
class SnapshotArtifactURLs {
/**
* Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
* */
static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
def artifactUrls = new JsonSlurper().parseText(apiResponse)
String manifestUrl = artifactUrls["manifest_url"]
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
apiResponse = manifestUrl.toURL().text
def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
return ["packageUrl": packageUrl, "packageShaUrl": packageShaUrl]
}
}