elasticsearch/distribution/build.gradle
Ryan Ernst e9ca58589b
Standardize build distribution internals on os/architecture (#105842) (#105846)
The build handles platform specific code which may be for arm or x86.
Yet there are multiple ways to describe 64bit x86, and the build
converts between the two in several places. This commit consolidates on
the x64 nomenclature in most places, except where necessary (eg ML still
uses x86_64).

relates #105715
2024-02-28 17:08:42 -05:00

632 lines
23 KiB
Groovy

/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import org.apache.tools.ant.filters.FixCrLfFilter
import org.apache.tools.ant.filters.ReplaceTokens
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.internal.ConcatFilesTask
import org.elasticsearch.gradle.internal.DependenciesInfoPlugin
import org.elasticsearch.gradle.internal.NoticeTask
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin
import java.nio.file.Files
import java.nio.file.Path
plugins {
id 'base'
id 'elasticsearch.distro'
}
configurations {
log4jConfig
dependencyInfos {
attributes {
attribute(Category.CATEGORY_ATTRIBUTE, project.getObjects().named(Category.class, Category.DOCUMENTATION))
}
}
featuresMetadata {
attributes {
attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
}
}
}
dependencies {
featuresMetadata project(':server')
}
def thisProj = project
rootProject.allprojects { proj ->
proj.plugins.withType(DependenciesInfoPlugin) {
thisProj.dependencies.add("dependencyInfos", project.dependencies.project(path: proj.path))
}
}
/*****************************************************************************
* Third party dependencies report *
*****************************************************************************/
// Concatenates the dependencies CSV files into a single file
tasks.register("generateDependenciesReport", ConcatFilesTask) {
files = configurations.dependencyInfos
headerLine = "name,version,url,license,sourceURL"
target = new File(providers.systemProperty('csv')
.orElse("${project.buildDir}/reports/dependencies/es-dependencies.csv")
.get()
)
// explicitly add our dependency on the JDK
String jdkVersion = VersionProperties.versions.get('bundled_jdk').split('@')[0]
String jdkMajorVersion = jdkVersion.split('[+.]')[0]
String sourceUrl = "https://github.com/openjdk/jdk${jdkMajorVersion}u/archive/refs/tags/jdk-${jdkVersion}.tar.gz"
additionalLines << "OpenJDK,${jdkVersion},https://openjdk.java.net/,GPL-2.0-with-classpath-exception,${sourceUrl}".toString()
// Explicitly add the dependency on the RHEL UBI Docker base image
String[] rhelUbiFields = [
'Red Hat Universal Base Image minimal',
'8',
'https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8',
'Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf',
'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/8/ubi-minimal-8-source.tar.gz'
]
additionalLines << rhelUbiFields.join(',')
}
/*****************************************************************************
* Notice file *
*****************************************************************************/
// integ test zip only uses server, so a different notice file is needed there
def buildServerNoticeTaskProvider = tasks.register("buildServerNotice", NoticeTask)
// other distributions include notices from modules as well, which are added below later
def buildDefaultNoticeTaskProvider = tasks.register("buildDefaultNotice", NoticeTask) {
licensesDir new File(project(':distribution').projectDir, 'licenses')
}
// The :server and :libs projects belong to all distributions
tasks.withType(NoticeTask).configureEach {
licensesDir project(':server').file('licenses')
source project(':server').file('src/main/java')
project(':libs').subprojects.each { Project lib ->
licensesDir lib.file('licenses')
source lib.file('src/main/java')
}
}
/*****************************************************************************
* Modules *
*****************************************************************************/
String defaultOutputs = 'build/outputs/default'
String systemdOutputs = 'build/outputs/systemd'
String integTestOutputs = 'build/outputs/integ-test-only'
String externalTestOutputs = 'build/outputs/external-test'
def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) {
into defaultOutputs
}
def processSystemdOutputsTaskProvider = tasks.register("processSystemdOutputs", Sync) {
into systemdOutputs
}
def processExternalTestOutputsTaskProvider = tasks.register("processExternalTestOutputs", Sync) {
into externalTestOutputs
}
// Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
// All transport modules are included so that they may be randomized for testing
def processIntegTestOutputsTaskProvider = tasks.register("processIntegTestOutputs", Sync) {
into integTestOutputs
}
def integTestConfigFiles = fileTree("${integTestOutputs}/config") {
builtBy processIntegTestOutputsTaskProvider
}
def integTestBinFiles = fileTree("${integTestOutputs}/bin") {
builtBy processIntegTestOutputsTaskProvider
}
def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
builtBy processDefaultOutputsTaskProvider
}
def defaultBinFiles = fileTree("${defaultOutputs}/bin") {
builtBy processDefaultOutputsTaskProvider
}
def defaultConfigFiles = fileTree("${defaultOutputs}/config") {
builtBy processDefaultOutputsTaskProvider
}
def systemdModuleFiles = fileTree("${systemdOutputs}/modules") {
builtBy processSystemdOutputsTaskProvider
}
def buildIntegTestModulesTaskProvider = tasks.register("buildIntegTestModules") {
dependsOn processIntegTestOutputsTaskProvider
outputs.dir "${integTestOutputs}/modules"
}
def buildExternalTestModulesTaskProvider = tasks.register("buildExternalTestModules") {
dependsOn "processExternalTestOutputs"
outputs.dir "${externalTestOutputs}/modules"
}
def buildDefaultLog4jConfigTaskProvider = tasks.register("buildDefaultLog4jConfig") {
mustRunAfter('processDefaultOutputs')
def outputFile = file("${defaultOutputs}/log4j2.properties")
def inputFiles = fileTree('src/config').matching { include 'log4j2.properties' }
project(':modules').subprojects.each {
inputFiles = inputFiles + it.fileTree('src/main/config').matching { include 'log4j2.properties' }
}
project(':x-pack:plugin').subprojects.each {
inputFiles = inputFiles + it.fileTree('src/main/config').matching { include 'log4j2.properties' }
}
inputs.files(inputFiles)
outputs.file outputFile
doLast {
outputFile.setText('', 'UTF-8')
inputFiles.files.eachWithIndex(
{ f, i ->
if (i != 0) {
outputFile.append('\n\n', 'UTF-8')
}
outputFile.append(f.text, 'UTF-8')
}
)
}
}
ext.restTestExpansions = [
'expected.modules.count': 0
]
// we create the buildOssModules task above but fill it here so we can do a single
// loop over modules to also setup cross task dependencies and increment our modules counter
project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { Project module ->
if (module.name == 'systemd') {
// the systemd module is only included in the package distributions
return
}
File licenses = new File(module.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNoticeTaskProvider.configure {
licensesDir licenses
source module.file('src/main/java')
}
}
distro.copyModule(processDefaultOutputsTaskProvider, module)
dependencies.add('featuresMetadata', module)
if (module.name.startsWith('transport-') || (BuildParams.snapshotBuild == false && module.name == 'apm')) {
distro.copyModule(processIntegTestOutputsTaskProvider, module)
}
restTestExpansions['expected.modules.count'] += 1
}
// use licenses from each of the bundled xpack plugins
Project xpack = project(':x-pack:plugin')
xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
File licenses = new File(xpackModule.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNoticeTaskProvider.configure {
licensesDir licenses
source xpackModule.file('src/main/java')
}
}
distro.copyModule(processDefaultOutputsTaskProvider, xpackModule)
dependencies.add('featuresMetadata', xpackModule)
if (xpackModule.name.equals('core') || xpackModule.name.equals('security')) {
distro.copyModule(processIntegTestOutputsTaskProvider, xpackModule)
}
}
distro.copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd'))
project(':test:external-modules').subprojects.each { Project testModule ->
distro.copyModule(processExternalTestOutputsTaskProvider, testModule)
}
configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
apply plugin: 'elasticsearch.jdk-download'
apply plugin: 'elasticsearch.repositories'
// Setup all required JDKs
project.jdks {
['darwin', 'windows', 'linux'].each { platform ->
(platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64'] : ['x64']).each { architecture ->
"bundled_${platform}_${architecture}" {
it.platform = platform
it.version = VersionProperties.bundledJdkVersion
it.vendor = VersionProperties.bundledJdkVendor
it.architecture = architecture
}
}
}
}
// TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
/*****************************************************************************
* Properties to expand when copying packaging files *
*****************************************************************************/
configurations {
['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole'].each {
create(it) {
canBeConsumed = false
canBeResolved = true
attributes {
attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY))
attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME))
attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL))
}
}
}
all {
resolutionStrategy.dependencySubstitution {
substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:elasticsearch-log4j") because "patched to remove JndiLookup clas"}
}
}
dependencies {
libs project(':server')
// this is a special circumstance of a jar that is not a dependency of server, but needs to be in the module path
libs project(':libs:elasticsearch-preallocate')
libsVersionChecker project(':distribution:tools:java-version-checker')
libsCliLauncher project(':distribution:tools:cli-launcher')
libsServerCli project(':distribution:tools:server-cli')
libsWindowsServiceCli project(':distribution:tools:windows-service-cli')
libsAnsiConsole project(':distribution:tools:ansi-console')
libsPluginCli project(':distribution:tools:plugin-cli')
libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
libsSecurityCli project(':x-pack:plugin:security:cli')
libsGeoIpCli project(':distribution:tools:geoip-cli')
}
project.ext {
/*****************************************************************************
* Common files in all distributions *
*****************************************************************************/
libFiles =
copySpec {
// Delay by using closures, since they have not yet been configured, so no jar task exists yet.
from(configurations.libs)
into('java-version-checker') {
from(configurations.libsVersionChecker)
}
into('cli-launcher') {
from(configurations.libsCliLauncher)
}
into('tools/server-cli') {
from(configurations.libsServerCli)
}
into('tools/windows-service-cli') {
from(configurations.libsWindowsServiceCli)
}
into('tools/geoip-cli') {
from(configurations.libsGeoIpCli)
}
into('tools/plugin-cli') {
from(configurations.libsPluginCli)
}
into('tools/keystore-cli') {
from(configurations.libsKeystoreCli)
}
into('tools/security-cli') {
from(configurations.libsSecurityCli)
}
into('tools/ansi-console') {
from(configurations.libsAnsiConsole)
}
}
modulesFiles = { os, architecture ->
copySpec {
eachFile {
if (it.relativePath.segments[-2] == 'bin' || (os == 'darwin' && it.relativePath.segments[-2] == 'MacOS')) {
// bin files, wherever they are within modules (eg platform specific) should be executable
// and MacOS is an alternative to bin on macOS
it.mode = 0755
} else {
it.mode = 0644
}
}
List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64']
if (os != null) {
String platform = os + '-' + architecture
if (architecture == 'x64') {
// ML platform dir uses the x86_64 nomenclature
platform = os + '-x86_64'
}
excludePlatforms.remove(excludePlatforms.indexOf(platform))
} else {
excludePlatforms = []
}
from(defaultModulesFiles) {
// geo registers the geo_shape mapper that is overridden by
// the geo_shape mapper registered in the x-pack-spatial plugin
exclude "**/geo/**"
for (String excludePlatform : excludePlatforms) {
exclude "**/platform/${excludePlatform}/**"
}
}
if (BuildParams.isSnapshotBuild()) {
from(buildExternalTestModulesTaskProvider)
}
if (project.path.startsWith(':distribution:packages')) {
from(systemdModuleFiles)
}
}
}
integTestModulesFiles = copySpec {
from buildIntegTestModulesTaskProvider
}
configFiles = { distributionType, isTestDistro ->
copySpec {
with copySpec {
// main config files, processed with distribution specific substitutions
from '../src/config'
exclude 'log4j2.properties' // this is handled separately below
filter("tokens" : expansionsForDistribution(distributionType, isTestDistro), ReplaceTokens.class)
}
from buildDefaultLog4jConfigTaskProvider
from isTestDistro ? integTestConfigFiles : defaultConfigFiles
}
}
binFiles = { distributionType, testDistro ->
copySpec {
// non-windows files, for all distributions
with copySpec {
from '../src/bin'
exclude '*.exe'
exclude '*.bat'
eachFile { it.setMode(0755) }
filter("tokens" : expansionsForDistribution(distributionType, testDistro), ReplaceTokens.class)
}
// windows files, only for zip
if (distributionType == 'zip') {
with copySpec {
from '../src/bin'
include '*.bat'
filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
filter("tokens" : expansionsForDistribution(distributionType, testDistro), ReplaceTokens.class)
}
with copySpec {
from '../src/bin'
include '*.exe'
}
}
// module provided bin files
with copySpec {
eachFile { it.setMode(0755) }
from(testDistro ? integTestBinFiles : defaultBinFiles)
if (distributionType != 'zip') {
exclude '*.bat'
}
}
}
}
noticeFile = { testDistro ->
copySpec {
if (testDistro) {
from buildServerNoticeTaskProvider
} else {
from (buildDefaultNoticeTaskProvider) {
fileMode = 0644
}
}
}
}
jdkFiles = { Project project, String os, String architecture ->
return copySpec {
from project.jdks."bundled_${os}_${architecture}"
exclude "demo/**"
/*
* The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude
* it from the build.
*/
if ("darwin".equals(os)) {
exclude "Contents/MacOS"
}
eachFile { FileCopyDetails details ->
if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
details.mode = 0755
}
if (details.name == 'src.zip') {
details.exclude()
}
}
}
}
}
}
/**
* Build some variables that are replaced in the packages. This includes both
* scripts like bin/elasticsearch and bin/elasticsearch-plugin that a user might run and also
* scripts like postinst which are run as part of the installation.
*
* <dl>
* <dt>package.name</dt>
* <dd>The name of the project. Its sprinkled throughout the scripts.</dd>
* <dt>package.version</dt>
* <dd>The version of the project. Its mostly used to find the exact jar name.
* </dt>
* <dt>path.conf</dt>
* <dd>The default directory from which to load configuration. This is used in
* the packaging scripts, but in that context it is always
* /etc/elasticsearch. Its also used in bin/elasticsearch-plugin, where it is
* /etc/elasticsearch for the os packages but $ESHOME/config otherwise.</dd>
* <dt>path.env</dt>
* <dd>The env file sourced before bin/elasticsearch to set environment
* variables. Think /etc/defaults/elasticsearch.</dd>
* <dt>scripts.footer</dt>
* <dd>Footer appended to control scripts embedded in the distribution that is
* (almost) entirely there for cosmetic reasons.</dd>
* </dl>
*/
subprojects {
ext.expansionsForDistribution = { distributionType, isTestDistro ->
final String packagingPathData = "path.data: /var/lib/elasticsearch"
final String pathLogs = "/var/log/elasticsearch"
final String packagingPathLogs = "path.logs: ${pathLogs}"
final String packagingLoggc = "${pathLogs}/gc.log"
String licenseText
if (isTestDistro) {
licenseText = rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8')
} else {
licenseText = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt').getText('UTF-8')
}
// license text needs to be indented with a single space
licenseText = ' ' + licenseText.replace('\n', '\n ')
String footer = "# Built for ${project.name}-${project.version} " +
"(${distributionType})"
Map<String, Object> expansions = [
'project.name': project.name,
'project.version': version,
'project.minor.version': "${VersionProperties.elasticsearchVersion.major}.${VersionProperties.elasticsearchVersion.minor}",
'path.conf': [
'deb': '/etc/elasticsearch',
'rpm': '/etc/elasticsearch',
'def': '"$ES_HOME"/config'
],
'path.data': [
'deb': packagingPathData,
'rpm': packagingPathData,
'def': '#path.data: /path/to/data'
],
'path.env': [
'deb': '/etc/default/elasticsearch',
'rpm': '/etc/sysconfig/elasticsearch',
/* There isn't one of these files for tar or zip but its important to
make an empty string here so the script can properly skip it. */
'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; done',
],
'source.path.env': [
'deb': 'source /etc/default/elasticsearch',
'rpm': 'source /etc/sysconfig/elasticsearch',
'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; fi',
],
'path.logs': [
'deb': packagingPathLogs,
'rpm': packagingPathLogs,
'def': '#path.logs: /path/to/logs'
],
'loggc': [
'deb': packagingLoggc,
'rpm': packagingLoggc,
'def': 'logs/gc.log'
],
'heap.dump.path': [
'deb': "-XX:HeapDumpPath=/var/lib/elasticsearch",
'rpm': "-XX:HeapDumpPath=/var/lib/elasticsearch",
'def': "-XX:HeapDumpPath=data"
],
'error.file': [
'deb': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
'rpm': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
'def': "-XX:ErrorFile=logs/hs_err_pid%p.log"
],
'scripts.footer': [
/* Debian needs exit 0 on these scripts so we add it here and preserve
the pretty footer. */
'deb': "exit 0\n${footer}",
'def': footer
],
'es.distribution.type': [
'deb': 'deb',
'rpm': 'rpm',
'tar': 'tar',
'zip': 'zip'
],
'license.name': [
'deb': 'Elastic-License'
],
'license.text': [
'deb': licenseText,
],
]
Map<String, String> result = [:]
expansions.each { key, value ->
if (value instanceof Map) {
// 'def' is for default but its three characters like 'rpm' and 'deb'
value = value[distributionType] ?: value['def']
if (value == null) {
return
}
}
// expansions is String->Object but result is String->String, so we have to coerce the values
result[key] = value.toString()
}
return result
}
ext.assertLinesInFile = { Path path, List<String> expectedLines ->
final List<String> actualLines = Files.readAllLines(path)
int line = 0
for (final String expectedLine : expectedLines) {
final String actualLine = actualLines.get(line)
if (expectedLine != actualLine) {
throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
}
line++
}
}
}
['archives:windows-zip',
'archives:darwin-tar',
'archives:darwin-aarch64-tar',
'archives:linux-aarch64-tar',
'archives:linux-tar',
'archives:integ-test-zip',
'packages:rpm', 'packages:deb',
'packages:aarch64-rpm', 'packages:aarch64-deb',
].forEach { subName ->
Project subproject = project("${project.path}:${subName}")
Configuration configuration = configurations.create(subproject.name)
dependencies {
"${configuration.name}" project(path: subproject.path, configuration:'default')
}
}
// This artifact makes it possible for other projects to pull
// in the final log4j2.properties configuration, as it appears in the
// archive distribution.
artifacts.add('log4jConfig', file("${defaultOutputs}/log4j2.properties")) {
type 'file'
name 'log4j2.properties'
builtBy 'buildDefaultLog4jConfig'
}
tasks.named('resolveAllDependencies') {
// We don't want to build all our distribution artifacts when priming our dependency cache
configs = []
}