From dbf39741a04b61e6cdf346cd3e5218e996ce223d Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 11 Jul 2022 08:46:54 +0200 Subject: [PATCH] Make LoggedExec gradle task configuration cache compatible (#87621) This changes the LoggedExec task to be configuration cache compatible. We changed the implementation to use `ExecOperations` instead of extending `Exec` task. As double checked with the Gradle team this task is not planned to be made configuration cache compatible out of the box anytime soon. This is part of the effort on https://github.com/elastic/elasticsearch/issues/57918 --- .../InternalBwcGitPluginFuncTest.groovy | 2 - ...lDistributionBwcSetupPluginFuncTest.groovy | 4 +- .../gradle/internal/AntFixtureStop.groovy | 16 +- .../gradle/internal/BwcSetupExtension.java | 58 +---- .../gradle/internal/InternalBwcGitPlugin.java | 24 +- .../InternalDistributionBwcSetupPlugin.java | 4 +- .../gradle/LoggedExecFuncTest.groovy | 166 ++++++++++++++ .../org/elasticsearch/gradle/LoggedExec.java | 215 ++++++++++++++---- .../fixtures/AbstractGradleFuncTest.groovy | 4 +- distribution/docker/build.gradle | 2 +- distribution/packages/build.gradle | 11 +- plugins/discovery-azure-classic/build.gradle | 2 +- 12 files changed, 381 insertions(+), 127 deletions(-) create mode 100644 build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy index 8a10ad587cb8..756562ab0272 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy @@ -14,8 +14,6 @@ import org.gradle.testkit.runner.TaskOutcome class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { - // using LoggedExec is not cc compatible - configurationCacheCompatible = false internalBuild() buildFile << """ import org.elasticsearch.gradle.Version; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index 48b2b52820c8..fc7ccd651d73 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -17,12 +17,11 @@ import spock.lang.Unroll /* * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM */ - @IgnoreIf({ Architecture.current() == Architecture.AARCH64 }) class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { - // used LoggedExec task is not configuration cache compatible and + // Cannot serialize BwcSetupExtension containing project object configurationCacheCompatible = false internalBuild() buildFile << """ @@ -119,4 +118,5 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" + "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT") } + } diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy index 09077cb2a9ea..2d398deceee0 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy @@ -12,22 +12,20 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.internal.test.AntFixture import org.gradle.api.file.FileSystemOperations +import org.gradle.api.file.ProjectLayout import org.gradle.api.tasks.Internal +import org.gradle.process.ExecOperations import javax.inject.Inject -class AntFixtureStop extends LoggedExec implements FixtureStop { +abstract class AntFixtureStop extends LoggedExec implements FixtureStop { @Internal AntFixture fixture - @Internal - FileSystemOperations fileSystemOperations - @Inject - AntFixtureStop(FileSystemOperations fileSystemOperations) { - super(fileSystemOperations) - this.fileSystemOperations = fileSystemOperations + AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) { + super(projectLayout, execOperations, fileSystemOperations) } void setFixture(AntFixture fixture) { @@ -40,10 +38,10 @@ class AntFixtureStop extends LoggedExec implements FixtureStop { } if (Os.isFamily(Os.FAMILY_WINDOWS)) { - executable = 'Taskkill' + getExecutable().set('Taskkill') args('/PID', pid, '/F') } else { - executable = 'kill' + getExecutable().set('kill') args('-9', pid) } doLast { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 439ec1f7dc8e..28b6c1e66992 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -15,15 +15,12 @@ import org.elasticsearch.gradle.Version; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; -import org.gradle.api.Task; import org.gradle.api.logging.LogLevel; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.TaskProvider; import java.io.File; import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -64,26 +61,21 @@ public class BwcSetupExtension { return project.getTasks().register(name, LoggedExec.class, loggedExec -> { loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.usesService(bwcTaskThrottleProvider); - loggedExec.setSpoolOutput(true); - loggedExec.setWorkingDir(checkoutDir.get()); - loggedExec.doFirst(new Action() { - @Override - public void execute(Task t) { - // Execution time so that the checkouts are available - String compilerVersionInfoPath = minimumCompilerVersionPath(unreleasedVersionInfo.get().version()); - String minimumCompilerVersion = readFromFile(new File(checkoutDir.get(), compilerVersionInfoPath)); - loggedExec.environment("JAVA_HOME", getJavaHome(Integer.parseInt(minimumCompilerVersion))); - } - }); + loggedExec.getWorkingDir().set(checkoutDir.get()); + + loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> { + String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version()))); + return getJavaHome(Integer.parseInt(minimumCompilerVersion)); + })); if (Os.isFamily(Os.FAMILY_WINDOWS)) { - loggedExec.executable("cmd"); + loggedExec.getExecutable().set("cmd"); loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString()); } else { - loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString()); + loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString()); } - loggedExec.args("-g", project.getGradle().getGradleUserHomeDir()); + loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString()); if (project.getGradle().getStartParameter().isOffline()) { loggedExec.args("--offline"); } @@ -93,8 +85,7 @@ public class BwcSetupExtension { loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl); } - loggedExec.args("-Dbuild.snapshot=true"); - loggedExec.args("-Dscan.tag.NESTED"); + loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED"); final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel(); List nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG); if (nonDefaultLogLevels.contains(logLevel)) { @@ -110,8 +101,7 @@ public class BwcSetupExtension { if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) { loggedExec.args("--parallel"); } - loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version())); - loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version())); + loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString())); configAction.execute(loggedExec); }); } @@ -122,32 +112,6 @@ public class BwcSetupExtension { : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; } - private static class IndentingOutputStream extends OutputStream { - - public final byte[] indent; - private final OutputStream delegate; - - IndentingOutputStream(OutputStream delegate, Object version) { - this.delegate = delegate; - indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8); - } - - @Override - public void write(int b) throws IOException { - int[] arr = { b }; - write(arr, 0, 1); - } - - public void write(int[] bytes, int offset, int length) throws IOException { - for (int i = 0; i < bytes.length; i++) { - delegate.write(bytes[i]); - if (bytes[i] == '\n') { - delegate.write(indent); - } - } - } - } - private static String readFromFile(File file) { try { return FileUtils.readFileToString(file).trim(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java index f4de66ca0f47..6754b168c3c1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java @@ -69,29 +69,27 @@ public class InternalBwcGitPlugin implements Plugin { TaskContainer tasks = project.getTasks(); TaskProvider createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> { createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false); - createClone.setCommandLine(asList("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get())); + createClone.commandLine("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get()); }); ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties(); TaskProvider findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> { findRemote.dependsOn(createCloneTaskProvider); - // TODO Gradle should provide property based configuration here - findRemote.setWorkingDir(gitExtension.getCheckoutDir().get()); - findRemote.setCommandLine(asList("git", "remote", "-v")); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - findRemote.setStandardOutput(output); - findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, output)); }); + findRemote.getWorkingDir().set(gitExtension.getCheckoutDir()); + findRemote.commandLine("git", "remote", "-v"); + findRemote.getCaptureOutput().set(true); + findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, findRemote.getOutput())); }); }); TaskProvider addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> { addRemote.dependsOn(findRemoteTaskProvider); addRemote.onlyIf(task -> ((boolean) extraProperties.get("remoteExists")) == false); - addRemote.setWorkingDir(gitExtension.getCheckoutDir().get()); + addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get()); String remoteRepo = remote.get(); // for testing only we can override the base remote url String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo") .getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git"); - addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl)); + addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl); }); boolean isOffline = project.getGradle().getStartParameter().isOffline(); @@ -107,8 +105,8 @@ public class InternalBwcGitPlugin implements Plugin { }); fetchLatest.onlyIf(t -> isOffline == false && gitFetchLatest.get()); fetchLatest.dependsOn(addRemoteTaskProvider); - fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get()); - fetchLatest.setCommandLine(asList("git", "fetch", "--all")); + fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get()); + fetchLatest.commandLine("git", "fetch", "--all"); }); String projectPath = project.getPath(); @@ -210,7 +208,7 @@ public class InternalBwcGitPlugin implements Plugin { return os.toString().trim(); } - private static boolean isRemoteAvailable(Provider remote, ByteArrayOutputStream output) { - return new String(output.toByteArray()).lines().anyMatch(l -> l.contains(remote.get() + "\t")); + private static boolean isRemoteAvailable(Provider remote, String output) { + return output.lines().anyMatch(l -> l.contains(remote.get() + "\t")); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 8973041f6fb6..3856e9826e7f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -241,9 +241,9 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { return BuildParams.isCi() && (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false); }); - c.args(projectPath.replace('/', ':') + ":" + assembleTaskName); + c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName); if (project.getGradle().getStartParameter().isBuildCacheEnabled()) { - c.args("--build-cache"); + c.getArgs().add("--build-cache"); } c.doLast(new Action() { @Override diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy new file mode 100644 index 000000000000..302fb2bcc225 --- /dev/null +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome +import spock.lang.Ignore +import spock.lang.IgnoreIf +import spock.lang.Unroll +import spock.util.environment.OperatingSystem + +@IgnoreIf({ os.isWindows() }) +class LoggedExecFuncTest extends AbstractGradleFuncTest { + + def setup() { + buildFile << """ + // we need apply any custom plugin + // to add build-logic to the build classpath + plugins { + id 'elasticsearch.distribution-download' + } + """ + } + + @Unroll + def "can configure spooling #spooling"() { + setup: + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + commandLine 'ls', '-lh' + spoolOutput = $spooling + } + """ + when: + def result = gradleRunner("loggedExec").build() + then: + result.task(':loggedExec').outcome == TaskOutcome.SUCCESS + file("build/buffered-output/loggedExec").exists() == spooling + where: + spooling << [false, true] + } + + @Unroll + def "failed tasks output logged to console when spooling #spooling"() { + setup: + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + commandLine 'ls', 'wtf' + spoolOutput = $spooling + } + """ + when: + def result = gradleRunner("loggedExec").buildAndFail() + then: + result.task(':loggedExec').outcome == TaskOutcome.FAILED + file("build/buffered-output/loggedExec").exists() == spooling + assertOutputContains(result.output, """\ + > Task :loggedExec FAILED + Output for ls:""".stripIndent()) + assertOutputContains(result.output, "No such file or directory") + where: + spooling << [false, true] + } + + def "can capture output"() { + setup: + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + commandLine 'echo', 'HELLO' + getCaptureOutput().set(true) + doLast { + println 'OUTPUT ' + output + } + } + + """ + when: + def result = gradleRunner("loggedExec").build() + then: + result.task(':loggedExec').outcome == TaskOutcome.SUCCESS + result.getOutput().contains("OUTPUT HELLO") + } + + def "capturing output with spooling enabled is not supported"() { + setup: + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + commandLine 'echo', 'HELLO' + getCaptureOutput().set(true) + spoolOutput = true + } + """ + when: + def result = gradleRunner("loggedExec").buildAndFail() + then: + result.task(':loggedExec').outcome == TaskOutcome.FAILED + assertOutputContains(result.output, '''\ + FAILURE: Build failed with an exception. + + * What went wrong: + Execution failed for task ':loggedExec'. + > Capturing output is not supported when spoolOutput is true.'''.stripIndent()) + } + + + def "can configure output indenting"() { + setup: + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + getIndentingConsoleOutput().set("CUSTOM") + commandLine('echo', ''' + HELLO + Darkness + my old friend''') + } + """ + when: + def result = gradleRunner("loggedExec", '-q').build() + then: + result.task(':loggedExec').outcome == TaskOutcome.SUCCESS + normalized(result.output) == ''' + [CUSTOM] HELLO + [CUSTOM] Darkness + [CUSTOM] my old friend'''.stripIndent(9) + } + + def "can provide standard input"() { + setup: + file('script.sh') << """ +#!/bin/bash + +# Read the user input + +echo "Enter the user input: " +read userInput +echo "The user input is \$userInput" +""" + buildFile << """ + import org.elasticsearch.gradle.LoggedExec + tasks.register('loggedExec', LoggedExec) { + getCaptureOutput().set(true) + commandLine 'bash', 'script.sh' + getStandardInput().set('FooBar') + doLast { + println output + } + } + """ + when: + def result = gradleRunner("loggedExec").build() + then: + result.task(':loggedExec').outcome == TaskOutcome.SUCCESS + result.getOutput().contains("The user input is FooBar") + } +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index ec7479a4867c..9740a0c2f542 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -8,12 +8,19 @@ package org.elasticsearch.gradle; import org.gradle.api.Action; +import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; -import org.gradle.api.Task; import org.gradle.api.file.FileSystemOperations; +import org.gradle.api.file.ProjectLayout; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; -import org.gradle.api.tasks.Exec; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.WorkResult; import org.gradle.process.BaseExecSpec; import org.gradle.process.ExecOperations; @@ -21,13 +28,16 @@ import org.gradle.process.ExecResult; import org.gradle.process.ExecSpec; import org.gradle.process.JavaExecSpec; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import java.util.regex.Pattern; @@ -35,52 +45,70 @@ import java.util.regex.Pattern; import javax.inject.Inject; /** - * A wrapper around gradle's Exec task to capture output and log on error. + * A wrapper around gradle's exec functionality to capture output and log on error. + * This Task is configuration cache-compatible in contrast to Gradle's built-in + * Exec task implementation. */ @SuppressWarnings("unchecked") -public class LoggedExec extends Exec implements FileSystemOperationsAware { +public abstract class LoggedExec extends DefaultTask implements FileSystemOperationsAware { private static final Logger LOGGER = Logging.getLogger(LoggedExec.class); - private Consumer outputLogger; - private FileSystemOperations fileSystemOperations; + protected FileSystemOperations fileSystemOperations; + private ProjectLayout projectLayout; + private ExecOperations execOperations; + private boolean spoolOutput; + + @Input + @Optional + abstract public ListProperty getArgs(); + + @Input + @Optional + abstract public MapProperty getEnvironment(); + + @Input + abstract public Property getExecutable(); + + @Input + @Optional + abstract public Property getStandardInput(); + + @Input + @Optional + abstract public Property getIndentingConsoleOutput(); + + @Input + @Optional + abstract public Property getCaptureOutput(); + + @Input + abstract public Property getWorkingDir(); + + private String output; @Inject - public LoggedExec(FileSystemOperations fileSystemOperations) { + public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) { + this.projectLayout = projectLayout; + this.execOperations = execOperations; this.fileSystemOperations = fileSystemOperations; - if (getLogger().isInfoEnabled() == false) { - setIgnoreExitValue(true); - setSpoolOutput(false); - // We use an anonymous inner class here because Gradle cannot properly snapshot this input for the purposes of - // incremental build if we use a lambda. This ensures LoggedExec tasks that declare output can be UP-TO-DATE. - doLast(new Action() { - @Override - public void execute(Task task) { - int exitValue = LoggedExec.this.getExecutionResult().get().getExitValue(); - if (exitValue != 0) { - try { - LoggedExec.this.getLogger().error("Output for " + LoggedExec.this.getExecutable() + ":"); - outputLogger.accept(LoggedExec.this.getLogger()); - } catch (Exception e) { - throw new GradleException("Failed to read exec output", e); - } - throw new GradleException( - String.format( - "Process '%s %s' finished with non-zero exit value %d", - LoggedExec.this.getExecutable(), - LoggedExec.this.getArgs(), - exitValue - ) - ); - } - } - }); - } + getWorkingDir().convention(projectLayout.getProjectDirectory().getAsFile()); + // For now mimic default behaviour of Gradle Exec task here + getEnvironment().putAll(System.getenv()); + getCaptureOutput().convention(false); } - public void setSpoolOutput(boolean spoolOutput) { - final OutputStream out; + @TaskAction + public void run() { + if (spoolOutput && getCaptureOutput().get()) { + throw new GradleException("Capturing output is not supported when spoolOutput is true."); + } + if (getCaptureOutput().getOrElse(false) && getIndentingConsoleOutput().isPresent()) { + throw new GradleException("Capturing output is not supported when indentingConsoleOutput is configured."); + } + Consumer outputLogger; + OutputStream out; if (spoolOutput) { - File spoolFile = new File(getProject().getBuildDir() + "/buffered-output/" + this.getName()); + File spoolFile = new File(projectLayout.getBuildDirectory().dir("buffered-output").get().getAsFile(), this.getName()); out = new LazyFileOutputStream(spoolFile); outputLogger = logger -> { try { @@ -94,10 +122,59 @@ public class LoggedExec extends Exec implements FileSystemOperationsAware { }; } else { out = new ByteArrayOutputStream(); - outputLogger = logger -> { logger.error(((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8)); }; + outputLogger = getIndentingConsoleOutput().isPresent() ? logger -> {} : logger -> logger.error(byteStreamToString(out)); } - setStandardOutput(out); - setErrorOutput(out); + + OutputStream finalOutputStream = getIndentingConsoleOutput().isPresent() + ? new IndentingOutputStream(System.out, getIndentingConsoleOutput().get()) + : out; + ExecResult execResult = execOperations.exec(execSpec -> { + execSpec.setIgnoreExitValue(true); + execSpec.setStandardOutput(finalOutputStream); + execSpec.setErrorOutput(finalOutputStream); + execSpec.setExecutable(getExecutable().get()); + execSpec.setEnvironment(getEnvironment().get()); + if (getArgs().isPresent()) { + execSpec.setArgs(getArgs().get()); + } + if (getWorkingDir().isPresent()) { + execSpec.setWorkingDir(getWorkingDir().get()); + } + if (getStandardInput().isPresent()) { + try { + execSpec.setStandardInput(new ByteArrayInputStream(getStandardInput().get().getBytes("UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new GradleException("Cannot set standard input", e); + } + } + }); + int exitValue = execResult.getExitValue(); + + if (exitValue == 0 && getCaptureOutput().get()) { + output = byteStreamToString(out); + } + if (getLogger().isInfoEnabled() == false) { + if (exitValue != 0) { + try { + getLogger().error("Output for " + getExecutable().get() + ":"); + outputLogger.accept(getLogger()); + } catch (Exception e) { + throw new GradleException("Failed to read exec output", e); + } + throw new GradleException( + String.format("Process '%s %s' finished with non-zero exit value %d", getExecutable().get(), getArgs().get(), exitValue) + ); + } + } + + } + + private String byteStreamToString(OutputStream out) { + return ((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8); + } + + public void setSpoolOutput(boolean spoolOutput) { + this.spoolOutput = spoolOutput; } public static ExecResult exec(ExecOperations execOperations, Action action) { @@ -139,4 +216,60 @@ public class LoggedExec extends Exec implements FileSystemOperationsAware { public WorkResult delete(Object... objects) { return fileSystemOperations.delete(d -> d.delete(objects)); } + + @Internal + public String getOutput() { + if (getCaptureOutput().get() == false) { + throw new GradleException( + "Capturing output was not enabled. Use " + getName() + ".getCapturedOutput.set(true) to enable output capturing." + ); + } + return output; + } + + private static class IndentingOutputStream extends OutputStream { + + public final byte[] indent; + private final OutputStream delegate; + + IndentingOutputStream(OutputStream delegate, Object version) { + this.delegate = delegate; + indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8); + } + + @Override + public void write(int b) throws IOException { + int[] arr = { b }; + write(arr, 0, 1); + } + + public void write(int[] bytes, int offset, int length) throws IOException { + for (int i = 0; i < bytes.length; i++) { + delegate.write(bytes[i]); + if (bytes[i] == '\n') { + delegate.write(indent); + } + } + } + } + + public void args(Object... args) { + args(List.of(args)); + } + + public void args(List args) { + getArgs().addAll(args); + } + + public void commandLine(Object... args) { + commandLine(List.of(args)); + } + + public void commandLine(List args) { + if (args.isEmpty()) { + throw new IllegalArgumentException("Cannot set commandline with empty list."); + } + getExecutable().set(args.get(0).toString()); + getArgs().set(args.subList(1, args.size())); + } } diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 9fbfe498c61b..2f7f2cefc0ee 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -47,9 +47,9 @@ abstract class AbstractGradleFuncTest extends Specification { } def cleanup() { - if (Boolean.getBoolean('test.keep.samplebuild')) { +// if (Boolean.getBoolean('test.keep.samplebuild')) { FileUtils.copyDirectory(testProjectDir.root, new File("build/test-debug/" + testProjectDir.root.name)) - } +// } } File subProject(String subProjectPath) { diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 7f53625dd847..a3be272a09b0 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -469,7 +469,7 @@ subprojects { Project subProject -> tasks.register(exportTaskName, LoggedExec) { inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker") - executable 'docker' + executable = 'docker' outputs.file(tarFile) args "save", "-o", diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 53fcbaef8caa..6cb3bcfd6c08 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -475,15 +475,13 @@ subprojects { if (project.name.contains('deb')) { checkLicenseMetadataTaskProvider.configure { LoggedExec exec -> onlyIf dpkgExists - final ByteArrayOutputStream output = new ByteArrayOutputStream() exec.commandLine 'dpkg-deb', '--info', "${-> buildDist.get().outputs.files.filter(debFilter).singleFile}" - exec.standardOutput = output + exec.getCaptureOutput().set(true) doLast { String expectedLicense expectedLicense = "Elastic-License" final Pattern pattern = Pattern.compile("\\s*License: (.+)") - final String info = output.toString('UTF-8') - final String[] actualLines = info.split("\n") + final String[] actualLines = getOutput().split("\n") int count = 0 for (final String actualLine : actualLines) { final Matcher matcher = pattern.matcher(actualLine) @@ -507,11 +505,10 @@ subprojects { assert project.name.contains('rpm') checkLicenseMetadataTaskProvider.configure { LoggedExec exec -> onlyIf rpmExists - final ByteArrayOutputStream output = new ByteArrayOutputStream() exec.commandLine 'rpm', '-qp', '--queryformat', '%{License}', "${-> buildDist.get().outputs.files.singleFile}" - exec.standardOutput = output + exec.getCaptureOutput().set(true) doLast { - String license = output.toString('UTF-8') + String license = getOutput() String expectedLicense expectedLicense = "Elastic License" if (license != expectedLicense) { diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index b7c3e42aa0c2..de47739d382b 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -64,7 +64,7 @@ TaskProvider createKey = tasks.register("createKey", LoggedExec) { } outputs.file(keystore).withPropertyName('keystoreFile') executable = "${BuildParams.runtimeJavaHome}/bin/keytool" - standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8')) + getStandardInput().set('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n') args '-genkey', '-alias', 'test-node', '-keystore', keystore,