From d9436dd760cb841898109b5c56745fd970dd25a6 Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 30 Jun 2017 13:50:56 +0200 Subject: [PATCH] Benchmark Tool Fixes #7629 --- .gitignore | 1 + settings.gradle | 3 +- tools/benchmark-cli/README.md | 71 ++++++ tools/benchmark-cli/build.gradle | 57 +++++ .../out/production/resources/apache.cfg | 29 +++ .../benchmark/cli/ls-benchmark.properties | 1 + .../org/logstash/benchmark/cli/metrics.json | 171 +++++++++++++ .../benchmark/cli/JRubyInstallation.java | 70 ++++++ .../benchmark/cli/LogstashInstallation.java | 227 ++++++++++++++++++ .../benchmark/cli/LsMetricsMonitor.java | 108 +++++++++ .../java/org/logstash/benchmark/cli/Main.java | 166 +++++++++++++ .../cli/cases/ApacheLogsComplex.java | 75 ++++++ .../logstash/benchmark/cli/cases/Case.java | 17 ++ .../cli/cases/GeneratorToStdout.java | 50 ++++ .../benchmark/cli/ui/LsMetricStats.java | 19 ++ .../benchmark/cli/ui/LsVersionType.java | 21 ++ .../logstash/benchmark/cli/ui/UserInput.java | 63 +++++ .../logstash/benchmark/cli/ui/UserOutput.java | 63 +++++ .../cli/util/LsBenchCompressUtil.java | 78 ++++++ .../benchmark/cli/util/LsBenchDownloader.java | 46 ++++ .../benchmark/cli/util/LsBenchFileUtil.java | 36 +++ .../benchmark/cli/util/LsBenchLsSetup.java | 39 +++ .../logstash/benchmark/cli/cases/apache.cfg | 29 +++ .../benchmark/cli/ls-benchmark.properties | 1 + .../benchmark/cli/LsMetricsMonitorTest.java | 76 ++++++ .../org/logstash/benchmark/cli/MainTest.java | 73 ++++++ .../org/logstash/benchmark/cli/metrics.json | 171 +++++++++++++ 27 files changed, 1760 insertions(+), 1 deletion(-) create mode 100644 tools/benchmark-cli/README.md create mode 100644 tools/benchmark-cli/build.gradle create mode 100644 tools/benchmark-cli/out/production/resources/apache.cfg create mode 100644 tools/benchmark-cli/out/production/resources/org/logstash/benchmark/cli/ls-benchmark.properties create mode 100644 tools/benchmark-cli/out/test/resources/org/logstash/benchmark/cli/metrics.json create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/JRubyInstallation.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LogstashInstallation.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LsMetricsMonitor.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/Main.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/ApacheLogsComplex.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/Case.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/GeneratorToStdout.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsMetricStats.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsVersionType.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserInput.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserOutput.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchCompressUtil.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchDownloader.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchFileUtil.java create mode 100644 tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchLsSetup.java create mode 100644 tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/cases/apache.cfg create mode 100644 tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/ls-benchmark.properties create mode 100644 tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/LsMetricsMonitorTest.java create mode 100644 tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/MainTest.java create mode 100644 tools/benchmark-cli/src/test/resources/org/logstash/benchmark/cli/metrics.json diff --git a/.gitignore b/.gitignore index a7a5e6419..404718983 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ qa/integration/services/installed/ **/.classpath logstash-core/bin plugins_version_docs.json +tools/benchmark-cli/out/ diff --git a/settings.gradle b/settings.gradle index daa74978e..aa7217dfb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ -include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter' +include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli' project(':logstash-core').projectDir = new File('./logstash-core') project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks') project(':ingest-converter').projectDir = new File('./tools/ingest-converter') +project(':benchmark-cli').projectDir = new File('./tools/benchmark-cli') diff --git a/tools/benchmark-cli/README.md b/tools/benchmark-cli/README.md new file mode 100644 index 000000000..de3618709 --- /dev/null +++ b/tools/benchmark-cli/README.md @@ -0,0 +1,71 @@ +### Benchmark CLI + +#### Build + +To build a self-contained archive of the benchmark tool simply run: + +```bash +gradle clean assemble +``` + +which will create the output jar under `build/libs/benchmark-cli.jar`. + +#### Running + +```bash +$ java -cp 'benchmark-cli.jar:*' org.logstash.benchmark.cli.Main --help +Option Description +------ ----------- +--distribution-version The version of a Logstash build to download + from elastic.co. +--git-hash Either a git tree (tag/branch or commit hash), + optionally prefixed by a Github username, + if ran against forks. + E.g. + 'ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7', + 'user- + name#ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7' + or 'master' +--local-path Path to the root of a local Logstash + distribution. + E.g. `/opt/logstash` +--testcase Currently available test cases are 'baseline' + and 'apache'. (default: baseline) +--workdir Working directory to store cached files in. + (default: ~/.logstash-benchmarks) +``` + +##### Example + +```bash +$ java -cp 'benchmark-cli.jar:*' org.logstash.benchmark.cli.Main --workdir=/tmp/benchmark2 --testcase=baseline --distribution-version=5.5.0 + ██╗ ██████╗ ██████╗ ███████╗████████╗ █████╗ ███████╗██╗ ██╗ + ██║ ██╔═══██╗██╔════╝ ██╔════╝╚══██╔══╝██╔══██╗██╔════╝██║ ██║ + ██║ ██║ ██║██║ ███╗███████╗ ██║ ███████║███████╗███████║ + ██║ ██║ ██║██║ ██║╚════██║ ██║ ██╔══██║╚════██║██╔══██║ + ███████╗╚██████╔╝╚██████╔╝███████║ ██║ ██║ ██║███████║██║ ██║ + ╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ + + ██████╗ ███████╗███╗ ██╗ ██████╗██╗ ██╗███╗ ███╗ █████╗ ██████╗ ██╗ ██╗ + ██╔══██╗██╔════╝████╗ ██║██╔════╝██║ ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝ + ██████╔╝█████╗ ██╔██╗ ██║██║ ███████║██╔████╔██║███████║██████╔╝█████╔╝ + ██╔══██╗██╔══╝ ██║╚██╗██║██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══██╗██╔═██╗ + ██████╔╝███████╗██║ ╚████║╚██████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║██║ ██╗ + ╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝ + + ------------------------------------------ + Benchmarking Version: 5.5.0 + Running Test Case: baseline + ------------------------------------------ + Start Time: Sat 7 22 21:28:45.4 2017 CEST + Statistical Summary: + + Elapsed Time: 33s + Num Events: 977816 + Throughput Min: 2000.00 + Throughput Max: 44500.00 + Throughput Mean: 37608.31 + Throughput StdDev: 8985.03 + Throughput Variance: 80730818.62 + Mean CPU Usage: 19.27% +``` diff --git a/tools/benchmark-cli/build.gradle b/tools/benchmark-cli/build.gradle new file mode 100644 index 000000000..e3ca5cad0 --- /dev/null +++ b/tools/benchmark-cli/build.gradle @@ -0,0 +1,57 @@ +import org.yaml.snakeyaml.Yaml + +// fetch version from Logstash's master versions.yml file +def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../../versions.yml").text) + +description = """Logstash End to End Benchmarking Utility""" +version = versionMap['logstash-core'] + +repositories { + mavenCentral() + jcenter() +} + +buildscript { + repositories { + mavenCentral() + jcenter() + } + dependencies { + classpath 'org.yaml:snakeyaml:1.17' + classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4' + } +} + +ext { + jmh = 1.18 +} + +dependencies { + compile 'net.sf.jopt-simple:jopt-simple:5.0.3' + compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3' + compile group: 'org.apache.commons', name: 'commons-compress', version: '1.14' + compile group: 'commons-io', name: 'commons-io', version: '2.5' + compile 'com.fasterxml.jackson.core:jackson-core:2.7.4' + compile 'com.fasterxml.jackson.core:jackson-databind:2.7.4' + compile "org.openjdk.jmh:jmh-core:$jmh" + testCompile group: 'com.github.tomakehurst', name: 'wiremock-standalone', version: '2.6.0' + testCompile "junit:junit:4.12" +} + +javadoc { + enabled = false +} + +test { + exclude '**/org/logstash/benchmark/cli/MainTest*' +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + baseName = 'benchmark-cli' + classifier = null + version = null +} + +assemble.dependsOn shadowJar diff --git a/tools/benchmark-cli/out/production/resources/apache.cfg b/tools/benchmark-cli/out/production/resources/apache.cfg new file mode 100644 index 000000000..8f5369d01 --- /dev/null +++ b/tools/benchmark-cli/out/production/resources/apache.cfg @@ -0,0 +1,29 @@ +input { + stdin { } +} + +filter { + grok { + match => { + "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}' + } + } + + date { + match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + geoip { + source => "clientip" + } + + useragent { + source => "agent" + target => "useragent" + } +} + +output { + stdout { codec => dots } +} diff --git a/tools/benchmark-cli/out/production/resources/org/logstash/benchmark/cli/ls-benchmark.properties b/tools/benchmark-cli/out/production/resources/org/logstash/benchmark/cli/ls-benchmark.properties new file mode 100644 index 000000000..f0ca21717 --- /dev/null +++ b/tools/benchmark-cli/out/production/resources/org/logstash/benchmark/cli/ls-benchmark.properties @@ -0,0 +1 @@ +org.logstash.benchmark.apache.dataset.url=https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_access_logs.tar.gz diff --git a/tools/benchmark-cli/out/test/resources/org/logstash/benchmark/cli/metrics.json b/tools/benchmark-cli/out/test/resources/org/logstash/benchmark/cli/metrics.json new file mode 100644 index 000000000..8168a7ea0 --- /dev/null +++ b/tools/benchmark-cli/out/test/resources/org/logstash/benchmark/cli/metrics.json @@ -0,0 +1,171 @@ +{ + "host": "localhost", + "version": "6.0.0-alpha3", + "http_address": "127.0.0.1:9600", + "id": "8bbabc13-ea58-4dcd-b94e-90ae5f692c17", + "name": "localhost", + "jvm": { + "threads": { + "count": 28, + "peak_count": 28 + }, + "mem": { + "heap_used_percent": 16, + "heap_committed_in_bytes": 259522560, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 168360000, + "non_heap_used_in_bytes": 113241032, + "non_heap_committed_in_bytes": 124989440, + "pools": { + "survivor": { + "peak_used_in_bytes": 8912896, + "used_in_bytes": 6872400, + "peak_max_in_bytes": 35782656, + "max_in_bytes": 35782656, + "committed_in_bytes": 8912896 + }, + "old": { + "peak_used_in_bytes": 141395984, + "used_in_bytes": 119128832, + "peak_max_in_bytes": 715849728, + "max_in_bytes": 715849728, + "committed_in_bytes": 178978816 + }, + "young": { + "peak_used_in_bytes": 71630848, + "used_in_bytes": 42358768, + "peak_max_in_bytes": 286326784, + "max_in_bytes": 286326784, + "committed_in_bytes": 71630848 + } + } + }, + "gc": { + "collectors": { + "old": { + "collection_time_in_millis": 89, + "collection_count": 3 + }, + "young": { + "collection_time_in_millis": 516, + "collection_count": 36 + } + } + }, + "uptime_in_millis": 15055 + }, + "process": { + "open_file_descriptors": 63, + "peak_open_file_descriptors": 63, + "max_file_descriptors": 10240, + "mem": { + "total_virtual_in_bytes": 5335916544 + }, + "cpu": { + "total_in_millis": 67919, + "percent": 63, + "load_average": { + "1m": 2.6826171875 + } + } + }, + "events": { + "in": 23101, + "filtered": 21052, + "out": 21052, + "duration_in_millis": 8939, + "queue_push_duration_in_millis": 3978 + }, + "pipelines": { + "main": { + "events": { + "duration_in_millis": 9250, + "in": 24125, + "filtered": 22076, + "out": 22076, + "queue_push_duration_in_millis": 4236 + }, + "plugins": { + "inputs": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-1", + "events": { + "out": 24125, + "queue_push_duration_in_millis": 4236 + }, + "name": "stdin" + } + ], + "filters": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-4", + "events": { + "duration_in_millis": 374, + "in": 23045, + "out": 23044 + }, + "name": "geoip" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-3", + "events": { + "duration_in_millis": 24, + "in": 23045, + "out": 23045 + }, + "matches": 23045, + "name": "date" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-5", + "events": { + "duration_in_millis": 1373, + "in": 23045, + "out": 23045 + }, + "name": "useragent" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-2", + "events": { + "duration_in_millis": 295, + "in": 23047, + "out": 23045 + }, + "matches": 23045, + "patterns_per_field": { + "message": 1 + }, + "name": "grok" + } + ], + "outputs": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-6", + "events": { + "duration_in_millis": 89, + "in": 22076, + "out": 22076 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "successes": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "failures": 0 + }, + "queue": { + "type": "memory" + } + } + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "os": {} +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/JRubyInstallation.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/JRubyInstallation.java new file mode 100644 index 000000000..2e59235de --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/JRubyInstallation.java @@ -0,0 +1,70 @@ +package org.logstash.benchmark.cli; + +import java.io.IOException; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import org.logstash.benchmark.cli.util.LsBenchDownloader; + +/** + * JRuby Installation. + */ +public final class JRubyInstallation { + + private static final String JRUBY_DEFAULT_VERSION = "9.1.10.0"; + + /** + * Path of the `gem` executable. + */ + private final Path gem; + + /** + * Path of the `rake` executable. + */ + private final Path rake; + + /** + * Path of the `ruby` executable. + */ + private final Path jruby; + + /** + * Sets up a JRuby used to bootstrap a Logstash Installation. + * @param pwd Cache Directory to work in + * @return instance + * @throws IOException On I/O Error during JRuby Download or Installation + * @throws NoSuchAlgorithmException On SSL Issue during JRuby Download + */ + public static JRubyInstallation bootstrapJruby(final Path pwd) + throws IOException, NoSuchAlgorithmException { + LsBenchDownloader.downloadDecompress( + pwd.resolve("jruby").toFile(), + String.format( + "http://jruby.org.s3.amazonaws.com/downloads/%s/jruby-bin-%s.tar.gz", + JRUBY_DEFAULT_VERSION, JRUBY_DEFAULT_VERSION + ), + false + ); + return new JRubyInstallation( + pwd.resolve("jruby").resolve(String.format("jruby-%s", JRUBY_DEFAULT_VERSION)) + ); + } + + private JRubyInstallation(final Path root) { + final Path bin = root.resolve("bin"); + this.gem = bin.resolve("gem"); + this.jruby = bin.resolve("jruby"); + this.rake = bin.resolve("rake"); + } + + public String gem() { + return gem.toAbsolutePath().toString(); + } + + public String rake() { + return rake.toAbsolutePath().toString(); + } + + public String jruby() { + return jruby.toAbsolutePath().toString(); + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LogstashInstallation.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LogstashInstallation.java new file mode 100644 index 000000000..9cbf7139d --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LogstashInstallation.java @@ -0,0 +1,227 @@ +package org.logstash.benchmark.cli; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.logstash.benchmark.cli.util.LsBenchDownloader; +import org.logstash.benchmark.cli.util.LsBenchFileUtil; + +public interface LogstashInstallation { + + /** + * Runs the Logstash Installation with the given configuration. + * @param configuration Configuration as String + * @throws IOException On I/O Exception + * @throws InterruptedException Iff Interrupted + */ + void execute(String configuration) throws IOException, InterruptedException; + + /** + * Runs the Logstash Installation with the given configuration and given File piped to + * standard input. + * @param configuration Configuration as String + * @param data Data file piped to standard input + * @throws IOException On I/O Exception + * @throws InterruptedException Iff Interrupted + */ + void execute(String configuration, File data) throws IOException, InterruptedException; + + /** + * Returns the url under which the metrics from uri `_node/stats/?pretty` can be found. + * @return Metrics URL + */ + String metrics(); + + final class FromRelease implements LogstashInstallation { + + private final LogstashInstallation base; + + public FromRelease(final File pwd, final String version) { + try { + LogstashInstallation.FromRelease.download(pwd, version); + this.base = LogstashInstallation.FromRelease.setup( + pwd.toPath().resolve(String.format("logstash-%s", version)) + ); + } catch (IOException | NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } + + @Override + public void execute(final String configuration) throws IOException, InterruptedException { + base.execute(configuration); + } + + @Override + public void execute(final String configuration, final File data) + throws IOException, InterruptedException { + base.execute(configuration, data); + } + + @Override + public String metrics() { + return base.metrics(); + } + + private static void download(final File pwd, final String version) + throws IOException, NoSuchAlgorithmException { + LsBenchDownloader.downloadDecompress( + pwd, + String.format( + "https://artifacts.elastic.co/downloads/logstash/logstash-%s.zip", + version + ), true + ); + } + + private static LogstashInstallation setup(final Path location) { + return new LogstashInstallation.FromLocalPath(location.toAbsolutePath().toString()); + } + } + + final class FromLocalPath implements LogstashInstallation { + + /** + * Metrics URL. + */ + private static final String METRICS_URL = "http://127.0.0.1:9600/_node/stats/?pretty"; + + private final Path location; + + private final ProcessBuilder pbuilder; + + public FromLocalPath(final String path) { + this.location = Paths.get(path); + this.pbuilder = new ProcessBuilder().directory(location.toFile()); + final Path jruby = location.resolve("vendor").resolve("jruby"); + LsBenchFileUtil.ensureExecutable(jruby.resolve("bin").resolve("jruby").toFile()); + final Map env = pbuilder.environment(); + env.put("JRUBY_HOME", jruby.toString()); + env.put("JAVA_OPTS", ""); + } + + @Override + public void execute(final String configuration) throws IOException, InterruptedException { + execute(configuration, null); + } + + @Override + public void execute(final String configuration, final File data) + throws IOException, InterruptedException { + final Path cfg = location.resolve("config.temp"); + Files.write( + cfg, + configuration.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ); + final Path lsbin = location.resolve("bin").resolve("logstash"); + LsBenchFileUtil.ensureExecutable(lsbin.toFile()); + final Process process = pbuilder.command(lsbin.toString(), "-w", "2", "-f", cfg.toString()).redirectOutput( + ProcessBuilder.Redirect.to(new File("/dev/null")) + ).start(); + if (data != null) { + try (final InputStream file = new FileInputStream(data); + final OutputStream out = process.getOutputStream()) { + IOUtils.copy(file, out, 16 * 4096); + } + } + if (process.waitFor() != 0) { + throw new IllegalStateException("Logstash failed to start!"); + } + LsBenchFileUtil.ensureDeleted(cfg.toFile()); + } + + @Override + public String metrics() { + return METRICS_URL; + } + } + + final class FromGithub implements LogstashInstallation { + + private final Path location; + + private final LogstashInstallation base; + + public FromGithub(final File pwd, final String hash, final JRubyInstallation jruby) { + this(pwd, "elastic", hash, jruby); + } + + public FromGithub(final File pwd, final String user, final String hash, + final JRubyInstallation jruby) { + this.location = pwd.toPath().resolve(String.format("logstash-%s", hash)); + try { + LogstashInstallation.FromGithub.download(pwd, user, hash); + this.base = LogstashInstallation.FromGithub.setup(jruby, this.location); + } catch (IOException | InterruptedException | NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } + + @Override + public void execute(final String configuration) throws IOException, InterruptedException { + base.execute(configuration); + } + + @Override + public void execute(final String configuration, final File data) + throws IOException, InterruptedException { + base.execute(configuration, data); + } + + @Override + public String metrics() { + return base.metrics(); + } + + private static void download(final File pwd, final String user, final String hash) + throws IOException, NoSuchAlgorithmException { + LsBenchDownloader.downloadDecompress( + pwd, String.format("https://github.com/%s/logstash/archive/%s.zip", user, hash), + true + ); + } + + private static LogstashInstallation setup(final JRubyInstallation ruby, final Path location) + throws IOException, InterruptedException { + final ProcessBuilder pbuilder = new ProcessBuilder().directory(location.toFile()); + final Map env = pbuilder.environment(); + env.put("JRUBY_HOME", Paths.get(ruby.gem()).getParent().getParent().toString()); + env.put("JAVA_OPTS", ""); + final File gem = new File(ruby.gem()); + if (!gem.setExecutable(true) || + pbuilder.command(gem.getAbsolutePath(), "install", "rake").inheritIO() + .start().waitFor() != 0) { + throw new IllegalStateException("Bootstrapping Rake Failed!"); + } + for (final File exec : location.resolve("bin").toFile().listFiles()) { + LsBenchFileUtil.ensureExecutable(exec); + } + final String rakepath = ruby.rake(); + final File rake = new File(rakepath); + if (!location.resolve("gradlew").toFile().setExecutable(true) || + !new File(ruby.jruby()).setExecutable(true) + || !rake.setExecutable(true) + || pbuilder.command(rakepath, "bootstrap").inheritIO().start() + .waitFor() != 0) { + throw new IllegalStateException("Bootstrapping Logstash Failed!"); + } + if (pbuilder.command(rakepath, "plugin:install-default").inheritIO().start() + .waitFor() != 0) { + throw new IllegalStateException("Bootstrapping Logstash Default Plugins Failed!"); + } + return new LogstashInstallation.FromLocalPath(location.toAbsolutePath().toString()); + } + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LsMetricsMonitor.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LsMetricsMonitor.java new file mode 100644 index 000000000..3c7647dec --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/LsMetricsMonitor.java @@ -0,0 +1,108 @@ +package org.logstash.benchmark.cli; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.openjdk.jmh.util.ListStatistics; + +public final class LsMetricsMonitor implements Callable> { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final String metrics; + + private volatile boolean running = true; + + public LsMetricsMonitor(final String metrics) { + this.metrics = metrics; + } + + @Override + public EnumMap call() { + final ListStatistics stats = new ListStatistics(); + long count = 0L; + final ListStatistics counts = new ListStatistics(); + final ListStatistics cpu = new ListStatistics(); + long start = System.nanoTime(); + while (running) { + try { + TimeUnit.SECONDS.sleep(1L); + final long[] newcounts = getCounts(); + final long newcount = newcounts[0]; + if (newcount < 0L) { + start = System.nanoTime(); + continue; + } + final long newstrt = System.nanoTime(); + stats.addValue( + (double) (newcount - count) / + (double) TimeUnit.SECONDS.convert(newstrt - start, TimeUnit.NANOSECONDS) + ); + start = newstrt; + count = newcount; + counts.addValue((double) count); + cpu.addValue(newcounts[1]); + } catch (final InterruptedException ex) { + throw new IllegalStateException(ex); + } + } + final EnumMap result = new EnumMap<>(LsMetricStats.class); + result.put(LsMetricStats.THROUGHPUT, stats); + result.put(LsMetricStats.COUNT, counts); + result.put(LsMetricStats.CPU_USAGE, cpu); + return result; + } + + public void stop() { + running = false; + } + + private long[] getCounts() { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (final CloseableHttpClient client = HttpClientBuilder.create().build()) { + baos.reset(); + try (final CloseableHttpResponse response = client + .execute(new HttpGet(metrics))) { + response.getEntity().writeTo(baos); + } catch (final IOException ex) { + return new long[]{-1L, -1L}; + } + final Map data = + OBJECT_MAPPER.readValue(baos.toByteArray(), HashMap.class); + final long count; + if (data.containsKey("pipeline")) { + count = getFiltered((Map) data.get("pipeline")); + + } else if (data.containsKey("events")) { + count = getFiltered(data); + } else { + count = -1L; + } + final long cpu; + if (count == -1L) { + cpu = -1L; + } else { + cpu = ((Number) ((Map) ((Map) data.get("process")) + .get("cpu")).get("percent")).longValue(); + } + return new long[]{count, cpu}; + } catch (final IOException ex) { + throw new IllegalStateException(ex); + } + } + + private static long getFiltered(final Map data) { + return ((Number) ((Map) (data.get("events"))) + .get("filtered")).longValue(); + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/Main.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/Main.java new file mode 100644 index 000000000..86d055fd6 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/Main.java @@ -0,0 +1,166 @@ +package org.logstash.benchmark.cli; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.AbstractMap; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import joptsimple.OptionException; +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import joptsimple.OptionSpecBuilder; +import org.logstash.benchmark.cli.cases.ApacheLogsComplex; +import org.logstash.benchmark.cli.cases.Case; +import org.logstash.benchmark.cli.cases.GeneratorToStdout; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.logstash.benchmark.cli.ui.LsVersionType; +import org.logstash.benchmark.cli.ui.UserInput; +import org.logstash.benchmark.cli.ui.UserOutput; +import org.logstash.benchmark.cli.util.LsBenchLsSetup; +import org.openjdk.jmh.util.ListStatistics; + +/** + * Benchmark CLI Main Entry Point. + */ +public final class Main { + + /** + * Ctor. + */ + private Main() { + // Utility Class. + } + + /** + * CLI Entrypoint. + * @param args Cli Args + */ + public static void main(final String... args) throws Exception { + final OptionParser parser = new OptionParser(); + final OptionSpecBuilder gitbuilder = + parser.accepts(UserInput.GIT_VERSION_PARAM, UserInput.GIT_VERSION_HELP); + final OptionSpecBuilder localbuilder = + parser.accepts(UserInput.LOCAL_VERSION_PARAM, UserInput.LOCAL_VERSION_HELP); + final OptionSpecBuilder distributionbuilder = + parser.accepts( + UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.DISTRIBUTION_VERSION_HELP + ); + final OptionSpec git = gitbuilder.requiredUnless( + UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.LOCAL_VERSION_PARAM + ).availableUnless(UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.LOCAL_VERSION_PARAM) + .withRequiredArg().ofType(String.class).forHelp(); + final OptionSpec distribution = distributionbuilder + .requiredUnless(UserInput.LOCAL_VERSION_PARAM, UserInput.GIT_VERSION_PARAM) + .availableUnless( + UserInput.LOCAL_VERSION_PARAM, UserInput.GIT_VERSION_PARAM + ).withRequiredArg().ofType(String.class).forHelp(); + final OptionSpec local = localbuilder.requiredUnless( + UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.GIT_VERSION_PARAM) + .availableUnless(UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.GIT_VERSION_PARAM) + .withRequiredArg().ofType(String.class).forHelp(); + final OptionSpec testcase = parser.accepts( + UserInput.TEST_CASE_PARAM, UserInput.TEST_CASE_HELP + ).withRequiredArg().ofType(String.class).defaultsTo(GeneratorToStdout.IDENTIFIER).forHelp(); + final OptionSpec pwd = parser.accepts( + UserInput.WORKING_DIRECTORY_PARAM, UserInput.WORKING_DIRECTORY_HELP + ).withRequiredArg().ofType(File.class).defaultsTo(UserInput.WORKING_DIRECTORY_DEFAULT) + .forHelp(); + final OptionSet options; + try { + options = parser.parse(args); + } catch (final OptionException ex) { + parser.printHelpOn(System.out); + throw ex; + } + final LsVersionType type; + final String version; + if (options.has(distribution)) { + type = LsVersionType.DISTRIBUTION; + version = options.valueOf(distribution); + } else if (options.has(git)) { + type = LsVersionType.GIT; + version = options.valueOf(git); + } else { + type = LsVersionType.LOCAL; + version = options.valueOf(local); + } + execute( + new UserOutput(System.out), loadSettings(), options.valueOf(testcase), + options.valueOf(pwd).toPath(), version, type + ); + } + + /** + * Programmatic Entrypoint. + * @param output Output Printer + * @param settings Properties + * @param test String identifier of the testcase to run + * @param cwd Working Directory to run in and write cache files to + * @param version Version of Logstash to benchmark + * @param type Type of Logstash version to benchmark + * @throws Exception On Failure + */ + public static void execute(final UserOutput output, final Properties settings, + final String test, final Path cwd, final String version, final LsVersionType type) + throws Exception { + output.printBanner(); + output.printLine(); + output.green(String.format("Benchmarking Version: %s", version)); + output.green(String.format("Running Test Case: %s", test)); + output.printLine(); + Files.createDirectories(cwd); + final LogstashInstallation logstash; + if (type == LsVersionType.GIT) { + logstash = LsBenchLsSetup.logstashFromGit( + cwd.toAbsolutePath().toString(), version, JRubyInstallation.bootstrapJruby(cwd) + ); + } else { + logstash = LsBenchLsSetup.setupLS(cwd.toAbsolutePath().toString(), version, type); + } + final Case testcase; + if (GeneratorToStdout.IDENTIFIER.equalsIgnoreCase(test)) { + testcase = new GeneratorToStdout(logstash); + } else if (ApacheLogsComplex.IDENTIFIER.equalsIgnoreCase(test)) { + testcase = new ApacheLogsComplex(logstash, cwd, settings); + } else { + throw new IllegalArgumentException(String.format("Unknown test case %s", test)); + } + output.printStartTime(); + final long start = System.currentTimeMillis(); + final AbstractMap stats = testcase.run(); + output.green("Statistical Summary:\n"); + output.green(String.format( + "Elapsed Time: %ds", + TimeUnit.SECONDS.convert( + System.currentTimeMillis() - start, TimeUnit.MILLISECONDS + ) + )); + output.green( + String.format("Num Events: %d", (long) stats.get(LsMetricStats.COUNT).getMax()) + ); + final ListStatistics throughput = stats.get(LsMetricStats.THROUGHPUT); + output.green(String.format("Throughput Min: %.2f", throughput.getMin())); + output.green(String.format("Throughput Max: %.2f", throughput.getMax())); + output.green(String.format("Throughput Mean: %.2f", throughput.getMean())); + output.green(String.format("Throughput StdDev: %.2f", throughput.getStandardDeviation())); + output.green(String.format("Throughput Variance: %.2f", throughput.getVariance())); + output.green( + String.format( + "Mean CPU Usage: %.2f%%", stats.get(LsMetricStats.CPU_USAGE).getMean() + ) + ); + } + + private static Properties loadSettings() throws IOException { + final Properties props = new Properties(); + try (final InputStream settings = + Main.class.getResource("ls-benchmark.properties").openStream()) { + props.load(settings); + } + return props; + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/ApacheLogsComplex.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/ApacheLogsComplex.java new file mode 100644 index 000000000..1f8685c2d --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/ApacheLogsComplex.java @@ -0,0 +1,75 @@ +package org.logstash.benchmark.cli.cases; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import java.util.EnumMap; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.commons.io.IOUtils; +import org.logstash.benchmark.cli.LogstashInstallation; +import org.logstash.benchmark.cli.LsMetricsMonitor; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.logstash.benchmark.cli.util.LsBenchDownloader; +import org.openjdk.jmh.util.ListStatistics; + +/** + * Test case running a set of Apache web-server logs, read from standard input, through the + * GeoIP and UserAgent filters and printing the result to standard out using the dots codec. + */ +public final class ApacheLogsComplex implements Case { + + /** + * Identifier used by the CLI interface. + */ + public static final String IDENTIFIER = "apache"; + + private final LogstashInstallation logstash; + + private final File data; + + public ApacheLogsComplex(final LogstashInstallation logstash, final Path cwd, + final Properties settings) throws IOException, NoSuchAlgorithmException { + this.data = cwd.resolve("data_apache").resolve("apache_access_logs").toFile(); + ensureDatafile(data.toPath().getParent().toFile(), settings); + this.logstash = logstash; + } + + @Override + public EnumMap run() { + final LsMetricsMonitor monitor = new LsMetricsMonitor(logstash.metrics()); + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future> future = exec.submit(monitor); + try { + final String config; + try (final InputStream cfg = ApacheLogsComplex.class + .getResourceAsStream("apache.cfg")) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + IOUtils.copy(cfg, baos); + config = baos.toString(); + } + logstash.execute(config, data); + monitor.stop(); + return future.get(20L, TimeUnit.SECONDS); + } catch (final IOException | InterruptedException | ExecutionException | TimeoutException ex) { + throw new IllegalStateException(ex); + } finally { + exec.shutdownNow(); + } + } + + private static void ensureDatafile(final File file, final Properties settings) + throws IOException, NoSuchAlgorithmException { + LsBenchDownloader.downloadDecompress( + file, settings.getProperty("org.logstash.benchmark.apache.dataset.url"), false + ); + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/Case.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/Case.java new file mode 100644 index 000000000..c88e3aca4 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/Case.java @@ -0,0 +1,17 @@ +package org.logstash.benchmark.cli.cases; + +import java.util.AbstractMap; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.openjdk.jmh.util.ListStatistics; + +/** + * Definition of an executable test case. + */ +public interface Case { + + /** + * Runs the Actual Test Case. + * @return Map Containing Test Results + */ + AbstractMap run(); +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/GeneratorToStdout.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/GeneratorToStdout.java new file mode 100644 index 000000000..2716bf813 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/cases/GeneratorToStdout.java @@ -0,0 +1,50 @@ +package org.logstash.benchmark.cli.cases; + +import java.io.IOException; +import java.util.EnumMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.logstash.benchmark.cli.LogstashInstallation; +import org.logstash.benchmark.cli.LsMetricsMonitor; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.openjdk.jmh.util.ListStatistics; + +/** + * Runs Generator Input and Outputs to StdOut. + */ +public final class GeneratorToStdout implements Case { + + /** + * Identifier used by the CLI interface. + */ + public static final String IDENTIFIER = "baseline"; + + private static final String CONFIGURATION = + "input { generator { threads => 1 count => 1000000 } } output { stdout { } }"; + + private final LogstashInstallation logstash; + + public GeneratorToStdout(final LogstashInstallation logstash) { + this.logstash = logstash; + } + + @Override + public EnumMap run() { + final LsMetricsMonitor monitor = new LsMetricsMonitor(logstash.metrics()); + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future> future = exec.submit(monitor); + try { + logstash.execute(GeneratorToStdout.CONFIGURATION); + monitor.stop(); + return future.get(20L, TimeUnit.SECONDS); + } catch (final IOException | InterruptedException | ExecutionException | TimeoutException ex) { + throw new IllegalStateException(ex); + } finally { + exec.shutdownNow(); + } + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsMetricStats.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsMetricStats.java new file mode 100644 index 000000000..90cb7d798 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsMetricStats.java @@ -0,0 +1,19 @@ +package org.logstash.benchmark.cli.ui; + +/** + * Groups of statistics this tool gathers. + */ +public enum LsMetricStats { + /** + * Statistics on the event throughput. + */ + THROUGHPUT, + /** + * Statistics on the number of events processed overall. + */ + COUNT, + /** + * Statistics on CPU usage. + */ + CPU_USAGE +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsVersionType.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsVersionType.java new file mode 100644 index 000000000..1f0787663 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/LsVersionType.java @@ -0,0 +1,21 @@ +package org.logstash.benchmark.cli.ui; + +/** + * Enum of the various types of Logstash versions. + */ +public enum LsVersionType { + /** + * A local version of Logstash that is assumed to have all dependencies installed and/or build. + */ + LOCAL, + + /** + * A release version of Logstash to be downloaded from elastic.co mirrors. + */ + DISTRIBUTION, + + /** + * A version build from a given GIT tree hash/identifier. + */ + GIT +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserInput.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserInput.java new file mode 100644 index 000000000..d8833f037 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserInput.java @@ -0,0 +1,63 @@ +package org.logstash.benchmark.cli.ui; + +import java.io.File; +import java.nio.file.Paths; + +/** + * User Input Definitions and Utility Methods. + */ +public final class UserInput { + + /** + * The Default Cache/Working-Directory. + */ + public static final File WORKING_DIRECTORY_DEFAULT = Paths.get( + System.getProperty("user.home"), ".logstash-benchmarks" + ).toFile(); + + /** + * Name of the testcase to run. + */ + public static final String TEST_CASE_PARAM = "testcase"; + + public static final String TEST_CASE_HELP = + "Currently available test cases are 'baseline' and 'apache'."; + + /** + * Version parameter to use for Logstash build downloaded from elastic.co. + */ + public static final String DISTRIBUTION_VERSION_PARAM = "distribution-version"; + + public static final String DISTRIBUTION_VERSION_HELP = + "The version of a Logstash build to download from elastic.co."; + + /** + * Version parameter to use for Logstash build form a Git has. + */ + public static final String GIT_VERSION_PARAM = "git-hash"; + + public static final String GIT_VERSION_HELP = String.join( + "\n", + "Either a git tree (tag/branch or commit hash), optionally prefixed by a Github username,", + "if ran against forks.", + "E.g. 'ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7',", + "'user-name#ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7' or 'master'" + ); + + public static final String LOCAL_VERSION_PARAM = "local-path"; + + public static final String LOCAL_VERSION_HELP = + "Path to the root of a local Logstash distribution.\n E.g. `/opt/logstash`"; + + public static final String WORKING_DIRECTORY_PARAM = "workdir"; + + public static final String WORKING_DIRECTORY_HELP = + "Working directory to store cached files in."; + + /** + * Constructor. + */ + private UserInput() { + // Utility Class + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserOutput.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserOutput.java new file mode 100644 index 000000000..7bb82a194 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/ui/UserOutput.java @@ -0,0 +1,63 @@ +package org.logstash.benchmark.cli.ui; + +import java.io.PrintStream; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; + +public final class UserOutput { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ofPattern("E")).appendLiteral(' ') + .append(DateTimeFormatter.ofPattern("L")).appendLiteral(' ') + .append(DateTimeFormatter.ofPattern("d")).appendLiteral(' ') + .append(DateTimeFormatter.ISO_LOCAL_TIME).appendLiteral(' ') + .append(DateTimeFormatter.ofPattern("yyyy")).appendLiteral(' ') + .append(DateTimeFormatter.ofPattern("z")).toFormatter(); + + private final PrintStream target; + + public UserOutput(final PrintStream target) { + this.target = target; + } + + public void printStartTime() { + green( + String.format( + "Start Time: %s", DATE_TIME_FORMATTER.format(ZonedDateTime.now().withNano(0)) + ) + ); + } + + public void printLine() { + green("------------------------------------------"); + } + + public void printBanner() { + green( + "██╗ ██████╗ ██████╗ ███████╗████████╗ █████╗ ███████╗██╗ ██╗ \n" + + "██║ ██╔═══██╗██╔════╝ ██╔════╝╚══██╔══╝██╔══██╗██╔════╝██║ ██║ \n" + + "██║ ██║ ██║██║ ███╗███████╗ ██║ ███████║███████╗███████║ \n" + + "██║ ██║ ██║██║ ██║╚════██║ ██║ ██╔══██║╚════██║██╔══██║ \n" + + "███████╗╚██████╔╝╚██████╔╝███████║ ██║ ██║ ██║███████║██║ ██║ \n" + + "╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ \n" + + " \n" + + "██████╗ ███████╗███╗ ██╗ ██████╗██╗ ██╗███╗ ███╗ █████╗ ██████╗ ██╗ ██╗\n" + + "██╔══██╗██╔════╝████╗ ██║██╔════╝██║ ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝\n" + + "██████╔╝█████╗ ██╔██╗ ██║██║ ███████║██╔████╔██║███████║██████╔╝█████╔╝ \n" + + "██╔══██╗██╔══╝ ██║╚██╗██║██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══██╗██╔═██╗ \n" + + "██████╔╝███████╗██║ ╚████║╚██████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║██║ ██╗\n" + + "╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝\n" + + " "); + } + + public void green(final String line) { + target.println(colorize(line, "\u001B[32m")); + } + + private static String colorize(final String line, final String prefix) { + final String reset = "\u001B[0m"; + return new StringBuilder(line.length() + 2 * reset.length()) + .append(prefix).append(line).append(reset).toString(); + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchCompressUtil.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchCompressUtil.java new file mode 100644 index 000000000..cfaeee4ab --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchCompressUtil.java @@ -0,0 +1,78 @@ +package org.logstash.benchmark.cli.util; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.util.UUID; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; + +/** + * Utility class for decompressing archives. + */ +final class LsBenchCompressUtil { + + private LsBenchCompressUtil() { + // Utility Class + } + + public static void unzipDir(final String zipFile, final File folder) throws IOException { + if (!folder.exists() && !folder.mkdir()) { + throw new IllegalStateException("unzip failed"); + } + try (ArchiveInputStream zis = new ZipArchiveInputStream(new FileInputStream(zipFile))) { + unpackDir(folder, zis); + } + } + + public static void gunzipDir(final File gzfile, final File file) throws IOException { + final File ball = + file.toPath().getParent().resolve(String.valueOf(UUID.randomUUID())).toFile(); + gunzipFile(gzfile, ball); + try (final TarArchiveInputStream tar = new TarArchiveInputStream( + new FileInputStream(ball))) { + unpackDir(file, tar); + } + LsBenchFileUtil.ensureDeleted(ball); + } + + private static void unpackDir(final File destination, final ArchiveInputStream archive) + throws IOException { + ArchiveEntry entry = archive.getNextEntry(); + while (entry != null) { + final File newFile = + Paths.get(destination.getAbsolutePath(), entry.getName()).toFile(); + if (!newFile.getParentFile().exists() && !newFile.getParentFile().mkdirs()) { + throw new IllegalStateException("unzip failed"); + } + if (entry.isDirectory()) { + if (!newFile.exists() && !newFile.mkdir()) { + throw new IllegalStateException("unzip failed"); + } + } else { + try (final FileOutputStream fos = new FileOutputStream(newFile)) { + IOUtils.copy(archive, fos); + } + } + entry = archive.getNextEntry(); + } + } + + private static void gunzipFile(final File gzfile, final File file) throws IOException { + try ( + final FileOutputStream uncompressed = new FileOutputStream(file); + final InputStream archive = new GzipCompressorInputStream( + new BufferedInputStream(new FileInputStream(gzfile)))) { + IOUtils.copy(archive, uncompressed); + } + LsBenchFileUtil.ensureDeleted(gzfile); + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchDownloader.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchDownloader.java new file mode 100644 index 000000000..8c4ab400e --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchDownloader.java @@ -0,0 +1,46 @@ +package org.logstash.benchmark.cli.util; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.NoSuchAlgorithmException; +import javax.net.ssl.SSLContext; +import org.apache.commons.compress.compressors.gzip.GzipUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +public final class LsBenchDownloader { + + public static void downloadDecompress(final File file, final String url, final boolean force) + throws IOException, NoSuchAlgorithmException { + if (force && file.exists()) { + LsBenchFileUtil.ensureDeleted(file); + } + if (!file.exists()) { + final File temp = file.getParentFile().toPath().resolve( + String.format("%s.download", file.getName())).toFile(); + LsBenchFileUtil.ensureDeleted(temp); + try (final OutputStream target = new BufferedOutputStream(new FileOutputStream(temp))) { + try (final CloseableHttpClient client = HttpClientBuilder.create().setSSLContext( + SSLContext.getDefault()).build()) { + try (final CloseableHttpResponse response = client + .execute(new HttpGet(url))) { + response.getEntity().writeTo(target); + } + target.flush(); + } + } + if (GzipUtils.isCompressedFilename(url)) { + LsBenchCompressUtil.gunzipDir(temp, file); + } + if (url.endsWith(".zip")) { + LsBenchCompressUtil.unzipDir(temp.getAbsolutePath(), file); + } + LsBenchFileUtil.ensureDeleted(temp); + } + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchFileUtil.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchFileUtil.java new file mode 100644 index 000000000..66e5696c1 --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchFileUtil.java @@ -0,0 +1,36 @@ +package org.logstash.benchmark.cli.util; + +import java.io.File; +import java.io.IOException; +import org.apache.commons.io.FileUtils; + +/** + * Utility class for file handling. + */ +public final class LsBenchFileUtil { + + private LsBenchFileUtil() { + //Utility Class + } + + public static void ensureDeleted(final File file) throws IOException { + if (file.exists()) { + if (file.isDirectory()) { + FileUtils.deleteDirectory(file); + } else { + if (!file.delete()) { + throw new IllegalStateException( + String.format("Failed to delete %s", file.getAbsolutePath()) + ); + } + } + } + } + + public static void ensureExecutable(final File file) { + if (!file.canExecute() && !file.setExecutable(true)) { + throw new IllegalStateException( + String.format("Failed to set %s executable", file.getAbsolutePath())); + } + } +} diff --git a/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchLsSetup.java b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchLsSetup.java new file mode 100644 index 000000000..305beb46f --- /dev/null +++ b/tools/benchmark-cli/src/main/java/org/logstash/benchmark/cli/util/LsBenchLsSetup.java @@ -0,0 +1,39 @@ +package org.logstash.benchmark.cli.util; + +import java.io.File; +import java.nio.file.Paths; +import org.logstash.benchmark.cli.JRubyInstallation; +import org.logstash.benchmark.cli.LogstashInstallation; +import org.logstash.benchmark.cli.ui.LsVersionType; + +public final class LsBenchLsSetup { + + private LsBenchLsSetup() { + } + + public static LogstashInstallation logstashFromGit(final String pwd, final String version, + final JRubyInstallation jruby) { + final File lsdir = Paths.get(pwd, "logstash").toFile(); + final LogstashInstallation logstash; + if (version.contains("#")) { + final String[] parts = version.split("#"); + logstash = new LogstashInstallation.FromGithub(lsdir, parts[0], parts[1], jruby); + } else { + logstash = new LogstashInstallation.FromGithub(lsdir, version, jruby); + } + return logstash; + } + + public static LogstashInstallation setupLS(final String pwd, final String version, + final LsVersionType type) { + final LogstashInstallation logstash; + if (type == LsVersionType.LOCAL) { + logstash = new LogstashInstallation.FromLocalPath(version); + } else { + logstash = new LogstashInstallation.FromRelease( + Paths.get(pwd, String.format("ls-release-%s", version)).toFile(), version + ); + } + return logstash; + } +} diff --git a/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/cases/apache.cfg b/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/cases/apache.cfg new file mode 100644 index 000000000..8f5369d01 --- /dev/null +++ b/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/cases/apache.cfg @@ -0,0 +1,29 @@ +input { + stdin { } +} + +filter { + grok { + match => { + "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}' + } + } + + date { + match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] + locale => en + } + + geoip { + source => "clientip" + } + + useragent { + source => "agent" + target => "useragent" + } +} + +output { + stdout { codec => dots } +} diff --git a/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/ls-benchmark.properties b/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/ls-benchmark.properties new file mode 100644 index 000000000..f0ca21717 --- /dev/null +++ b/tools/benchmark-cli/src/main/resources/org/logstash/benchmark/cli/ls-benchmark.properties @@ -0,0 +1 @@ +org.logstash.benchmark.apache.dataset.url=https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_access_logs.tar.gz diff --git a/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/LsMetricsMonitorTest.java b/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/LsMetricsMonitorTest.java new file mode 100644 index 000000000..693c8899d --- /dev/null +++ b/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/LsMetricsMonitorTest.java @@ -0,0 +1,76 @@ +package org.logstash.benchmark.cli; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.EnumMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.junit.Rule; +import org.junit.Test; +import org.logstash.benchmark.cli.ui.LsMetricStats; +import org.openjdk.jmh.util.ListStatistics; +import org.openjdk.jmh.util.Statistics; + +/** + * Tests for {@link LsMetricsMonitor}. + */ +public final class LsMetricsMonitorTest { + + @Rule + public WireMockRule http = new WireMockRule(0); + + @Test + public void parsesFilteredCount() throws Exception { + final String path = "/_node/stats/?pretty"; + http.stubFor(WireMock.get(WireMock.urlEqualTo(path)).willReturn(WireMock.okJson( + new String( + Files.readAllBytes( + Paths.get(LsMetricsMonitorTest.class.getResource("metrics.json").getPath() + )) + , StandardCharsets.UTF_8) + ))); + final ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + final LsMetricsMonitor monitor = + new LsMetricsMonitor(String.format("http://127.0.0.1:%d/%s", http.port(), path)); + final Future> future = executor.submit(monitor); + TimeUnit.SECONDS.sleep(5L); + monitor.stop(); + final Statistics stats = future.get().get(LsMetricStats.THROUGHPUT); + MatcherAssert.assertThat(stats.getMax(), CoreMatchers.is(21052.0D)); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void parsesCpuUsage() throws Exception { + final String path = "/_node/stats/?pretty"; + http.stubFor(WireMock.get(WireMock.urlEqualTo(path)).willReturn(WireMock.okJson( + new String( + Files.readAllBytes( + Paths.get(LsMetricsMonitorTest.class.getResource("metrics.json").getPath() + )) + , StandardCharsets.UTF_8) + ))); + final ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + final LsMetricsMonitor monitor = + new LsMetricsMonitor(String.format("http://127.0.0.1:%d/%s", http.port(), path)); + final Future> future = executor.submit(monitor); + TimeUnit.SECONDS.sleep(5L); + monitor.stop(); + final Statistics stats = future.get().get(LsMetricStats.CPU_USAGE); + MatcherAssert.assertThat(stats.getMax(), CoreMatchers.is(63.0D)); + } finally { + executor.shutdownNow(); + } + } +} diff --git a/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/MainTest.java b/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/MainTest.java new file mode 100644 index 000000000..882529e0f --- /dev/null +++ b/tools/benchmark-cli/src/test/java/org/logstash/benchmark/cli/MainTest.java @@ -0,0 +1,73 @@ +package org.logstash.benchmark.cli; + +import java.io.File; +import java.nio.file.Path; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.logstash.benchmark.cli.ui.UserInput; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for {@link Main}. + * todo: These tests are ignored for now, their runtime is simply unreasonable for any CI scenario. + * We will have to find a reasonable trade-off here for making sure the benchmark code is functional + * without increasing test runtime by many minutes. + */ +public final class MainTest { + + @Rule + public final TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void downloadsDependenciesForGithub() throws Exception { + final File pwd = temp.newFolder(); + Main.main(String.format("--workdir=%s", pwd.getAbsolutePath())); + final Path logstash = pwd.toPath().resolve("logstash").resolve("logstash-master"); + assertThat(logstash.toFile().exists(), is(true)); + final File jruby = pwd.toPath().resolve("jruby").toFile(); + assertThat(jruby.exists(), is(true)); + assertThat(jruby.isDirectory(), is(true)); + assertThat(logstash.resolve("Gemfile").toFile().exists(), is(true)); + } + + /** + * @throws Exception On Failure + * @todo cleanup path here, works though if you plug in a correct path + */ + @Test + public void runsAgainstLocal() throws Exception { + final File pwd = temp.newFolder(); + Main.main(String.format( + "--version=local:%s", + System.getProperty("logstash.benchmark.test.local.path") + ), String.format("--workdir=%s", pwd.getAbsolutePath())); + } + + /** + * @throws Exception On Failure + */ + @Test + public void runsAgainstRelease() throws Exception { + final File pwd = temp.newFolder(); + Main.main( + String.format("--%s=5.5.0", UserInput.DISTRIBUTION_VERSION_PARAM), + String.format("--workdir=%s", pwd.getAbsolutePath()) + ); + } + + /** + * @throws Exception On Failure + */ + @Test + public void runsApacheAgainstRelease() throws Exception { + final File pwd = temp.newFolder(); + Main.main( + String.format("--%s=5.5.0", UserInput.DISTRIBUTION_VERSION_PARAM), + String.format("--%s=apache", UserInput.TEST_CASE_PARAM), + String.format("--workdir=%s", pwd.getAbsolutePath()) + ); + } +} diff --git a/tools/benchmark-cli/src/test/resources/org/logstash/benchmark/cli/metrics.json b/tools/benchmark-cli/src/test/resources/org/logstash/benchmark/cli/metrics.json new file mode 100644 index 000000000..8168a7ea0 --- /dev/null +++ b/tools/benchmark-cli/src/test/resources/org/logstash/benchmark/cli/metrics.json @@ -0,0 +1,171 @@ +{ + "host": "localhost", + "version": "6.0.0-alpha3", + "http_address": "127.0.0.1:9600", + "id": "8bbabc13-ea58-4dcd-b94e-90ae5f692c17", + "name": "localhost", + "jvm": { + "threads": { + "count": 28, + "peak_count": 28 + }, + "mem": { + "heap_used_percent": 16, + "heap_committed_in_bytes": 259522560, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 168360000, + "non_heap_used_in_bytes": 113241032, + "non_heap_committed_in_bytes": 124989440, + "pools": { + "survivor": { + "peak_used_in_bytes": 8912896, + "used_in_bytes": 6872400, + "peak_max_in_bytes": 35782656, + "max_in_bytes": 35782656, + "committed_in_bytes": 8912896 + }, + "old": { + "peak_used_in_bytes": 141395984, + "used_in_bytes": 119128832, + "peak_max_in_bytes": 715849728, + "max_in_bytes": 715849728, + "committed_in_bytes": 178978816 + }, + "young": { + "peak_used_in_bytes": 71630848, + "used_in_bytes": 42358768, + "peak_max_in_bytes": 286326784, + "max_in_bytes": 286326784, + "committed_in_bytes": 71630848 + } + } + }, + "gc": { + "collectors": { + "old": { + "collection_time_in_millis": 89, + "collection_count": 3 + }, + "young": { + "collection_time_in_millis": 516, + "collection_count": 36 + } + } + }, + "uptime_in_millis": 15055 + }, + "process": { + "open_file_descriptors": 63, + "peak_open_file_descriptors": 63, + "max_file_descriptors": 10240, + "mem": { + "total_virtual_in_bytes": 5335916544 + }, + "cpu": { + "total_in_millis": 67919, + "percent": 63, + "load_average": { + "1m": 2.6826171875 + } + } + }, + "events": { + "in": 23101, + "filtered": 21052, + "out": 21052, + "duration_in_millis": 8939, + "queue_push_duration_in_millis": 3978 + }, + "pipelines": { + "main": { + "events": { + "duration_in_millis": 9250, + "in": 24125, + "filtered": 22076, + "out": 22076, + "queue_push_duration_in_millis": 4236 + }, + "plugins": { + "inputs": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-1", + "events": { + "out": 24125, + "queue_push_duration_in_millis": 4236 + }, + "name": "stdin" + } + ], + "filters": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-4", + "events": { + "duration_in_millis": 374, + "in": 23045, + "out": 23044 + }, + "name": "geoip" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-3", + "events": { + "duration_in_millis": 24, + "in": 23045, + "out": 23045 + }, + "matches": 23045, + "name": "date" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-5", + "events": { + "duration_in_millis": 1373, + "in": 23045, + "out": 23045 + }, + "name": "useragent" + }, + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-2", + "events": { + "duration_in_millis": 295, + "in": 23047, + "out": 23045 + }, + "matches": 23045, + "patterns_per_field": { + "message": 1 + }, + "name": "grok" + } + ], + "outputs": [ + { + "id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-6", + "events": { + "duration_in_millis": 89, + "in": 22076, + "out": 22076 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "successes": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "failures": 0 + }, + "queue": { + "type": "memory" + } + } + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "os": {} +}