mirror of
https://github.com/elastic/logstash.git
synced 2025-06-28 09:46:03 -04:00
Armin
Armin Braun
parent
10a72ce015
commit
d9436dd760
27 changed files with 1760 additions and 1 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -49,3 +49,4 @@ qa/integration/services/installed/
|
|||
**/.classpath
|
||||
logstash-core/bin
|
||||
plugins_version_docs.json
|
||||
tools/benchmark-cli/out/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter'
|
||||
include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli'
|
||||
project(':logstash-core').projectDir = new File('./logstash-core')
|
||||
project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks')
|
||||
project(':ingest-converter').projectDir = new File('./tools/ingest-converter')
|
||||
project(':benchmark-cli').projectDir = new File('./tools/benchmark-cli')
|
||||
|
|
|
|||
71
tools/benchmark-cli/README.md
Normal file
71
tools/benchmark-cli/README.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
### Benchmark CLI
|
||||
|
||||
#### Build
|
||||
|
||||
To build a self-contained archive of the benchmark tool simply run:
|
||||
|
||||
```bash
|
||||
gradle clean assemble
|
||||
```
|
||||
|
||||
which will create the output jar under `build/libs/benchmark-cli.jar`.
|
||||
|
||||
#### Running
|
||||
|
||||
```bash
|
||||
$ java -cp 'benchmark-cli.jar:*' org.logstash.benchmark.cli.Main --help
|
||||
Option Description
|
||||
------ -----------
|
||||
--distribution-version <String> The version of a Logstash build to download
|
||||
from elastic.co.
|
||||
--git-hash <String> Either a git tree (tag/branch or commit hash),
|
||||
optionally prefixed by a Github username,
|
||||
if ran against forks.
|
||||
E.g.
|
||||
'ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7',
|
||||
'user-
|
||||
name#ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7'
|
||||
or 'master'
|
||||
--local-path <String> Path to the root of a local Logstash
|
||||
distribution.
|
||||
E.g. `/opt/logstash`
|
||||
--testcase <String> Currently available test cases are 'baseline'
|
||||
and 'apache'. (default: baseline)
|
||||
--workdir <File> Working directory to store cached files in.
|
||||
(default: ~/.logstash-benchmarks)
|
||||
```
|
||||
|
||||
##### Example
|
||||
|
||||
```bash
|
||||
$ java -cp 'benchmark-cli.jar:*' org.logstash.benchmark.cli.Main --workdir=/tmp/benchmark2 --testcase=baseline --distribution-version=5.5.0
|
||||
██╗ ██████╗ ██████╗ ███████╗████████╗ █████╗ ███████╗██╗ ██╗
|
||||
██║ ██╔═══██╗██╔════╝ ██╔════╝╚══██╔══╝██╔══██╗██╔════╝██║ ██║
|
||||
██║ ██║ ██║██║ ███╗███████╗ ██║ ███████║███████╗███████║
|
||||
██║ ██║ ██║██║ ██║╚════██║ ██║ ██╔══██║╚════██║██╔══██║
|
||||
███████╗╚██████╔╝╚██████╔╝███████║ ██║ ██║ ██║███████║██║ ██║
|
||||
╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝
|
||||
|
||||
██████╗ ███████╗███╗ ██╗ ██████╗██╗ ██╗███╗ ███╗ █████╗ ██████╗ ██╗ ██╗
|
||||
██╔══██╗██╔════╝████╗ ██║██╔════╝██║ ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝
|
||||
██████╔╝█████╗ ██╔██╗ ██║██║ ███████║██╔████╔██║███████║██████╔╝█████╔╝
|
||||
██╔══██╗██╔══╝ ██║╚██╗██║██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══██╗██╔═██╗
|
||||
██████╔╝███████╗██║ ╚████║╚██████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║██║ ██╗
|
||||
╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝
|
||||
|
||||
------------------------------------------
|
||||
Benchmarking Version: 5.5.0
|
||||
Running Test Case: baseline
|
||||
------------------------------------------
|
||||
Start Time: Sat 7 22 21:28:45.4 2017 CEST
|
||||
Statistical Summary:
|
||||
|
||||
Elapsed Time: 33s
|
||||
Num Events: 977816
|
||||
Throughput Min: 2000.00
|
||||
Throughput Max: 44500.00
|
||||
Throughput Mean: 37608.31
|
||||
Throughput StdDev: 8985.03
|
||||
Throughput Variance: 80730818.62
|
||||
Mean CPU Usage: 19.27%
|
||||
```
|
||||
57
tools/benchmark-cli/build.gradle
Normal file
57
tools/benchmark-cli/build.gradle
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import org.yaml.snakeyaml.Yaml
|
||||
|
||||
// fetch version from Logstash's master versions.yml file
|
||||
def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../../versions.yml").text)
|
||||
|
||||
description = """Logstash End to End Benchmarking Utility"""
|
||||
version = versionMap['logstash-core']
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
jcenter()
|
||||
}
|
||||
|
||||
buildscript {
|
||||
repositories {
|
||||
mavenCentral()
|
||||
jcenter()
|
||||
}
|
||||
dependencies {
|
||||
classpath 'org.yaml:snakeyaml:1.17'
|
||||
classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4'
|
||||
}
|
||||
}
|
||||
|
||||
ext {
|
||||
jmh = 1.18
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile 'net.sf.jopt-simple:jopt-simple:5.0.3'
|
||||
compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
|
||||
compile group: 'org.apache.commons', name: 'commons-compress', version: '1.14'
|
||||
compile group: 'commons-io', name: 'commons-io', version: '2.5'
|
||||
compile 'com.fasterxml.jackson.core:jackson-core:2.7.4'
|
||||
compile 'com.fasterxml.jackson.core:jackson-databind:2.7.4'
|
||||
compile "org.openjdk.jmh:jmh-core:$jmh"
|
||||
testCompile group: 'com.github.tomakehurst', name: 'wiremock-standalone', version: '2.6.0'
|
||||
testCompile "junit:junit:4.12"
|
||||
}
|
||||
|
||||
javadoc {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
test {
|
||||
exclude '**/org/logstash/benchmark/cli/MainTest*'
|
||||
}
|
||||
|
||||
apply plugin: 'com.github.johnrengelman.shadow'
|
||||
|
||||
shadowJar {
|
||||
baseName = 'benchmark-cli'
|
||||
classifier = null
|
||||
version = null
|
||||
}
|
||||
|
||||
assemble.dependsOn shadowJar
|
||||
29
tools/benchmark-cli/out/production/resources/apache.cfg
Normal file
29
tools/benchmark-cli/out/production/resources/apache.cfg
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
input {
|
||||
stdin { }
|
||||
}
|
||||
|
||||
filter {
|
||||
grok {
|
||||
match => {
|
||||
"message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
|
||||
}
|
||||
}
|
||||
|
||||
date {
|
||||
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
|
||||
locale => en
|
||||
}
|
||||
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
|
||||
useragent {
|
||||
source => "agent"
|
||||
target => "useragent"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => dots }
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
org.logstash.benchmark.apache.dataset.url=https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_access_logs.tar.gz
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
{
|
||||
"host": "localhost",
|
||||
"version": "6.0.0-alpha3",
|
||||
"http_address": "127.0.0.1:9600",
|
||||
"id": "8bbabc13-ea58-4dcd-b94e-90ae5f692c17",
|
||||
"name": "localhost",
|
||||
"jvm": {
|
||||
"threads": {
|
||||
"count": 28,
|
||||
"peak_count": 28
|
||||
},
|
||||
"mem": {
|
||||
"heap_used_percent": 16,
|
||||
"heap_committed_in_bytes": 259522560,
|
||||
"heap_max_in_bytes": 1037959168,
|
||||
"heap_used_in_bytes": 168360000,
|
||||
"non_heap_used_in_bytes": 113241032,
|
||||
"non_heap_committed_in_bytes": 124989440,
|
||||
"pools": {
|
||||
"survivor": {
|
||||
"peak_used_in_bytes": 8912896,
|
||||
"used_in_bytes": 6872400,
|
||||
"peak_max_in_bytes": 35782656,
|
||||
"max_in_bytes": 35782656,
|
||||
"committed_in_bytes": 8912896
|
||||
},
|
||||
"old": {
|
||||
"peak_used_in_bytes": 141395984,
|
||||
"used_in_bytes": 119128832,
|
||||
"peak_max_in_bytes": 715849728,
|
||||
"max_in_bytes": 715849728,
|
||||
"committed_in_bytes": 178978816
|
||||
},
|
||||
"young": {
|
||||
"peak_used_in_bytes": 71630848,
|
||||
"used_in_bytes": 42358768,
|
||||
"peak_max_in_bytes": 286326784,
|
||||
"max_in_bytes": 286326784,
|
||||
"committed_in_bytes": 71630848
|
||||
}
|
||||
}
|
||||
},
|
||||
"gc": {
|
||||
"collectors": {
|
||||
"old": {
|
||||
"collection_time_in_millis": 89,
|
||||
"collection_count": 3
|
||||
},
|
||||
"young": {
|
||||
"collection_time_in_millis": 516,
|
||||
"collection_count": 36
|
||||
}
|
||||
}
|
||||
},
|
||||
"uptime_in_millis": 15055
|
||||
},
|
||||
"process": {
|
||||
"open_file_descriptors": 63,
|
||||
"peak_open_file_descriptors": 63,
|
||||
"max_file_descriptors": 10240,
|
||||
"mem": {
|
||||
"total_virtual_in_bytes": 5335916544
|
||||
},
|
||||
"cpu": {
|
||||
"total_in_millis": 67919,
|
||||
"percent": 63,
|
||||
"load_average": {
|
||||
"1m": 2.6826171875
|
||||
}
|
||||
}
|
||||
},
|
||||
"events": {
|
||||
"in": 23101,
|
||||
"filtered": 21052,
|
||||
"out": 21052,
|
||||
"duration_in_millis": 8939,
|
||||
"queue_push_duration_in_millis": 3978
|
||||
},
|
||||
"pipelines": {
|
||||
"main": {
|
||||
"events": {
|
||||
"duration_in_millis": 9250,
|
||||
"in": 24125,
|
||||
"filtered": 22076,
|
||||
"out": 22076,
|
||||
"queue_push_duration_in_millis": 4236
|
||||
},
|
||||
"plugins": {
|
||||
"inputs": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-1",
|
||||
"events": {
|
||||
"out": 24125,
|
||||
"queue_push_duration_in_millis": 4236
|
||||
},
|
||||
"name": "stdin"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-4",
|
||||
"events": {
|
||||
"duration_in_millis": 374,
|
||||
"in": 23045,
|
||||
"out": 23044
|
||||
},
|
||||
"name": "geoip"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-3",
|
||||
"events": {
|
||||
"duration_in_millis": 24,
|
||||
"in": 23045,
|
||||
"out": 23045
|
||||
},
|
||||
"matches": 23045,
|
||||
"name": "date"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-5",
|
||||
"events": {
|
||||
"duration_in_millis": 1373,
|
||||
"in": 23045,
|
||||
"out": 23045
|
||||
},
|
||||
"name": "useragent"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-2",
|
||||
"events": {
|
||||
"duration_in_millis": 295,
|
||||
"in": 23047,
|
||||
"out": 23045
|
||||
},
|
||||
"matches": 23045,
|
||||
"patterns_per_field": {
|
||||
"message": 1
|
||||
},
|
||||
"name": "grok"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-6",
|
||||
"events": {
|
||||
"duration_in_millis": 89,
|
||||
"in": 22076,
|
||||
"out": 22076
|
||||
},
|
||||
"name": "stdout"
|
||||
}
|
||||
]
|
||||
},
|
||||
"reloads": {
|
||||
"last_error": null,
|
||||
"successes": 0,
|
||||
"last_success_timestamp": null,
|
||||
"last_failure_timestamp": null,
|
||||
"failures": 0
|
||||
},
|
||||
"queue": {
|
||||
"type": "memory"
|
||||
}
|
||||
}
|
||||
},
|
||||
"reloads": {
|
||||
"successes": 0,
|
||||
"failures": 0
|
||||
},
|
||||
"os": {}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import org.logstash.benchmark.cli.util.LsBenchDownloader;
|
||||
|
||||
/**
|
||||
* JRuby Installation.
|
||||
*/
|
||||
public final class JRubyInstallation {
|
||||
|
||||
private static final String JRUBY_DEFAULT_VERSION = "9.1.10.0";
|
||||
|
||||
/**
|
||||
* Path of the `gem` executable.
|
||||
*/
|
||||
private final Path gem;
|
||||
|
||||
/**
|
||||
* Path of the `rake` executable.
|
||||
*/
|
||||
private final Path rake;
|
||||
|
||||
/**
|
||||
* Path of the `ruby` executable.
|
||||
*/
|
||||
private final Path jruby;
|
||||
|
||||
/**
|
||||
* Sets up a JRuby used to bootstrap a Logstash Installation.
|
||||
* @param pwd Cache Directory to work in
|
||||
* @return instance
|
||||
* @throws IOException On I/O Error during JRuby Download or Installation
|
||||
* @throws NoSuchAlgorithmException On SSL Issue during JRuby Download
|
||||
*/
|
||||
public static JRubyInstallation bootstrapJruby(final Path pwd)
|
||||
throws IOException, NoSuchAlgorithmException {
|
||||
LsBenchDownloader.downloadDecompress(
|
||||
pwd.resolve("jruby").toFile(),
|
||||
String.format(
|
||||
"http://jruby.org.s3.amazonaws.com/downloads/%s/jruby-bin-%s.tar.gz",
|
||||
JRUBY_DEFAULT_VERSION, JRUBY_DEFAULT_VERSION
|
||||
),
|
||||
false
|
||||
);
|
||||
return new JRubyInstallation(
|
||||
pwd.resolve("jruby").resolve(String.format("jruby-%s", JRUBY_DEFAULT_VERSION))
|
||||
);
|
||||
}
|
||||
|
||||
private JRubyInstallation(final Path root) {
|
||||
final Path bin = root.resolve("bin");
|
||||
this.gem = bin.resolve("gem");
|
||||
this.jruby = bin.resolve("jruby");
|
||||
this.rake = bin.resolve("rake");
|
||||
}
|
||||
|
||||
public String gem() {
|
||||
return gem.toAbsolutePath().toString();
|
||||
}
|
||||
|
||||
public String rake() {
|
||||
return rake.toAbsolutePath().toString();
|
||||
}
|
||||
|
||||
public String jruby() {
|
||||
return jruby.toAbsolutePath().toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,227 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.Map;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.logstash.benchmark.cli.util.LsBenchDownloader;
|
||||
import org.logstash.benchmark.cli.util.LsBenchFileUtil;
|
||||
|
||||
public interface LogstashInstallation {
|
||||
|
||||
/**
|
||||
* Runs the Logstash Installation with the given configuration.
|
||||
* @param configuration Configuration as String
|
||||
* @throws IOException On I/O Exception
|
||||
* @throws InterruptedException Iff Interrupted
|
||||
*/
|
||||
void execute(String configuration) throws IOException, InterruptedException;
|
||||
|
||||
/**
|
||||
* Runs the Logstash Installation with the given configuration and given File piped to
|
||||
* standard input.
|
||||
* @param configuration Configuration as String
|
||||
* @param data Data file piped to standard input
|
||||
* @throws IOException On I/O Exception
|
||||
* @throws InterruptedException Iff Interrupted
|
||||
*/
|
||||
void execute(String configuration, File data) throws IOException, InterruptedException;
|
||||
|
||||
/**
|
||||
* Returns the url under which the metrics from uri `_node/stats/?pretty` can be found.
|
||||
* @return Metrics URL
|
||||
*/
|
||||
String metrics();
|
||||
|
||||
final class FromRelease implements LogstashInstallation {
|
||||
|
||||
private final LogstashInstallation base;
|
||||
|
||||
public FromRelease(final File pwd, final String version) {
|
||||
try {
|
||||
LogstashInstallation.FromRelease.download(pwd, version);
|
||||
this.base = LogstashInstallation.FromRelease.setup(
|
||||
pwd.toPath().resolve(String.format("logstash-%s", version))
|
||||
);
|
||||
} catch (IOException | NoSuchAlgorithmException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration) throws IOException, InterruptedException {
|
||||
base.execute(configuration);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration, final File data)
|
||||
throws IOException, InterruptedException {
|
||||
base.execute(configuration, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String metrics() {
|
||||
return base.metrics();
|
||||
}
|
||||
|
||||
private static void download(final File pwd, final String version)
|
||||
throws IOException, NoSuchAlgorithmException {
|
||||
LsBenchDownloader.downloadDecompress(
|
||||
pwd,
|
||||
String.format(
|
||||
"https://artifacts.elastic.co/downloads/logstash/logstash-%s.zip",
|
||||
version
|
||||
), true
|
||||
);
|
||||
}
|
||||
|
||||
private static LogstashInstallation setup(final Path location) {
|
||||
return new LogstashInstallation.FromLocalPath(location.toAbsolutePath().toString());
|
||||
}
|
||||
}
|
||||
|
||||
final class FromLocalPath implements LogstashInstallation {
|
||||
|
||||
/**
|
||||
* Metrics URL.
|
||||
*/
|
||||
private static final String METRICS_URL = "http://127.0.0.1:9600/_node/stats/?pretty";
|
||||
|
||||
private final Path location;
|
||||
|
||||
private final ProcessBuilder pbuilder;
|
||||
|
||||
public FromLocalPath(final String path) {
|
||||
this.location = Paths.get(path);
|
||||
this.pbuilder = new ProcessBuilder().directory(location.toFile());
|
||||
final Path jruby = location.resolve("vendor").resolve("jruby");
|
||||
LsBenchFileUtil.ensureExecutable(jruby.resolve("bin").resolve("jruby").toFile());
|
||||
final Map<String, String> env = pbuilder.environment();
|
||||
env.put("JRUBY_HOME", jruby.toString());
|
||||
env.put("JAVA_OPTS", "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration) throws IOException, InterruptedException {
|
||||
execute(configuration, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration, final File data)
|
||||
throws IOException, InterruptedException {
|
||||
final Path cfg = location.resolve("config.temp");
|
||||
Files.write(
|
||||
cfg,
|
||||
configuration.getBytes(StandardCharsets.UTF_8),
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.CREATE
|
||||
);
|
||||
final Path lsbin = location.resolve("bin").resolve("logstash");
|
||||
LsBenchFileUtil.ensureExecutable(lsbin.toFile());
|
||||
final Process process = pbuilder.command(lsbin.toString(), "-w", "2", "-f", cfg.toString()).redirectOutput(
|
||||
ProcessBuilder.Redirect.to(new File("/dev/null"))
|
||||
).start();
|
||||
if (data != null) {
|
||||
try (final InputStream file = new FileInputStream(data);
|
||||
final OutputStream out = process.getOutputStream()) {
|
||||
IOUtils.copy(file, out, 16 * 4096);
|
||||
}
|
||||
}
|
||||
if (process.waitFor() != 0) {
|
||||
throw new IllegalStateException("Logstash failed to start!");
|
||||
}
|
||||
LsBenchFileUtil.ensureDeleted(cfg.toFile());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String metrics() {
|
||||
return METRICS_URL;
|
||||
}
|
||||
}
|
||||
|
||||
final class FromGithub implements LogstashInstallation {
|
||||
|
||||
private final Path location;
|
||||
|
||||
private final LogstashInstallation base;
|
||||
|
||||
public FromGithub(final File pwd, final String hash, final JRubyInstallation jruby) {
|
||||
this(pwd, "elastic", hash, jruby);
|
||||
}
|
||||
|
||||
public FromGithub(final File pwd, final String user, final String hash,
|
||||
final JRubyInstallation jruby) {
|
||||
this.location = pwd.toPath().resolve(String.format("logstash-%s", hash));
|
||||
try {
|
||||
LogstashInstallation.FromGithub.download(pwd, user, hash);
|
||||
this.base = LogstashInstallation.FromGithub.setup(jruby, this.location);
|
||||
} catch (IOException | InterruptedException | NoSuchAlgorithmException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration) throws IOException, InterruptedException {
|
||||
base.execute(configuration);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(final String configuration, final File data)
|
||||
throws IOException, InterruptedException {
|
||||
base.execute(configuration, data);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String metrics() {
|
||||
return base.metrics();
|
||||
}
|
||||
|
||||
private static void download(final File pwd, final String user, final String hash)
|
||||
throws IOException, NoSuchAlgorithmException {
|
||||
LsBenchDownloader.downloadDecompress(
|
||||
pwd, String.format("https://github.com/%s/logstash/archive/%s.zip", user, hash),
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
private static LogstashInstallation setup(final JRubyInstallation ruby, final Path location)
|
||||
throws IOException, InterruptedException {
|
||||
final ProcessBuilder pbuilder = new ProcessBuilder().directory(location.toFile());
|
||||
final Map<String, String> env = pbuilder.environment();
|
||||
env.put("JRUBY_HOME", Paths.get(ruby.gem()).getParent().getParent().toString());
|
||||
env.put("JAVA_OPTS", "");
|
||||
final File gem = new File(ruby.gem());
|
||||
if (!gem.setExecutable(true) ||
|
||||
pbuilder.command(gem.getAbsolutePath(), "install", "rake").inheritIO()
|
||||
.start().waitFor() != 0) {
|
||||
throw new IllegalStateException("Bootstrapping Rake Failed!");
|
||||
}
|
||||
for (final File exec : location.resolve("bin").toFile().listFiles()) {
|
||||
LsBenchFileUtil.ensureExecutable(exec);
|
||||
}
|
||||
final String rakepath = ruby.rake();
|
||||
final File rake = new File(rakepath);
|
||||
if (!location.resolve("gradlew").toFile().setExecutable(true) ||
|
||||
!new File(ruby.jruby()).setExecutable(true)
|
||||
|| !rake.setExecutable(true)
|
||||
|| pbuilder.command(rakepath, "bootstrap").inheritIO().start()
|
||||
.waitFor() != 0) {
|
||||
throw new IllegalStateException("Bootstrapping Logstash Failed!");
|
||||
}
|
||||
if (pbuilder.command(rakepath, "plugin:install-default").inheritIO().start()
|
||||
.waitFor() != 0) {
|
||||
throw new IllegalStateException("Bootstrapping Logstash Default Plugins Failed!");
|
||||
}
|
||||
return new LogstashInstallation.FromLocalPath(location.toAbsolutePath().toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
|
||||
public final class LsMetricsMonitor implements Callable<EnumMap<LsMetricStats, ListStatistics>> {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final String metrics;
|
||||
|
||||
private volatile boolean running = true;
|
||||
|
||||
public LsMetricsMonitor(final String metrics) {
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EnumMap<LsMetricStats, ListStatistics> call() {
|
||||
final ListStatistics stats = new ListStatistics();
|
||||
long count = 0L;
|
||||
final ListStatistics counts = new ListStatistics();
|
||||
final ListStatistics cpu = new ListStatistics();
|
||||
long start = System.nanoTime();
|
||||
while (running) {
|
||||
try {
|
||||
TimeUnit.SECONDS.sleep(1L);
|
||||
final long[] newcounts = getCounts();
|
||||
final long newcount = newcounts[0];
|
||||
if (newcount < 0L) {
|
||||
start = System.nanoTime();
|
||||
continue;
|
||||
}
|
||||
final long newstrt = System.nanoTime();
|
||||
stats.addValue(
|
||||
(double) (newcount - count) /
|
||||
(double) TimeUnit.SECONDS.convert(newstrt - start, TimeUnit.NANOSECONDS)
|
||||
);
|
||||
start = newstrt;
|
||||
count = newcount;
|
||||
counts.addValue((double) count);
|
||||
cpu.addValue(newcounts[1]);
|
||||
} catch (final InterruptedException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
final EnumMap<LsMetricStats, ListStatistics> result = new EnumMap<>(LsMetricStats.class);
|
||||
result.put(LsMetricStats.THROUGHPUT, stats);
|
||||
result.put(LsMetricStats.COUNT, counts);
|
||||
result.put(LsMetricStats.CPU_USAGE, cpu);
|
||||
return result;
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
running = false;
|
||||
}
|
||||
|
||||
private long[] getCounts() {
|
||||
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
try (final CloseableHttpClient client = HttpClientBuilder.create().build()) {
|
||||
baos.reset();
|
||||
try (final CloseableHttpResponse response = client
|
||||
.execute(new HttpGet(metrics))) {
|
||||
response.getEntity().writeTo(baos);
|
||||
} catch (final IOException ex) {
|
||||
return new long[]{-1L, -1L};
|
||||
}
|
||||
final Map<String, Object> data =
|
||||
OBJECT_MAPPER.readValue(baos.toByteArray(), HashMap.class);
|
||||
final long count;
|
||||
if (data.containsKey("pipeline")) {
|
||||
count = getFiltered((Map<String, Object>) data.get("pipeline"));
|
||||
|
||||
} else if (data.containsKey("events")) {
|
||||
count = getFiltered(data);
|
||||
} else {
|
||||
count = -1L;
|
||||
}
|
||||
final long cpu;
|
||||
if (count == -1L) {
|
||||
cpu = -1L;
|
||||
} else {
|
||||
cpu = ((Number) ((Map<String, Object>) ((Map<String, Object>) data.get("process"))
|
||||
.get("cpu")).get("percent")).longValue();
|
||||
}
|
||||
return new long[]{count, cpu};
|
||||
} catch (final IOException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static long getFiltered(final Map<String, Object> data) {
|
||||
return ((Number) ((Map<String, Object>) (data.get("events")))
|
||||
.get("filtered")).longValue();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import joptsimple.OptionException;
|
||||
import joptsimple.OptionParser;
|
||||
import joptsimple.OptionSet;
|
||||
import joptsimple.OptionSpec;
|
||||
import joptsimple.OptionSpecBuilder;
|
||||
import org.logstash.benchmark.cli.cases.ApacheLogsComplex;
|
||||
import org.logstash.benchmark.cli.cases.Case;
|
||||
import org.logstash.benchmark.cli.cases.GeneratorToStdout;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.logstash.benchmark.cli.ui.LsVersionType;
|
||||
import org.logstash.benchmark.cli.ui.UserInput;
|
||||
import org.logstash.benchmark.cli.ui.UserOutput;
|
||||
import org.logstash.benchmark.cli.util.LsBenchLsSetup;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
|
||||
/**
|
||||
* Benchmark CLI Main Entry Point.
|
||||
*/
|
||||
public final class Main {
|
||||
|
||||
/**
|
||||
* Ctor.
|
||||
*/
|
||||
private Main() {
|
||||
// Utility Class.
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI Entrypoint.
|
||||
* @param args Cli Args
|
||||
*/
|
||||
public static void main(final String... args) throws Exception {
|
||||
final OptionParser parser = new OptionParser();
|
||||
final OptionSpecBuilder gitbuilder =
|
||||
parser.accepts(UserInput.GIT_VERSION_PARAM, UserInput.GIT_VERSION_HELP);
|
||||
final OptionSpecBuilder localbuilder =
|
||||
parser.accepts(UserInput.LOCAL_VERSION_PARAM, UserInput.LOCAL_VERSION_HELP);
|
||||
final OptionSpecBuilder distributionbuilder =
|
||||
parser.accepts(
|
||||
UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.DISTRIBUTION_VERSION_HELP
|
||||
);
|
||||
final OptionSpec<String> git = gitbuilder.requiredUnless(
|
||||
UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.LOCAL_VERSION_PARAM
|
||||
).availableUnless(UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.LOCAL_VERSION_PARAM)
|
||||
.withRequiredArg().ofType(String.class).forHelp();
|
||||
final OptionSpec<String> distribution = distributionbuilder
|
||||
.requiredUnless(UserInput.LOCAL_VERSION_PARAM, UserInput.GIT_VERSION_PARAM)
|
||||
.availableUnless(
|
||||
UserInput.LOCAL_VERSION_PARAM, UserInput.GIT_VERSION_PARAM
|
||||
).withRequiredArg().ofType(String.class).forHelp();
|
||||
final OptionSpec<String> local = localbuilder.requiredUnless(
|
||||
UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.GIT_VERSION_PARAM)
|
||||
.availableUnless(UserInput.DISTRIBUTION_VERSION_PARAM, UserInput.GIT_VERSION_PARAM)
|
||||
.withRequiredArg().ofType(String.class).forHelp();
|
||||
final OptionSpec<String> testcase = parser.accepts(
|
||||
UserInput.TEST_CASE_PARAM, UserInput.TEST_CASE_HELP
|
||||
).withRequiredArg().ofType(String.class).defaultsTo(GeneratorToStdout.IDENTIFIER).forHelp();
|
||||
final OptionSpec<File> pwd = parser.accepts(
|
||||
UserInput.WORKING_DIRECTORY_PARAM, UserInput.WORKING_DIRECTORY_HELP
|
||||
).withRequiredArg().ofType(File.class).defaultsTo(UserInput.WORKING_DIRECTORY_DEFAULT)
|
||||
.forHelp();
|
||||
final OptionSet options;
|
||||
try {
|
||||
options = parser.parse(args);
|
||||
} catch (final OptionException ex) {
|
||||
parser.printHelpOn(System.out);
|
||||
throw ex;
|
||||
}
|
||||
final LsVersionType type;
|
||||
final String version;
|
||||
if (options.has(distribution)) {
|
||||
type = LsVersionType.DISTRIBUTION;
|
||||
version = options.valueOf(distribution);
|
||||
} else if (options.has(git)) {
|
||||
type = LsVersionType.GIT;
|
||||
version = options.valueOf(git);
|
||||
} else {
|
||||
type = LsVersionType.LOCAL;
|
||||
version = options.valueOf(local);
|
||||
}
|
||||
execute(
|
||||
new UserOutput(System.out), loadSettings(), options.valueOf(testcase),
|
||||
options.valueOf(pwd).toPath(), version, type
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Programmatic Entrypoint.
|
||||
* @param output Output Printer
|
||||
* @param settings Properties
|
||||
* @param test String identifier of the testcase to run
|
||||
* @param cwd Working Directory to run in and write cache files to
|
||||
* @param version Version of Logstash to benchmark
|
||||
* @param type Type of Logstash version to benchmark
|
||||
* @throws Exception On Failure
|
||||
*/
|
||||
public static void execute(final UserOutput output, final Properties settings,
|
||||
final String test, final Path cwd, final String version, final LsVersionType type)
|
||||
throws Exception {
|
||||
output.printBanner();
|
||||
output.printLine();
|
||||
output.green(String.format("Benchmarking Version: %s", version));
|
||||
output.green(String.format("Running Test Case: %s", test));
|
||||
output.printLine();
|
||||
Files.createDirectories(cwd);
|
||||
final LogstashInstallation logstash;
|
||||
if (type == LsVersionType.GIT) {
|
||||
logstash = LsBenchLsSetup.logstashFromGit(
|
||||
cwd.toAbsolutePath().toString(), version, JRubyInstallation.bootstrapJruby(cwd)
|
||||
);
|
||||
} else {
|
||||
logstash = LsBenchLsSetup.setupLS(cwd.toAbsolutePath().toString(), version, type);
|
||||
}
|
||||
final Case testcase;
|
||||
if (GeneratorToStdout.IDENTIFIER.equalsIgnoreCase(test)) {
|
||||
testcase = new GeneratorToStdout(logstash);
|
||||
} else if (ApacheLogsComplex.IDENTIFIER.equalsIgnoreCase(test)) {
|
||||
testcase = new ApacheLogsComplex(logstash, cwd, settings);
|
||||
} else {
|
||||
throw new IllegalArgumentException(String.format("Unknown test case %s", test));
|
||||
}
|
||||
output.printStartTime();
|
||||
final long start = System.currentTimeMillis();
|
||||
final AbstractMap<LsMetricStats, ListStatistics> stats = testcase.run();
|
||||
output.green("Statistical Summary:\n");
|
||||
output.green(String.format(
|
||||
"Elapsed Time: %ds",
|
||||
TimeUnit.SECONDS.convert(
|
||||
System.currentTimeMillis() - start, TimeUnit.MILLISECONDS
|
||||
)
|
||||
));
|
||||
output.green(
|
||||
String.format("Num Events: %d", (long) stats.get(LsMetricStats.COUNT).getMax())
|
||||
);
|
||||
final ListStatistics throughput = stats.get(LsMetricStats.THROUGHPUT);
|
||||
output.green(String.format("Throughput Min: %.2f", throughput.getMin()));
|
||||
output.green(String.format("Throughput Max: %.2f", throughput.getMax()));
|
||||
output.green(String.format("Throughput Mean: %.2f", throughput.getMean()));
|
||||
output.green(String.format("Throughput StdDev: %.2f", throughput.getStandardDeviation()));
|
||||
output.green(String.format("Throughput Variance: %.2f", throughput.getVariance()));
|
||||
output.green(
|
||||
String.format(
|
||||
"Mean CPU Usage: %.2f%%", stats.get(LsMetricStats.CPU_USAGE).getMean()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private static Properties loadSettings() throws IOException {
|
||||
final Properties props = new Properties();
|
||||
try (final InputStream settings =
|
||||
Main.class.getResource("ls-benchmark.properties").openStream()) {
|
||||
props.load(settings);
|
||||
}
|
||||
return props;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package org.logstash.benchmark.cli.cases;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.EnumMap;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.logstash.benchmark.cli.LogstashInstallation;
|
||||
import org.logstash.benchmark.cli.LsMetricsMonitor;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.logstash.benchmark.cli.util.LsBenchDownloader;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
|
||||
/**
|
||||
* Test case running a set of Apache web-server logs, read from standard input, through the
|
||||
* GeoIP and UserAgent filters and printing the result to standard out using the dots codec.
|
||||
*/
|
||||
public final class ApacheLogsComplex implements Case {
|
||||
|
||||
/**
|
||||
* Identifier used by the CLI interface.
|
||||
*/
|
||||
public static final String IDENTIFIER = "apache";
|
||||
|
||||
private final LogstashInstallation logstash;
|
||||
|
||||
private final File data;
|
||||
|
||||
public ApacheLogsComplex(final LogstashInstallation logstash, final Path cwd,
|
||||
final Properties settings) throws IOException, NoSuchAlgorithmException {
|
||||
this.data = cwd.resolve("data_apache").resolve("apache_access_logs").toFile();
|
||||
ensureDatafile(data.toPath().getParent().toFile(), settings);
|
||||
this.logstash = logstash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EnumMap<LsMetricStats, ListStatistics> run() {
|
||||
final LsMetricsMonitor monitor = new LsMetricsMonitor(logstash.metrics());
|
||||
final ExecutorService exec = Executors.newSingleThreadExecutor();
|
||||
final Future<EnumMap<LsMetricStats, ListStatistics>> future = exec.submit(monitor);
|
||||
try {
|
||||
final String config;
|
||||
try (final InputStream cfg = ApacheLogsComplex.class
|
||||
.getResourceAsStream("apache.cfg")) {
|
||||
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
IOUtils.copy(cfg, baos);
|
||||
config = baos.toString();
|
||||
}
|
||||
logstash.execute(config, data);
|
||||
monitor.stop();
|
||||
return future.get(20L, TimeUnit.SECONDS);
|
||||
} catch (final IOException | InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
} finally {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private static void ensureDatafile(final File file, final Properties settings)
|
||||
throws IOException, NoSuchAlgorithmException {
|
||||
LsBenchDownloader.downloadDecompress(
|
||||
file, settings.getProperty("org.logstash.benchmark.apache.dataset.url"), false
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package org.logstash.benchmark.cli.cases;
|
||||
|
||||
import java.util.AbstractMap;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
|
||||
/**
|
||||
* Definition of an executable test case.
|
||||
*/
|
||||
public interface Case {
|
||||
|
||||
/**
|
||||
* Runs the Actual Test Case.
|
||||
* @return Map Containing Test Results
|
||||
*/
|
||||
AbstractMap<LsMetricStats, ListStatistics> run();
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package org.logstash.benchmark.cli.cases;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.EnumMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.logstash.benchmark.cli.LogstashInstallation;
|
||||
import org.logstash.benchmark.cli.LsMetricsMonitor;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
|
||||
/**
|
||||
* Runs Generator Input and Outputs to StdOut.
|
||||
*/
|
||||
public final class GeneratorToStdout implements Case {
|
||||
|
||||
/**
|
||||
* Identifier used by the CLI interface.
|
||||
*/
|
||||
public static final String IDENTIFIER = "baseline";
|
||||
|
||||
private static final String CONFIGURATION =
|
||||
"input { generator { threads => 1 count => 1000000 } } output { stdout { } }";
|
||||
|
||||
private final LogstashInstallation logstash;
|
||||
|
||||
public GeneratorToStdout(final LogstashInstallation logstash) {
|
||||
this.logstash = logstash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EnumMap<LsMetricStats, ListStatistics> run() {
|
||||
final LsMetricsMonitor monitor = new LsMetricsMonitor(logstash.metrics());
|
||||
final ExecutorService exec = Executors.newSingleThreadExecutor();
|
||||
final Future<EnumMap<LsMetricStats, ListStatistics>> future = exec.submit(monitor);
|
||||
try {
|
||||
logstash.execute(GeneratorToStdout.CONFIGURATION);
|
||||
monitor.stop();
|
||||
return future.get(20L, TimeUnit.SECONDS);
|
||||
} catch (final IOException | InterruptedException | ExecutionException | TimeoutException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
} finally {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package org.logstash.benchmark.cli.ui;
|
||||
|
||||
/**
|
||||
* Groups of statistics this tool gathers.
|
||||
*/
|
||||
public enum LsMetricStats {
|
||||
/**
|
||||
* Statistics on the event throughput.
|
||||
*/
|
||||
THROUGHPUT,
|
||||
/**
|
||||
* Statistics on the number of events processed overall.
|
||||
*/
|
||||
COUNT,
|
||||
/**
|
||||
* Statistics on CPU usage.
|
||||
*/
|
||||
CPU_USAGE
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package org.logstash.benchmark.cli.ui;
|
||||
|
||||
/**
|
||||
* Enum of the various types of Logstash versions.
|
||||
*/
|
||||
public enum LsVersionType {
|
||||
/**
|
||||
* A local version of Logstash that is assumed to have all dependencies installed and/or build.
|
||||
*/
|
||||
LOCAL,
|
||||
|
||||
/**
|
||||
* A release version of Logstash to be downloaded from elastic.co mirrors.
|
||||
*/
|
||||
DISTRIBUTION,
|
||||
|
||||
/**
|
||||
* A version build from a given GIT tree hash/identifier.
|
||||
*/
|
||||
GIT
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
package org.logstash.benchmark.cli.ui;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* User Input Definitions and Utility Methods.
|
||||
*/
|
||||
public final class UserInput {
|
||||
|
||||
/**
|
||||
* The Default Cache/Working-Directory.
|
||||
*/
|
||||
public static final File WORKING_DIRECTORY_DEFAULT = Paths.get(
|
||||
System.getProperty("user.home"), ".logstash-benchmarks"
|
||||
).toFile();
|
||||
|
||||
/**
|
||||
* Name of the testcase to run.
|
||||
*/
|
||||
public static final String TEST_CASE_PARAM = "testcase";
|
||||
|
||||
public static final String TEST_CASE_HELP =
|
||||
"Currently available test cases are 'baseline' and 'apache'.";
|
||||
|
||||
/**
|
||||
* Version parameter to use for Logstash build downloaded from elastic.co.
|
||||
*/
|
||||
public static final String DISTRIBUTION_VERSION_PARAM = "distribution-version";
|
||||
|
||||
public static final String DISTRIBUTION_VERSION_HELP =
|
||||
"The version of a Logstash build to download from elastic.co.";
|
||||
|
||||
/**
|
||||
* Version parameter to use for Logstash build form a Git has.
|
||||
*/
|
||||
public static final String GIT_VERSION_PARAM = "git-hash";
|
||||
|
||||
public static final String GIT_VERSION_HELP = String.join(
|
||||
"\n",
|
||||
"Either a git tree (tag/branch or commit hash), optionally prefixed by a Github username,",
|
||||
"if ran against forks.",
|
||||
"E.g. 'ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7',",
|
||||
"'user-name#ab1cfe8cf7e20114df58bcc6c996abcb2b0650d7' or 'master'"
|
||||
);
|
||||
|
||||
public static final String LOCAL_VERSION_PARAM = "local-path";
|
||||
|
||||
public static final String LOCAL_VERSION_HELP =
|
||||
"Path to the root of a local Logstash distribution.\n E.g. `/opt/logstash`";
|
||||
|
||||
public static final String WORKING_DIRECTORY_PARAM = "workdir";
|
||||
|
||||
public static final String WORKING_DIRECTORY_HELP =
|
||||
"Working directory to store cached files in.";
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
private UserInput() {
|
||||
// Utility Class
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
package org.logstash.benchmark.cli.ui;
|
||||
|
||||
import java.io.PrintStream;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeFormatterBuilder;
|
||||
|
||||
public final class UserOutput {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
|
||||
.append(DateTimeFormatter.ofPattern("E")).appendLiteral(' ')
|
||||
.append(DateTimeFormatter.ofPattern("L")).appendLiteral(' ')
|
||||
.append(DateTimeFormatter.ofPattern("d")).appendLiteral(' ')
|
||||
.append(DateTimeFormatter.ISO_LOCAL_TIME).appendLiteral(' ')
|
||||
.append(DateTimeFormatter.ofPattern("yyyy")).appendLiteral(' ')
|
||||
.append(DateTimeFormatter.ofPattern("z")).toFormatter();
|
||||
|
||||
private final PrintStream target;
|
||||
|
||||
public UserOutput(final PrintStream target) {
|
||||
this.target = target;
|
||||
}
|
||||
|
||||
public void printStartTime() {
|
||||
green(
|
||||
String.format(
|
||||
"Start Time: %s", DATE_TIME_FORMATTER.format(ZonedDateTime.now().withNano(0))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public void printLine() {
|
||||
green("------------------------------------------");
|
||||
}
|
||||
|
||||
public void printBanner() {
|
||||
green(
|
||||
"██╗ ██████╗ ██████╗ ███████╗████████╗ █████╗ ███████╗██╗ ██╗ \n" +
|
||||
"██║ ██╔═══██╗██╔════╝ ██╔════╝╚══██╔══╝██╔══██╗██╔════╝██║ ██║ \n" +
|
||||
"██║ ██║ ██║██║ ███╗███████╗ ██║ ███████║███████╗███████║ \n" +
|
||||
"██║ ██║ ██║██║ ██║╚════██║ ██║ ██╔══██║╚════██║██╔══██║ \n" +
|
||||
"███████╗╚██████╔╝╚██████╔╝███████║ ██║ ██║ ██║███████║██║ ██║ \n" +
|
||||
"╚══════╝ ╚═════╝ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ \n" +
|
||||
" \n" +
|
||||
"██████╗ ███████╗███╗ ██╗ ██████╗██╗ ██╗███╗ ███╗ █████╗ ██████╗ ██╗ ██╗\n" +
|
||||
"██╔══██╗██╔════╝████╗ ██║██╔════╝██║ ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝\n" +
|
||||
"██████╔╝█████╗ ██╔██╗ ██║██║ ███████║██╔████╔██║███████║██████╔╝█████╔╝ \n" +
|
||||
"██╔══██╗██╔══╝ ██║╚██╗██║██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══██╗██╔═██╗ \n" +
|
||||
"██████╔╝███████╗██║ ╚████║╚██████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║██║ ██╗\n" +
|
||||
"╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝\n" +
|
||||
" ");
|
||||
}
|
||||
|
||||
public void green(final String line) {
|
||||
target.println(colorize(line, "\u001B[32m"));
|
||||
}
|
||||
|
||||
private static String colorize(final String line, final String prefix) {
|
||||
final String reset = "\u001B[0m";
|
||||
return new StringBuilder(line.length() + 2 * reset.length())
|
||||
.append(prefix).append(line).append(reset).toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package org.logstash.benchmark.cli.util;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.UUID;
|
||||
import org.apache.commons.compress.archivers.ArchiveEntry;
|
||||
import org.apache.commons.compress.archivers.ArchiveInputStream;
|
||||
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
|
||||
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
|
||||
/**
|
||||
* Utility class for decompressing archives.
|
||||
*/
|
||||
final class LsBenchCompressUtil {
|
||||
|
||||
private LsBenchCompressUtil() {
|
||||
// Utility Class
|
||||
}
|
||||
|
||||
public static void unzipDir(final String zipFile, final File folder) throws IOException {
|
||||
if (!folder.exists() && !folder.mkdir()) {
|
||||
throw new IllegalStateException("unzip failed");
|
||||
}
|
||||
try (ArchiveInputStream zis = new ZipArchiveInputStream(new FileInputStream(zipFile))) {
|
||||
unpackDir(folder, zis);
|
||||
}
|
||||
}
|
||||
|
||||
public static void gunzipDir(final File gzfile, final File file) throws IOException {
|
||||
final File ball =
|
||||
file.toPath().getParent().resolve(String.valueOf(UUID.randomUUID())).toFile();
|
||||
gunzipFile(gzfile, ball);
|
||||
try (final TarArchiveInputStream tar = new TarArchiveInputStream(
|
||||
new FileInputStream(ball))) {
|
||||
unpackDir(file, tar);
|
||||
}
|
||||
LsBenchFileUtil.ensureDeleted(ball);
|
||||
}
|
||||
|
||||
private static void unpackDir(final File destination, final ArchiveInputStream archive)
|
||||
throws IOException {
|
||||
ArchiveEntry entry = archive.getNextEntry();
|
||||
while (entry != null) {
|
||||
final File newFile =
|
||||
Paths.get(destination.getAbsolutePath(), entry.getName()).toFile();
|
||||
if (!newFile.getParentFile().exists() && !newFile.getParentFile().mkdirs()) {
|
||||
throw new IllegalStateException("unzip failed");
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
if (!newFile.exists() && !newFile.mkdir()) {
|
||||
throw new IllegalStateException("unzip failed");
|
||||
}
|
||||
} else {
|
||||
try (final FileOutputStream fos = new FileOutputStream(newFile)) {
|
||||
IOUtils.copy(archive, fos);
|
||||
}
|
||||
}
|
||||
entry = archive.getNextEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private static void gunzipFile(final File gzfile, final File file) throws IOException {
|
||||
try (
|
||||
final FileOutputStream uncompressed = new FileOutputStream(file);
|
||||
final InputStream archive = new GzipCompressorInputStream(
|
||||
new BufferedInputStream(new FileInputStream(gzfile)))) {
|
||||
IOUtils.copy(archive, uncompressed);
|
||||
}
|
||||
LsBenchFileUtil.ensureDeleted(gzfile);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package org.logstash.benchmark.cli.util;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import org.apache.commons.compress.compressors.gzip.GzipUtils;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
|
||||
public final class LsBenchDownloader {
|
||||
|
||||
public static void downloadDecompress(final File file, final String url, final boolean force)
|
||||
throws IOException, NoSuchAlgorithmException {
|
||||
if (force && file.exists()) {
|
||||
LsBenchFileUtil.ensureDeleted(file);
|
||||
}
|
||||
if (!file.exists()) {
|
||||
final File temp = file.getParentFile().toPath().resolve(
|
||||
String.format("%s.download", file.getName())).toFile();
|
||||
LsBenchFileUtil.ensureDeleted(temp);
|
||||
try (final OutputStream target = new BufferedOutputStream(new FileOutputStream(temp))) {
|
||||
try (final CloseableHttpClient client = HttpClientBuilder.create().setSSLContext(
|
||||
SSLContext.getDefault()).build()) {
|
||||
try (final CloseableHttpResponse response = client
|
||||
.execute(new HttpGet(url))) {
|
||||
response.getEntity().writeTo(target);
|
||||
}
|
||||
target.flush();
|
||||
}
|
||||
}
|
||||
if (GzipUtils.isCompressedFilename(url)) {
|
||||
LsBenchCompressUtil.gunzipDir(temp, file);
|
||||
}
|
||||
if (url.endsWith(".zip")) {
|
||||
LsBenchCompressUtil.unzipDir(temp.getAbsolutePath(), file);
|
||||
}
|
||||
LsBenchFileUtil.ensureDeleted(temp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package org.logstash.benchmark.cli.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
/**
|
||||
* Utility class for file handling.
|
||||
*/
|
||||
public final class LsBenchFileUtil {
|
||||
|
||||
private LsBenchFileUtil() {
|
||||
//Utility Class
|
||||
}
|
||||
|
||||
public static void ensureDeleted(final File file) throws IOException {
|
||||
if (file.exists()) {
|
||||
if (file.isDirectory()) {
|
||||
FileUtils.deleteDirectory(file);
|
||||
} else {
|
||||
if (!file.delete()) {
|
||||
throw new IllegalStateException(
|
||||
String.format("Failed to delete %s", file.getAbsolutePath())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void ensureExecutable(final File file) {
|
||||
if (!file.canExecute() && !file.setExecutable(true)) {
|
||||
throw new IllegalStateException(
|
||||
String.format("Failed to set %s executable", file.getAbsolutePath()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package org.logstash.benchmark.cli.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Paths;
|
||||
import org.logstash.benchmark.cli.JRubyInstallation;
|
||||
import org.logstash.benchmark.cli.LogstashInstallation;
|
||||
import org.logstash.benchmark.cli.ui.LsVersionType;
|
||||
|
||||
public final class LsBenchLsSetup {
|
||||
|
||||
private LsBenchLsSetup() {
|
||||
}
|
||||
|
||||
public static LogstashInstallation logstashFromGit(final String pwd, final String version,
|
||||
final JRubyInstallation jruby) {
|
||||
final File lsdir = Paths.get(pwd, "logstash").toFile();
|
||||
final LogstashInstallation logstash;
|
||||
if (version.contains("#")) {
|
||||
final String[] parts = version.split("#");
|
||||
logstash = new LogstashInstallation.FromGithub(lsdir, parts[0], parts[1], jruby);
|
||||
} else {
|
||||
logstash = new LogstashInstallation.FromGithub(lsdir, version, jruby);
|
||||
}
|
||||
return logstash;
|
||||
}
|
||||
|
||||
public static LogstashInstallation setupLS(final String pwd, final String version,
|
||||
final LsVersionType type) {
|
||||
final LogstashInstallation logstash;
|
||||
if (type == LsVersionType.LOCAL) {
|
||||
logstash = new LogstashInstallation.FromLocalPath(version);
|
||||
} else {
|
||||
logstash = new LogstashInstallation.FromRelease(
|
||||
Paths.get(pwd, String.format("ls-release-%s", version)).toFile(), version
|
||||
);
|
||||
}
|
||||
return logstash;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
input {
|
||||
stdin { }
|
||||
}
|
||||
|
||||
filter {
|
||||
grok {
|
||||
match => {
|
||||
"message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
|
||||
}
|
||||
}
|
||||
|
||||
date {
|
||||
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
|
||||
locale => en
|
||||
}
|
||||
|
||||
geoip {
|
||||
source => "clientip"
|
||||
}
|
||||
|
||||
useragent {
|
||||
source => "agent"
|
||||
target => "useragent"
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout { codec => dots }
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
org.logstash.benchmark.apache.dataset.url=https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_access_logs.tar.gz
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import com.github.tomakehurst.wiremock.client.WireMock;
|
||||
import com.github.tomakehurst.wiremock.junit.WireMockRule;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.EnumMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.hamcrest.MatcherAssert;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.logstash.benchmark.cli.ui.LsMetricStats;
|
||||
import org.openjdk.jmh.util.ListStatistics;
|
||||
import org.openjdk.jmh.util.Statistics;
|
||||
|
||||
/**
|
||||
* Tests for {@link LsMetricsMonitor}.
|
||||
*/
|
||||
public final class LsMetricsMonitorTest {
|
||||
|
||||
@Rule
|
||||
public WireMockRule http = new WireMockRule(0);
|
||||
|
||||
@Test
|
||||
public void parsesFilteredCount() throws Exception {
|
||||
final String path = "/_node/stats/?pretty";
|
||||
http.stubFor(WireMock.get(WireMock.urlEqualTo(path)).willReturn(WireMock.okJson(
|
||||
new String(
|
||||
Files.readAllBytes(
|
||||
Paths.get(LsMetricsMonitorTest.class.getResource("metrics.json").getPath()
|
||||
))
|
||||
, StandardCharsets.UTF_8)
|
||||
)));
|
||||
final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try {
|
||||
final LsMetricsMonitor monitor =
|
||||
new LsMetricsMonitor(String.format("http://127.0.0.1:%d/%s", http.port(), path));
|
||||
final Future<EnumMap<LsMetricStats, ListStatistics>> future = executor.submit(monitor);
|
||||
TimeUnit.SECONDS.sleep(5L);
|
||||
monitor.stop();
|
||||
final Statistics stats = future.get().get(LsMetricStats.THROUGHPUT);
|
||||
MatcherAssert.assertThat(stats.getMax(), CoreMatchers.is(21052.0D));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parsesCpuUsage() throws Exception {
|
||||
final String path = "/_node/stats/?pretty";
|
||||
http.stubFor(WireMock.get(WireMock.urlEqualTo(path)).willReturn(WireMock.okJson(
|
||||
new String(
|
||||
Files.readAllBytes(
|
||||
Paths.get(LsMetricsMonitorTest.class.getResource("metrics.json").getPath()
|
||||
))
|
||||
, StandardCharsets.UTF_8)
|
||||
)));
|
||||
final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try {
|
||||
final LsMetricsMonitor monitor =
|
||||
new LsMetricsMonitor(String.format("http://127.0.0.1:%d/%s", http.port(), path));
|
||||
final Future<EnumMap<LsMetricStats, ListStatistics>> future = executor.submit(monitor);
|
||||
TimeUnit.SECONDS.sleep(5L);
|
||||
monitor.stop();
|
||||
final Statistics stats = future.get().get(LsMetricStats.CPU_USAGE);
|
||||
MatcherAssert.assertThat(stats.getMax(), CoreMatchers.is(63.0D));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package org.logstash.benchmark.cli;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Path;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.logstash.benchmark.cli.ui.UserInput;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.is;
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
|
||||
/**
|
||||
* Tests for {@link Main}.
|
||||
* todo: These tests are ignored for now, their runtime is simply unreasonable for any CI scenario.
|
||||
* We will have to find a reasonable trade-off here for making sure the benchmark code is functional
|
||||
* without increasing test runtime by many minutes.
|
||||
*/
|
||||
public final class MainTest {
|
||||
|
||||
@Rule
|
||||
public final TemporaryFolder temp = new TemporaryFolder();
|
||||
|
||||
@Test
|
||||
public void downloadsDependenciesForGithub() throws Exception {
|
||||
final File pwd = temp.newFolder();
|
||||
Main.main(String.format("--workdir=%s", pwd.getAbsolutePath()));
|
||||
final Path logstash = pwd.toPath().resolve("logstash").resolve("logstash-master");
|
||||
assertThat(logstash.toFile().exists(), is(true));
|
||||
final File jruby = pwd.toPath().resolve("jruby").toFile();
|
||||
assertThat(jruby.exists(), is(true));
|
||||
assertThat(jruby.isDirectory(), is(true));
|
||||
assertThat(logstash.resolve("Gemfile").toFile().exists(), is(true));
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception On Failure
|
||||
* @todo cleanup path here, works though if you plug in a correct path
|
||||
*/
|
||||
@Test
|
||||
public void runsAgainstLocal() throws Exception {
|
||||
final File pwd = temp.newFolder();
|
||||
Main.main(String.format(
|
||||
"--version=local:%s",
|
||||
System.getProperty("logstash.benchmark.test.local.path")
|
||||
), String.format("--workdir=%s", pwd.getAbsolutePath()));
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception On Failure
|
||||
*/
|
||||
@Test
|
||||
public void runsAgainstRelease() throws Exception {
|
||||
final File pwd = temp.newFolder();
|
||||
Main.main(
|
||||
String.format("--%s=5.5.0", UserInput.DISTRIBUTION_VERSION_PARAM),
|
||||
String.format("--workdir=%s", pwd.getAbsolutePath())
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception On Failure
|
||||
*/
|
||||
@Test
|
||||
public void runsApacheAgainstRelease() throws Exception {
|
||||
final File pwd = temp.newFolder();
|
||||
Main.main(
|
||||
String.format("--%s=5.5.0", UserInput.DISTRIBUTION_VERSION_PARAM),
|
||||
String.format("--%s=apache", UserInput.TEST_CASE_PARAM),
|
||||
String.format("--workdir=%s", pwd.getAbsolutePath())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
{
|
||||
"host": "localhost",
|
||||
"version": "6.0.0-alpha3",
|
||||
"http_address": "127.0.0.1:9600",
|
||||
"id": "8bbabc13-ea58-4dcd-b94e-90ae5f692c17",
|
||||
"name": "localhost",
|
||||
"jvm": {
|
||||
"threads": {
|
||||
"count": 28,
|
||||
"peak_count": 28
|
||||
},
|
||||
"mem": {
|
||||
"heap_used_percent": 16,
|
||||
"heap_committed_in_bytes": 259522560,
|
||||
"heap_max_in_bytes": 1037959168,
|
||||
"heap_used_in_bytes": 168360000,
|
||||
"non_heap_used_in_bytes": 113241032,
|
||||
"non_heap_committed_in_bytes": 124989440,
|
||||
"pools": {
|
||||
"survivor": {
|
||||
"peak_used_in_bytes": 8912896,
|
||||
"used_in_bytes": 6872400,
|
||||
"peak_max_in_bytes": 35782656,
|
||||
"max_in_bytes": 35782656,
|
||||
"committed_in_bytes": 8912896
|
||||
},
|
||||
"old": {
|
||||
"peak_used_in_bytes": 141395984,
|
||||
"used_in_bytes": 119128832,
|
||||
"peak_max_in_bytes": 715849728,
|
||||
"max_in_bytes": 715849728,
|
||||
"committed_in_bytes": 178978816
|
||||
},
|
||||
"young": {
|
||||
"peak_used_in_bytes": 71630848,
|
||||
"used_in_bytes": 42358768,
|
||||
"peak_max_in_bytes": 286326784,
|
||||
"max_in_bytes": 286326784,
|
||||
"committed_in_bytes": 71630848
|
||||
}
|
||||
}
|
||||
},
|
||||
"gc": {
|
||||
"collectors": {
|
||||
"old": {
|
||||
"collection_time_in_millis": 89,
|
||||
"collection_count": 3
|
||||
},
|
||||
"young": {
|
||||
"collection_time_in_millis": 516,
|
||||
"collection_count": 36
|
||||
}
|
||||
}
|
||||
},
|
||||
"uptime_in_millis": 15055
|
||||
},
|
||||
"process": {
|
||||
"open_file_descriptors": 63,
|
||||
"peak_open_file_descriptors": 63,
|
||||
"max_file_descriptors": 10240,
|
||||
"mem": {
|
||||
"total_virtual_in_bytes": 5335916544
|
||||
},
|
||||
"cpu": {
|
||||
"total_in_millis": 67919,
|
||||
"percent": 63,
|
||||
"load_average": {
|
||||
"1m": 2.6826171875
|
||||
}
|
||||
}
|
||||
},
|
||||
"events": {
|
||||
"in": 23101,
|
||||
"filtered": 21052,
|
||||
"out": 21052,
|
||||
"duration_in_millis": 8939,
|
||||
"queue_push_duration_in_millis": 3978
|
||||
},
|
||||
"pipelines": {
|
||||
"main": {
|
||||
"events": {
|
||||
"duration_in_millis": 9250,
|
||||
"in": 24125,
|
||||
"filtered": 22076,
|
||||
"out": 22076,
|
||||
"queue_push_duration_in_millis": 4236
|
||||
},
|
||||
"plugins": {
|
||||
"inputs": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-1",
|
||||
"events": {
|
||||
"out": 24125,
|
||||
"queue_push_duration_in_millis": 4236
|
||||
},
|
||||
"name": "stdin"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-4",
|
||||
"events": {
|
||||
"duration_in_millis": 374,
|
||||
"in": 23045,
|
||||
"out": 23044
|
||||
},
|
||||
"name": "geoip"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-3",
|
||||
"events": {
|
||||
"duration_in_millis": 24,
|
||||
"in": 23045,
|
||||
"out": 23045
|
||||
},
|
||||
"matches": 23045,
|
||||
"name": "date"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-5",
|
||||
"events": {
|
||||
"duration_in_millis": 1373,
|
||||
"in": 23045,
|
||||
"out": 23045
|
||||
},
|
||||
"name": "useragent"
|
||||
},
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-2",
|
||||
"events": {
|
||||
"duration_in_millis": 295,
|
||||
"in": 23047,
|
||||
"out": 23045
|
||||
},
|
||||
"matches": 23045,
|
||||
"patterns_per_field": {
|
||||
"message": 1
|
||||
},
|
||||
"name": "grok"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"id": "1db6e3e8163d4cf302e5b5ee12f6fc3dcfe783ba-6",
|
||||
"events": {
|
||||
"duration_in_millis": 89,
|
||||
"in": 22076,
|
||||
"out": 22076
|
||||
},
|
||||
"name": "stdout"
|
||||
}
|
||||
]
|
||||
},
|
||||
"reloads": {
|
||||
"last_error": null,
|
||||
"successes": 0,
|
||||
"last_success_timestamp": null,
|
||||
"last_failure_timestamp": null,
|
||||
"failures": 0
|
||||
},
|
||||
"queue": {
|
||||
"type": "memory"
|
||||
}
|
||||
}
|
||||
},
|
||||
"reloads": {
|
||||
"successes": 0,
|
||||
"failures": 0
|
||||
},
|
||||
"os": {}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue