diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 000000000..acc19206d --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,59 @@ +# integration tests + +## performance tests + +### run.rb + +executes a single test. + +a test can be execute for a specific number of events of for a specific duration. + +- logstash config are in `test/integration/config` +- sample input files are in `test/integration/input` + +#### by number of events + +``` +ruby test/integration/run.rb --events [number of events] --config [logstash config file] --input [sample input events file] +``` + +the sample input events file will be sent to logstash stdin repetedly until the required number of events is reached + +#### by target duration + +``` +ruby test/integration/run.rb --time [number of seconds] --config [logstash config file] --input [sample input events file] +``` + +the sample input events file will be sent to logstash stdin repetedly until the test elaspsed time reached the target time + + +### suite.rb + +- suites are in `test/integration/suite` + +``` +ruby test/integration/suite.rb [suite file] +``` + +a suite file defines a series of tests to run. + +#### suite file format + +```ruby +# each test can be executed by either target duration using :time => N secs +# or by number of events with :events => N +# +#[ +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 30}, +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :events => 50000}, +#] +# +[ + {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 60}, + {:name => "simple line out", :config => "config/simple.conf", :input => "input/simple_10.txt", :time => 60}, + {:name => "json codec", :config => "config/json_inout_codec.conf", :input => "input/json_medium.txt", :time => 60}, + {:name => "json filter", :config => "config/json_inout_filter.conf", :input => "input/json_medium.txt", :time => 60}, + {:name => "complex syslog", :config => "config/complex_syslog.conf", :input => "input/syslog_acl_10.txt", :time => 60}, +] +``` \ No newline at end of file diff --git a/test/integration/config/complex_syslog.conf b/test/integration/config/complex_syslog.conf new file mode 100644 index 000000000..c7db7bf51 --- /dev/null +++ b/test/integration/config/complex_syslog.conf @@ -0,0 +1,46 @@ +input { + stdin { + type => syslog + } +} + +filter { + if [type] == "syslog" { + grok { + match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{syslog_hostname}" ] + } + syslog_pri { } + date { + match => ["syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + + if [syslog_timestamp] { + mutate { + add_field => [ "[times][created_at]", "%{syslog_timestamp}"] + add_field => [ "[times][received_at]", "%{@timestamp}"] + } + } + + mutate { + add_field => [ "[hosts][source]", "%{received_from}"] + add_field => [ "[level][facility]", "%{syslog_facility}"] + add_field => [ "[level][severity]", "%{syslog_severity}"] + } + + if !("_grokparsefailure" in [tags]) { + mutate { + replace => [ "@source_host", "%{syslog_hostname}" ] + replace => [ "@message", "%{syslog_message}" ] + } + } + mutate { + remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ] + } + } +} + +output { + stdout { codec => json_lines } +} diff --git a/test/integration/config/json_inout_codec.conf b/test/integration/config/json_inout_codec.conf new file mode 100644 index 000000000..d8b79e219 --- /dev/null +++ b/test/integration/config/json_inout_codec.conf @@ -0,0 +1,11 @@ +input { + stdin { codec => "json_lines" } +} + +filter { + noop {} +} + +output { + stdout { codec => json_lines } +} diff --git a/test/integration/config/json_inout_filter.conf b/test/integration/config/json_inout_filter.conf new file mode 100644 index 000000000..afad781c5 --- /dev/null +++ b/test/integration/config/json_inout_filter.conf @@ -0,0 +1,11 @@ +input { + stdin {} +} + +filter { + json { source => "message" } +} + +output { + stdout { codec => json_lines } +} diff --git a/test/integration/config/simple.conf b/test/integration/config/simple.conf new file mode 100644 index 000000000..a3967bd12 --- /dev/null +++ b/test/integration/config/simple.conf @@ -0,0 +1,11 @@ +input { + stdin {} +} + +filter { + noop {} +} + +output { + stdout { codec => line } +} diff --git a/test/integration/config/simple_json_out.conf b/test/integration/config/simple_json_out.conf new file mode 100644 index 000000000..2c98527c3 --- /dev/null +++ b/test/integration/config/simple_json_out.conf @@ -0,0 +1,11 @@ +input { + stdin {} +} + +filter { + noop {} +} + +output { + stdout { codec => json_lines } +} diff --git a/test/integration/input/json_medium.txt b/test/integration/input/json_medium.txt new file mode 100644 index 000000000..96b808f22 --- /dev/null +++ b/test/integration/input/json_medium.txt @@ -0,0 +1,10 @@ +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } +{"_scroll_id":"xxx", "took":5, "timed_out":false, "_shards":{"total":15,"successful":15,"failed":0}, "hits":{"total":1000050, "max_score":1.0, "hits":[{"_index":"logstash2", "_type":"logs", "_id":"AmaqL7VuSWKF-F6N_Gz72g", "_score":1.0, "_source" : {"message":"foobar", "@version":"1", "@timestamp":"2014-05-19T21:08:39.000Z", "host":"colin-mbp13r"} } ] } } diff --git a/test/integration/input/simple_10.txt b/test/integration/input/simple_10.txt new file mode 100644 index 000000000..8a9b58e04 --- /dev/null +++ b/test/integration/input/simple_10.txt @@ -0,0 +1,10 @@ +test 01 +test 02 +test 03 +test 04 +test 05 +test 06 +test 07 +test 08 +test 09 +test 10 \ No newline at end of file diff --git a/test/integration/input/syslog_acl_10.txt b/test/integration/input/syslog_acl_10.txt new file mode 100644 index 000000000..d277d8566 --- /dev/null +++ b/test/integration/input/syslog_acl_10.txt @@ -0,0 +1,10 @@ +<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 6 15:20:25 2.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.4/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 1 15:21:25 3.2.3.4 %ASA-4-106023: Allow tcp src DRAC:10.1.2.5/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 30 15:22:25 4.2.3.4 %ASA-4-106023: Allow tcp src DRAC:10.1.2.6/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 6 15:20:25 2.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.4/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 1 15:21:25 3.2.3.4 %ASA-4-106023: Allow tcp src DRAC:10.1.2.5/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 30 15:22:25 4.2.3.4 %ASA-4-106023: Allow tcp src DRAC:10.1.2.6/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] +<164>Oct 6 15:20:25 2.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.4/43434 dst outside:192.168.0.1/53 by access-group "acl_drac" [0x0, 0x0] diff --git a/test/integration/run.rb b/test/integration/run.rb new file mode 100644 index 000000000..ad94d5fba --- /dev/null +++ b/test/integration/run.rb @@ -0,0 +1,124 @@ +# encoding: utf-8 + +require "benchmark" + +INITIAL_MESSAGE = ">>> lorem ipsum start".freeze +LAST_MESSAGE = ">>> lorem ipsum stop".freeze +LOGSTASH_BIN = File.join(File.expand_path("../../../bin/", __FILE__), "logstash") + +Thread.abort_on_exception = true + +def feed_input_events(io, events_count, lines, last_message) + loop_count = (events_count / lines.size).ceil # how many time we send the input file over + + (1..loop_count).each{lines.each {|line| io.puts(line)}} + + io.puts(last_message) + io.flush + + loop_count * lines.size +end + +def feed_input_interval(io, seconds, lines, last_message) + loop_count = (2000 / lines.size).ceil # check time every ~2000(ceil) input lines + lines_per_iteration = loop_count * lines.size + start_time = Time.now + count = 0 + + while true + (1..loop_count).each{lines.each {|line| io.puts(line)}} + count += lines_per_iteration + break if (Time.now - start_time) >= seconds + end + + io.puts(last_message) + io.flush + + count +end + +def output_reader(io, regex) + Thread.new(io, regex) do |io, regex| + expect_output(io, regex) + end +end + +def read_input_file(file_path) + IO.readlines(file_path).map(&:chomp) +end + +def expect_output(io, regex) + io.each_line do |line| + puts("received: #{line}") if @debug + break if line =~ regex + end +end + +# +## script main + +# standalone quick & dirty options parsing +args = ARGV.dup +if args.size != 6 + $stderr.puts("usage: ruby run.rb --events [events count] --config [config file] --input [input file]") + $stderr.puts(" ruby run.rb --time [seconds] --config [config file] --input [input file]") + exit(1) +end + +options = {} +while !args.empty? + config = args.shift.to_s.strip + option = args.shift.to_s.strip + raise(IllegalArgumentException, "invalid option for #{config}") if option.empty? + case config + when "--events" + options[:events] = option + when "--time" + options[:time] = option + when "--config" + options[:config] = option + when "--input" + options[:input] = option + else + raise(IllegalArgumentException, "invalid config #{config}") + end +end + +@debug = !!ENV["DEBUG"] + +required_events_count = options[:events].to_i # total number of events to feed, independant of input file size +required_run_time = options[:time].to_i +input_lines = read_input_file(options[:input]) + +puts("will run with config file=#{options[:config]}, input file=#{options[:input]}") if @debug + +command = [LOGSTASH_BIN, "-f", options[:config], "2>&1"] +puts("launching #{command.join(" ")}") if @debug + +real_events_count = 0 + +IO.popen(command.join(" "), "r+") do |io| + puts("sending initial event") if @debug + io.puts(INITIAL_MESSAGE) + io.flush + + puts("waiting for initial event") if @debug + expect_output(io, /#{INITIAL_MESSAGE}/) + + puts("starting output reader thread") if @debug + @reader = output_reader(io, /#{LAST_MESSAGE}/) + puts("starting feeding input") if @debug + + elaspsed = Benchmark.realtime do + real_events_count = if required_events_count > 0 + feed_input_events(io, [required_events_count, input_lines.size].max, input_lines, LAST_MESSAGE) + else + feed_input_interval(io, required_run_time, input_lines, LAST_MESSAGE) + end + + puts("waiting for output reader to complete") if @debug + @reader.join + end + + puts("elaspsed=#{"%.2f" % elaspsed}s, events=#{real_events_count}, tps=#{"%.0f" % (real_events_count / elaspsed)}") +end diff --git a/test/integration/suite.rb b/test/integration/suite.rb new file mode 100644 index 000000000..b0bcf7787 --- /dev/null +++ b/test/integration/suite.rb @@ -0,0 +1,25 @@ +# encoding: utf-8 + +RUNNER = File.join(File.expand_path(File.dirname(__FILE__)), "run.rb") +BASE_DIR = File.expand_path(File.dirname(__FILE__)) + +# +## script main + +if ARGV.size != 1 + $stderr.puts("usage: ruby suite.rb [suite file]") + exit(1) +end + +@debug = !!ENV["DEBUG"] + +tests = eval(IO.read(ARGV[0])) + +tests.each do |test| + duration = test[:events] ? ["--events", test[:events]] : ["--time", test[:time]] + command = ["ruby", RUNNER, *duration, "--config", File.join(BASE_DIR, test[:config]), "--input", File.join(BASE_DIR, test[:input])] + IO.popen(command.join(" "), "r") do |io| + print("name=#{test[:name]}, ") + io.each_line{|line| puts(line)} + end +end diff --git a/test/integration/suite/basic_performance_long.rb b/test/integration/suite/basic_performance_long.rb new file mode 100644 index 000000000..c3218e044 --- /dev/null +++ b/test/integration/suite/basic_performance_long.rb @@ -0,0 +1,16 @@ +# format description: +# each test can be executed by either target duration using :time => N secs +# or by number of events with :events => N +# +#[ +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 30}, +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :events => 50000}, +#] +# +[ + {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 120}, + {:name => "simple line out", :config => "config/simple.conf", :input => "input/simple_10.txt", :time => 120}, + {:name => "json codec", :config => "config/json_inout_codec.conf", :input => "input/json_medium.txt", :time => 120}, + {:name => "json filter", :config => "config/json_inout_filter.conf", :input => "input/json_medium.txt", :time => 120}, + {:name => "complex syslog", :config => "config/complex_syslog.conf", :input => "input/syslog_acl_10.txt", :time => 120}, +] \ No newline at end of file diff --git a/test/integration/suite/basic_performance_quick.rb b/test/integration/suite/basic_performance_quick.rb new file mode 100644 index 000000000..120a5169f --- /dev/null +++ b/test/integration/suite/basic_performance_quick.rb @@ -0,0 +1,16 @@ +# format description: +# each test can be executed by either target duration using :time => N secs +# or by number of events with :events => N +# +#[ +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 30}, +# {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :events => 50000}, +#] +# +[ + {:name => "simple json out", :config => "config/simple_json_out.conf", :input => "input/simple_10.txt", :time => 30}, + {:name => "simple line out", :config => "config/simple.conf", :input => "input/simple_10.txt", :time => 30}, + {:name => "json codec", :config => "config/json_inout_codec.conf", :input => "input/json_medium.txt", :time => 30}, + {:name => "json filter", :config => "config/json_inout_filter.conf", :input => "input/json_medium.txt", :time => 30}, + {:name => "complex syslog", :config => "config/complex_syslog.conf", :input => "input/syslog_acl_10.txt", :time => 30}, +] \ No newline at end of file