github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 06:37:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1187 from colinsurprenant/disconnected_specs

Browse source 
Disconnected specs
This commit is contained in:
Jordan Sissel 2014-04-01 16:28:31 -07:00
commit 816d9662d1
14 changed files with 183 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -43,7 +43,7 @@ default:
@echo " tarball -- builds the tarball package"
@echo " tarball-test -- runs the test suite against the tarball package"
TESTS=$(wildcard spec/**/*.rb)
TESTS=$(wildcard spec/**/*.rb spec/*.rb)
# The 'version' is generated based on the logstash version, git revision, etc.
.VERSION.mk: REVISION=$(shell git rev-parse --short HEAD | tr -d ' ')

4
lib/logstash/event.rb
View file

@ -129,12 +129,12 @@ class LogStash::Event
end # def []
public
# keep []= implementation in sync with spec/test_utils.rb monkey patch
# which redefines []= but using @accessors.strict_set
def []=(str, value)
if str == TIMESTAMP && !value.is_a?(Time)
raise TypeError, "The field '@timestamp' must be a Time, not a #{value.class} (#{value})"
end
# return LogStash::Util::FieldReference.set(str, value, @data)
@accessors.set(str, value)
end # def []=

75
lib/logstash/runner.rb
View file

@ -53,6 +53,40 @@ I18n.load_path << File.expand_path(
File.join(File.dirname(__FILE__), "../../locales/en.yml")
)
class LogStash::RSpecsRunner
def initialize(args)
@args = args.collect do |arg|
# if the arg ends in .rb or has a "/" in it, assume it's a path.
if arg =~ /\.rb$/ || arg =~ /\//
# check if it's a file, if not, try inside the jar if we are in it.
if !File.exists?(arg) && __FILE__ =~ /file:.*\.jar!\//
# Try inside the jar.
jar_root = __FILE__.gsub(/!.*/,"!")
newpath = File.join(jar_root, arg)
# Strip leading 'jar:' path (JRUBY_6970)
newpath.gsub!(/^jar:/, "")
if File.exists?(newpath)
# Add the 'spec' dir to the load path so specs can run
specpath = File.join(jar_root, "spec")
$LOAD_PATH << specpath unless $LOAD_PATH.include?(specpath)
next newpath
end
end
end
next arg
end # args.collect
end
def run
@result = RSpec::Core::Runner.run(@args)
end
def wait
return @result
end
end
class LogStash::Runner
include LogStash::Program
@ -122,48 +156,9 @@ class LogStash::Runner
"rspec" => lambda do
require "rspec/core/runner"
require "rspec"
fixedargs = args.collect do |arg|
# if the arg ends in .rb or has a "/" in it, assume it's a path.
if arg =~ /\.rb$/ || arg =~ /\//
# check if it's a file, if not, try inside the jar if we are in it.
if !File.exists?(arg) && __FILE__ =~ /file:.*\.jar!\//
# Try inside the jar.
jar_root = __FILE__.gsub(/!.*/,"!")
newpath = File.join(jar_root, arg)
# Strip leading 'jar:' path (JRUBY_6970)
newpath.gsub!(/^jar:/, "")
if File.exists?(newpath)
# Add the 'spec' dir to the load path so specs can run
specpath = File.join(jar_root, "spec")
$LOAD_PATH << specpath unless $LOAD_PATH.include?(specpath)
next newpath
end
end
end
next arg
end # args.collect
# Hack up a runner
runner = Class.new do
def initialize(args)
@args = args
end
def run
@thread = Thread.new do
@result = RSpec::Core::Runner.run(@args)
end
end
def wait
@thread.join
return @result
end
end
$LOAD_PATH << File.expand_path("#{File.dirname(__FILE__)}/../../spec")
require "test_utils"
#p :args => fixedargs
rspec = runner.new(fixedargs)
rspec = LogStash::RSpecsRunner.new(args)
rspec.run
@runners << rspec
return []

25
lib/logstash/util/accessors.rb
View file

@ -1,8 +1,8 @@
# encoding: utf-8
require "logstash/namespace"
require "logstash/util"
module LogStash::Util
# PathCache is a singleton which globally caches a parsed fields path for the path to the
@ -21,7 +21,6 @@ module LogStash::Util
end
end
# Accessors uses a lookup table to speedup access of an accessor field of the type
# "[hello][world]" to the underlying store hash into {"hello" => {"world" => "foo"}}
class Accessors
@ -41,6 +40,10 @@ module LogStash::Util
target[key] = value
end
def strict_set(accessor, value)
set(accessor, strict_value(value))
end
def del(accessor)
target, key = lookup(accessor)
target.delete(key)
@ -58,5 +61,19 @@ module LogStash::Util
[target, key]
end
end
end # module LogStash::Util::Accessors
def strict_value(value)
case value
when String
raise("expected UTF-8 encoding for value=#{value}, encoding=#{value.encoding.inspect}") unless value.encoding == Encoding::UTF_8
raise("invalid UTF-8 encoding for value=#{value}, encoding=#{value.encoding.inspect}") unless value.valid_encoding?
value
when Array
value.each{|v| strict_value(v)} # don't map, return original object
value
else
value
end
end
end # class Accessors
end # module LogStash::Util

6
spec/codecs/json.rb
View file

@ -18,7 +18,7 @@ describe LogStash::Codecs::JSON do
end
end
it "should be fast", :if => ENV["SPEEDTEST"] do
it "should be fast", :performance => true do
json = '{"message":"Hello world!","@timestamp":"2013-12-21T07:01:25.616Z","@version":"1","host":"Macintosh.local","sequence":1572456}'
iterations = 500000
count = 0
@ -34,9 +34,9 @@ describe LogStash::Codecs::JSON do
end
duration = Time.now - start
insist { count } == iterations
puts "codecs/json speed: #{iterations/duration}/sec"
puts "codecs/json rate: #{"%02.0f/sec" % (iterations / duration)}, elapsed: #{duration}s"
end
context "processing plain text" do
it "falls back to plain text" do
decoded = false

14
spec/event.rb
View file

@ -1,3 +1,5 @@
# encoding: utf-8
require "logstash/event"
require "insist"
@ -104,11 +106,13 @@ describe LogStash::Event do
end
it "should be fast?", :if => ENV["SPEEDTEST"] do
it "should be fast?", :performance => true do
count = 1000000
2.times do
start = Time.now
100000.times { subject["[j][k1]"] }
puts "Duration: #{Time.now - start}"
count.times { subject["[j][k1]"] }
duration = Time.now - start
puts "event #[] rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
end
end
end
@ -190,7 +194,7 @@ describe LogStash::Event do
end
end
it "timestamp parsing speed", :if => ENV["SPEEDTEST"] do
it "timestamp parsing speed", :performance => true do
warmup = 10000
count = 1000000
@ -207,7 +211,7 @@ describe LogStash::Event do
end
duration = Time.now - start
end
puts "event @timestamp parse rate: #{count / duration}/sec"
puts "event @timestamp parse rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
end
context "acceptable @timestamp formats" do

4
spec/filters/date_performance.rb
View file

@ -5,7 +5,7 @@ puts "Skipping date tests because this ruby is not jruby" if RUBY_ENGINE != "jru
describe LogStash::Filters::Date, :if => RUBY_ENGINE == "jruby" do
extend LogStash::RSpec
describe "speed test of date parsing", :if => ENV["SPEEDTEST"] do
describe "speed test of date parsing", :performance => true do
it "should be fast" do
event_count = 100000
min_rate = 4000
@ -24,7 +24,7 @@ describe LogStash::Filters::Date, :if => RUBY_ENGINE == "jruby" do
end
duration = Time.now - start
end
puts "date parse rate: #{event_count / duration}"
puts "filters/date parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
insist { duration } < max_duration
end
end

26
spec/filters/dns.rb
View file

@ -7,20 +7,20 @@ require "resolv"
describe LogStash::Filters::DNS do
extend LogStash::RSpec
describe "dns reverse lookup, replace" do
config <<-CONFIG
filter {
dns {
reverse => "host"
action => "replace"
}
}
CONFIG
before(:all) do
begin
Resolv.new.getaddress("elasticsearch.com")
rescue Errno::ENOENT
$stderr.puts("DNS resolver error, no network? mocking resolver")
@mock_resolv = true
end
end
address = Resolv.new.getaddress("aspmx.l.google.com")
expected = Resolv.new.getname(address)
sample("host" => address) do
insist { subject["host"] } == expected
before(:each) do
if @mock_resolv
allow_any_instance_of(Resolv).to receive(:getaddress).with("carrera.databits.net").and_return("199.192.228.250")
allow_any_instance_of(Resolv).to receive(:getaddress).with("does.not.exist").and_return(nil)
allow_any_instance_of(Resolv).to receive(:getname).with("199.192.228.250").and_return("carrera.databits.net")
end
end

10
spec/filters/grok.rb
View file

@ -423,9 +423,9 @@ describe LogStash::Filters::Grok do
end
end
describe "performance test", :if => ENV["SPEEDTEST"] do
describe "performance test", :performance => true do
event_count = 100000
min_rate = 4000
min_rate = 2000
max_duration = event_count / min_rate
input = "Nov 24 01:29:01 -0800"
@ -447,9 +447,11 @@ describe LogStash::Filters::Grok do
CONFIG
2.times do
start = Time.now
agent do
puts "grok parse rate: #{event_count / @duration}"
insist { @duration } < max_duration
duration = (Time.now - start)
puts "filters/grok parse rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
insist { duration } < max_duration
end
end
end

6
spec/filters/kv.rb
View file

@ -102,7 +102,7 @@ describe LogStash::Filters::KV do
end
describe "speed test", :if => ENV["SPEEDTEST"] do
describe "speed test", :performance => true do
count = 10000 + rand(3000)
config <<-CONFIG
input {
@ -122,8 +122,10 @@ describe LogStash::Filters::KV do
}
CONFIG
start = Time.now
agent do
p :duration => @duration, :rate => count/@duration
duration = (Time.now - start)
puts "filters/kv rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
end
end

2
spec/inputs/generator.rb
View file

@ -23,7 +23,7 @@ describe "inputs/generator", :performance => true do
insist { event["sequence"] } == i
end
duration = Time.now - start
puts "Generator Rate: #{"%02.0f/sec" % (event_count / duration)}, Elapsed: #{duration}s"
puts "inputs/generator rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s"
pipeline.shutdown
end # input
end

6
spec/speed.rb
View file

@ -1,6 +1,6 @@
require "test_utils"
describe "speed tests" do
describe "speed tests", :performance => true do
extend LogStash::RSpec
count = 1000000
@ -14,7 +14,9 @@ describe "speed tests" do
output { null { } }
CONFIG
start = Time.now
agent do
puts "Rate: #{count / @duration}"
duration = (Time.now - start)
puts "speed rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
end
end

12
spec/test_utils.rb
View file

@ -37,6 +37,18 @@ else
$logger.level = :error
end
puts("Using Accessor#strict_set for specs")
# mokey path LogStash::Event to use strict_set in tests
# ugly, I know, but this avoids adding conditionals in performance critical section
class LogStash::Event
def []=(str, value)
if str == TIMESTAMP && !value.is_a?(Time)
raise TypeError, "The field '@timestamp' must be a Time, not a #{value.class} (#{value})"
end
@accessors.strict_set(str, value)
end # def []=
end
RSpec.configure do |config|
config.filter_run_excluding :redis => true, :socket => true, :performance => true, :elasticsearch => true, :broken => true
end

78
spec/util/accessors_spec.rb
View file

@ -36,46 +36,54 @@ describe LogStash::Util::Accessors, :if => true do
insist { data }.empty?
end
it "should set value" do
it "should set string value" do
str = "simple"
data = {}
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.set(str, "things") } == "things"
insist { data } == { "simple" => "things" }
end
it "should set array value" do
str = "simple"
data = {}
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.set(str, ["foo", "bar"]) } == ["foo", "bar"]
insist { data } == { "simple" => ["foo", "bar"]}
end
end
context "using field path" do
it "should get shallow value of word key" do
it "should get shallow string value of word key" do
str = "[hello]"
data = { "hello" => "world" }
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.get(str) } == "world"
end
it "should get shallow value of key with spaces" do
it "should get shallow string value of key with spaces" do
str = "[hel lo]"
data = { "hel lo" => "world" }
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.get(str) } == "world"
end
it "should get shallow value of numeric key string" do
it "should get shallow string value of numeric key string" do
str = "[1]"
data = { "1" => "world" }
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.get(str) } == "world"
end
it "should get deep value" do
it "should get deep string value" do
str = "[hello][world]"
data = { "hello" => { "world" => "foo", "bar" => "baz" } }
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.get(str) } == data["hello"]["world"]
end
it "should get deep value" do
it "should get deep string value" do
str = "[hello][world]"
data = { "hello" => { "world" => "foo", "bar" => "baz" } }
accessors = LogStash::Util::Accessors.new(data)
@ -92,7 +100,7 @@ describe LogStash::Util::Accessors, :if => true do
insist { data["hello"] } == { "bar" => "baz" }
end
it "should set shallow value" do
it "should set shallow string value" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
@ -100,7 +108,15 @@ describe LogStash::Util::Accessors, :if => true do
insist { data } == { "hello" => "foo" }
end
it "should set deep value" do
it "should strict_set shallow string value" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.strict_set(str, "foo") } == "foo"
insist { data } == { "hello" => "foo" }
end
it "should set deep string value" do
str = "[hello][world]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
@ -108,6 +124,22 @@ describe LogStash::Util::Accessors, :if => true do
insist { data } == { "hello" => { "world" => "foo" } }
end
it "should set deep array value" do
str = "[hello][world]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.set(str, ["foo", "bar"]) } == ["foo", "bar"]
insist { data } == { "hello" => { "world" => ["foo", "bar"] } }
end
it "should strict_set deep array value" do
str = "[hello][world]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
insist { accessors.strict_set(str, ["foo", "bar"]) } == ["foo", "bar"]
insist { data } == { "hello" => { "world" => ["foo", "bar"] } }
end
it "should retrieve array item" do
data = { "hello" => { "world" => ["a", "b"], "bar" => "baz" } }
accessors = LogStash::Util::Accessors.new(data)
@ -115,4 +147,34 @@ describe LogStash::Util::Accessors, :if => true do
insist { accessors.get("[hello][world][1]") } == data["hello"]["world"][1]
end
end
context "using invalid encoding" do
it "strinct_set should raise on non UTF-8 string encoding" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
expect { accessors.strict_set(str, "foo".encode("US-ASCII")) }.to raise_error
end
it "strinct_set should raise on non UTF-8 string encoding in array" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
expect { accessors.strict_set(str, ["foo", "bar".encode("US-ASCII")]) }.to raise_error
end
it "strinct_set should raise on invalid UTF-8 string encoding" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
expect { accessors.strict_set(str, "foo \xED\xB9\x81\xC3") }.to raise_error
end
it "strinct_set should raise on invalid UTF-8 string encoding in array" do
str = "[hello]"
data = {}
accessors = LogStash::Util::Accessors.new(data)
expect { accessors.strict_set(str, ["foo", "bar \xED\xB9\x81\xC3"]) }.to raise_error
end
end
end