github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Make event specs pass under event_v1

Browse source 
This required implementing Event#append. Some semantics have changed.
"@message" and "@tags" are no longer handled specially (mainly because
they aren't described by event v1 schema). This likely has broken the
multiline filter tests (intentionally). The multiline filter will need
to have custom merging of certain fields (like joining message with
newlines instead of putting it all in an array)
This commit is contained in:
Jordan Sissel 2013-05-29 15:55:24 -07:00
parent f58216c3b0
commit e494fb9a8c
3 changed files with 123 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

77
lib/logstash/event_v1.rb
View file

@ -35,7 +35,7 @@ module LogStash::EventV1
@cancelled = false
@data = data
@data["@timestamp"] = LogStash::Time.now if !@data.include?("@timestamp")
@data["@timestamp"] = ::Time.now if !@data.include?("@timestamp")
@data["@version"] = "1" if !@data.include?("@version")
end # def initialize
@ -85,12 +85,24 @@ module LogStash::EventV1
# field-related access
public
def [](key)
# TODO(sissel): Implement
if key[0] == '['
val = @data
key.gsub(/(?<=\[).+?(?=\])/).each do |tok|
if val.is_a? Array
val = val[tok.to_i]
else
val = val[tok]
end
end
return val
else
return @data[key]
end
end # def []
public
def []=(key, value)
# TODO(sissel): Implement
@data[key] = value
end # def []=
public
@ -104,7 +116,7 @@ module LogStash::EventV1
end # def to_json
def to_hash
raise DeprecatedMethod
return @data
end # def to_hash
public
@ -120,8 +132,9 @@ module LogStash::EventV1
# Append an event to this one.
public
def append(event)
raise NotImplementedError, "LogStash::EventV1#append needs implementing"
end
# non-destructively merge that event with ourselves.
LogStash::Util.hash_merge(@data, event.to_hash)
end # append
# Remove a field. Returns the value of that field when deleted
public
@ -157,40 +170,30 @@ module LogStash::EventV1
if key == "+%s"
# Got %{+%s}, support for unix epoch time
if RUBY_ENGINE != "jruby"
# This is really slow. See LOGSTASH-217
Time.parse(self.timestamp).to_i
else
datetime = @@date_parser.parseDateTime(self.timestamp)
(datetime.getMillis / 1000).to_i
end
next @data["@timestamp"].to_i
elsif key[0,1] == "+"
# We got a %{+TIMEFORMAT} so use joda to format it.
if RUBY_ENGINE != "jruby"
# This is really slow. See LOGSTASH-217
datetime = Date.parse(self.timestamp)
format = key[1 .. -1]
datetime.strftime(format)
else
datetime = @@date_parser.parseDateTime(self.timestamp)
format = key[1 .. -1]
datetime.toString(format) # return requested time format
end
t = @data["@timestamp"]
next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).format(key[1 .. -1])
else
# Use an event field.
value = self[key]
case value
when nil
tok # leave the %{foo} if this field does not exist in this event.
when Array
value.join(",") # Join by ',' if value is an array
when Hash
value.to_json # Convert hashes to json
else
value # otherwise return the value
end
end
end
when nil
tok # leave the %{foo} if this field does not exist in this event.
when Array
value.join(",") # Join by ',' if value is an array
when Hash
value.to_json # Convert hashes to json
else
value # otherwise return the value
end # case value
end # 'key' checking
end # format.gsub...
end # def sprintf
# Shims to remove after event v1 is the default.
def tags=(value); self["tags"] = value; end
def message=(value); self["message"] = value; end
def source=(value); self["source"] = value; end
def type=(value); self["type"] = value; end
def type; return self["type"]; end
end # module LogStash::EventV1

34
lib/logstash/util.rb
View file

@ -32,4 +32,38 @@ module LogStash::Util
LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
end
end # def set_thread_name
# Merge hash 'src' into 'dst' nondestructively
#
# Duplicate keys will become array values
#
# [ src["foo"], dst["foo"] ]
def self.hash_merge(dst, src)
src.each do |name, svalue|
if dst.include?(name)
dvalue = dst[name]
if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
dvalue = hash_merge(dvalue, svalue)
elsif svalue.is_a?(Array)
if dvalue.is_a?(Array)
# merge arrays without duplicates.
dvalue |= svalue
else
dvalue = [dvalue] | svalue
end
else
if dvalue.is_a?(Array)
dvalue << svalue unless dvalue.include?(svalue)
else
dvalue = [dvalue, svalue] unless dvalue == svalue
end
end
dst[name] = dvalue
else
# dst doesn't have this key, just set it.
dst[name] = svalue
end
end
end # def self.hash_merge
end # module LogStash::Util

112
spec/event.rb
View file

@ -2,96 +2,82 @@ require "logstash/event"
require "insist"
describe LogStash::Event do
before :each do
@event = LogStash::Event.new
@event.timestamp = "2013-01-01T00:00:00.000Z"
@event.type = "sprintf"
@event.message = "hello world"
@event.tags = [ "tag1" ]
@event.source = "/home/foo"
@event["@fields"] = {
"a" => "b",
"c" => {
"d" => "f",
"e.f" => "g"
},
"c.d" => "e",
"f.g" => {
"h" => "i"
},
"j" => {
"k1" => "v",
"k2" => [
"w",
"x"
],
"k3.4" => "m",
5 => 6,
"5" => 7
}
}
subject do
event = LogStash::Event.new
event.timestamp = Time.at(1356998400) #"2013-01-01T00:00:00.000Z"
event.type = "sprintf"
event.message = "hello world"
event.tags = [ "tag1" ]
event.source = "/home/foo"
event.to_hash.merge!(
"a" => "b",
"c" => { "d" => "f", "e.f" => "g" },
"c.d" => "e",
"f.g" => { "h" => "i" },
"j" => {
"k1" => "v",
"k2" => [ "w", "x" ],
"k3.4" => "m",
5 => 6,
"5" => 7
}
)
next event
end
subject { @event }
context "#sprintf" do
it "should report a unix timestamp for %{+%s}" do
insist { @event.sprintf("%{+%s}") } == "1356998400"
insist { subject.sprintf("%{+%s}") } == "1356998400"
end
it "should report a time with %{+format} syntax" do
insist { @event.sprintf("%{+YYYY}") } == "2013"
insist { @event.sprintf("%{+MM}") } == "01"
insist { @event.sprintf("%{+HH}") } == "00"
insist { subject.sprintf("%{+YYYY}") } == "2013"
insist { subject.sprintf("%{+MM}") } == "01"
insist { subject.sprintf("%{+HH}") } == "00"
end
it "should report fields with %{field} syntax" do
insist { @event.sprintf("%{@type}") } == "sprintf"
insist { @event.sprintf("%{@message}") } == subject["@message"]
insist { subject.sprintf("%{@type}") } == "sprintf"
insist { subject.sprintf("%{@message}") } == subject["@message"]
end
it "should print deep fields" do
insist { @event.sprintf("%{j.k1}") } == "v"
insist { @event.sprintf("%{j.k2.0}") } == "w"
insist { subject.sprintf("%{j.k1}") } == "v"
insist { subject.sprintf("%{j.k2.0}") } == "w"
end
end
context "#[]" do
it "should fetch data" do
insist { @event["@type"] } == "sprintf"
insist { subject["@type"] } == "sprintf"
end
it "should fetch fields" do
insist { @event["a"] } == "b"
insist { @event['c\.d'] } == "e"
insist { subject["a"] } == "b"
insist { subject['c\.d'] } == "e"
end
it "should fetch deep fields" do
insist { @event["j.k1"] } == "v"
insist { @event["c.d"] } == "f"
insist { @event['f\.g.h'] } == "i"
insist { @event['j.k3\.4'] } == "m"
insist { @event['j.5'] } == 7
insist { subject["j.k1"] } == "v"
insist { subject["c.d"] } == "f"
insist { subject['f\.g.h'] } == "i"
insist { subject['j.k3\.4'] } == "m"
insist { subject['j.5'] } == 7
end
end
context "#append" do
it "should append message with \\n" do
subject.append(LogStash::Event.new("@message" => "hello world"))
insist { subject.message } == "hello world\nhello world"
end
it "should concatenate tags" do
subject.append(LogStash::Event.new("@tags" => [ "tag2" ]))
insist { subject.tags } == [ "tag1", "tag2" ]
subject.append(LogStash::Event.new("tags" => [ "tag2" ]))
insist { subject["tags"] } == [ "tag1", "tag2" ]
end
context "when event field is nil" do
it "should add single value as string" do
subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
subject.append(LogStash::Event.new("field1" => "append1"))
insist { subject[ "field1" ] } == "append1"
end
it "should add multi values as array" do
subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","append2" ]}))
subject.append(LogStash::Event.new("field1" => [ "append1","append2" ]))
insist { subject[ "field1" ] } == [ "append1","append2" ]
end
end
@ -100,19 +86,19 @@ describe LogStash::Event do
before { subject[ "field1" ] = "original1" }
it "should append string to values, if different from current" do
subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
subject.append(LogStash::Event.new("field1" => "append1"))
insist { subject[ "field1" ] } == [ "original1", "append1" ]
end
it "should not change value, if appended value is equal current" do
subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"}))
insist { subject[ "field1" ] } == [ "original1" ]
subject.append(LogStash::Event.new("field1" => "original1"))
insist { subject[ "field1" ] } == "original1"
end
it "should concatenate values in an array" do
subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1" ]}))
subject.append(LogStash::Event.new("field1" => [ "append1" ]))
insist { subject[ "field1" ] } == [ "original1", "append1" ]
end
it "should join array, removing duplicates" do
subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]}))
subject.append(LogStash::Event.new("field1" => [ "append1","original1" ]))
insist { subject[ "field1" ] } == [ "original1", "append1" ]
end
end
@ -120,15 +106,15 @@ describe LogStash::Event do
before { subject[ "field1" ] = [ "original1", "original2" ] }
it "should append string values to array, if not present in array" do
subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
subject.append(LogStash::Event.new("field1" => "append1"))
insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
end
it "should not append string values, if the array already contains it" do
subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"}))
subject.append(LogStash::Event.new("field1" => "original1"))
insist { subject[ "field1" ] } == [ "original1", "original2" ]
end
it "should join array, removing duplicates" do
subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]}))
subject.append(LogStash::Event.new("field1" => [ "append1","original1" ]))
insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
end
end