From e494fb9a8cd64fa8c67f25b079e80c7f26e11002 Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Wed, 29 May 2013 15:55:24 -0700 Subject: [PATCH] Make event specs pass under event_v1 This required implementing Event#append. Some semantics have changed. "@message" and "@tags" are no longer handled specially (mainly because they aren't described by event v1 schema). This likely has broken the multiline filter tests (intentionally). The multiline filter will need to have custom merging of certain fields (like joining message with newlines instead of putting it all in an array) --- lib/logstash/event_v1.rb | 77 ++++++++++++++------------- lib/logstash/util.rb | 34 ++++++++++++ spec/event.rb | 112 +++++++++++++++++---------------------- 3 files changed, 123 insertions(+), 100 deletions(-) diff --git a/lib/logstash/event_v1.rb b/lib/logstash/event_v1.rb index 047caf649..acc48d488 100644 --- a/lib/logstash/event_v1.rb +++ b/lib/logstash/event_v1.rb @@ -35,7 +35,7 @@ module LogStash::EventV1 @cancelled = false @data = data - @data["@timestamp"] = LogStash::Time.now if !@data.include?("@timestamp") + @data["@timestamp"] = ::Time.now if !@data.include?("@timestamp") @data["@version"] = "1" if !@data.include?("@version") end # def initialize @@ -85,12 +85,24 @@ module LogStash::EventV1 # field-related access public def [](key) - # TODO(sissel): Implement + if key[0] == '[' + val = @data + key.gsub(/(?<=\[).+?(?=\])/).each do |tok| + if val.is_a? Array + val = val[tok.to_i] + else + val = val[tok] + end + end + return val + else + return @data[key] + end end # def [] public def []=(key, value) - # TODO(sissel): Implement + @data[key] = value end # def []= public @@ -104,7 +116,7 @@ module LogStash::EventV1 end # def to_json def to_hash - raise DeprecatedMethod + return @data end # def to_hash public @@ -120,8 +132,9 @@ module LogStash::EventV1 # Append an event to this one. public def append(event) - raise NotImplementedError, "LogStash::EventV1#append needs implementing" - end + # non-destructively merge that event with ourselves. + LogStash::Util.hash_merge(@data, event.to_hash) + end # append # Remove a field. Returns the value of that field when deleted public @@ -157,40 +170,30 @@ module LogStash::EventV1 if key == "+%s" # Got %{+%s}, support for unix epoch time - if RUBY_ENGINE != "jruby" - # This is really slow. See LOGSTASH-217 - Time.parse(self.timestamp).to_i - else - datetime = @@date_parser.parseDateTime(self.timestamp) - (datetime.getMillis / 1000).to_i - end + next @data["@timestamp"].to_i elsif key[0,1] == "+" - # We got a %{+TIMEFORMAT} so use joda to format it. - if RUBY_ENGINE != "jruby" - # This is really slow. See LOGSTASH-217 - datetime = Date.parse(self.timestamp) - format = key[1 .. -1] - datetime.strftime(format) - else - datetime = @@date_parser.parseDateTime(self.timestamp) - format = key[1 .. -1] - datetime.toString(format) # return requested time format - end + t = @data["@timestamp"] + next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).format(key[1 .. -1]) else - # Use an event field. value = self[key] - case value - when nil - tok # leave the %{foo} if this field does not exist in this event. - when Array - value.join(",") # Join by ',' if value is an array - when Hash - value.to_json # Convert hashes to json - else - value # otherwise return the value - end - end - end + when nil + tok # leave the %{foo} if this field does not exist in this event. + when Array + value.join(",") # Join by ',' if value is an array + when Hash + value.to_json # Convert hashes to json + else + value # otherwise return the value + end # case value + end # 'key' checking + end # format.gsub... end # def sprintf + + # Shims to remove after event v1 is the default. + def tags=(value); self["tags"] = value; end + def message=(value); self["message"] = value; end + def source=(value); self["source"] = value; end + def type=(value); self["type"] = value; end + def type; return self["type"]; end end # module LogStash::EventV1 diff --git a/lib/logstash/util.rb b/lib/logstash/util.rb index b8ab637c1..feeeb052d 100644 --- a/lib/logstash/util.rb +++ b/lib/logstash/util.rb @@ -32,4 +32,38 @@ module LogStash::Util LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0) end end # def set_thread_name + + # Merge hash 'src' into 'dst' nondestructively + # + # Duplicate keys will become array values + # + # [ src["foo"], dst["foo"] ] + def self.hash_merge(dst, src) + src.each do |name, svalue| + if dst.include?(name) + dvalue = dst[name] + if dvalue.is_a?(Hash) && svalue.is_a?(Hash) + dvalue = hash_merge(dvalue, svalue) + elsif svalue.is_a?(Array) + if dvalue.is_a?(Array) + # merge arrays without duplicates. + dvalue |= svalue + else + dvalue = [dvalue] | svalue + end + else + if dvalue.is_a?(Array) + dvalue << svalue unless dvalue.include?(svalue) + else + dvalue = [dvalue, svalue] unless dvalue == svalue + end + end + + dst[name] = dvalue + else + # dst doesn't have this key, just set it. + dst[name] = svalue + end + end + end # def self.hash_merge end # module LogStash::Util diff --git a/spec/event.rb b/spec/event.rb index 62c03af5d..97f9f6223 100644 --- a/spec/event.rb +++ b/spec/event.rb @@ -2,96 +2,82 @@ require "logstash/event" require "insist" describe LogStash::Event do - before :each do - @event = LogStash::Event.new - @event.timestamp = "2013-01-01T00:00:00.000Z" - @event.type = "sprintf" - @event.message = "hello world" - @event.tags = [ "tag1" ] - @event.source = "/home/foo" - @event["@fields"] = { - "a" => "b", - "c" => { - "d" => "f", - "e.f" => "g" - }, - "c.d" => "e", - "f.g" => { - "h" => "i" - }, - "j" => { - "k1" => "v", - "k2" => [ - "w", - "x" - ], - "k3.4" => "m", - 5 => 6, - "5" => 7 - } - } + subject do + event = LogStash::Event.new + event.timestamp = Time.at(1356998400) #"2013-01-01T00:00:00.000Z" + event.type = "sprintf" + event.message = "hello world" + event.tags = [ "tag1" ] + event.source = "/home/foo" + event.to_hash.merge!( + "a" => "b", + "c" => { "d" => "f", "e.f" => "g" }, + "c.d" => "e", + "f.g" => { "h" => "i" }, + "j" => { + "k1" => "v", + "k2" => [ "w", "x" ], + "k3.4" => "m", + 5 => 6, + "5" => 7 + } + ) + next event end - subject { @event } - context "#sprintf" do it "should report a unix timestamp for %{+%s}" do - insist { @event.sprintf("%{+%s}") } == "1356998400" + insist { subject.sprintf("%{+%s}") } == "1356998400" end it "should report a time with %{+format} syntax" do - insist { @event.sprintf("%{+YYYY}") } == "2013" - insist { @event.sprintf("%{+MM}") } == "01" - insist { @event.sprintf("%{+HH}") } == "00" + insist { subject.sprintf("%{+YYYY}") } == "2013" + insist { subject.sprintf("%{+MM}") } == "01" + insist { subject.sprintf("%{+HH}") } == "00" end it "should report fields with %{field} syntax" do - insist { @event.sprintf("%{@type}") } == "sprintf" - insist { @event.sprintf("%{@message}") } == subject["@message"] + insist { subject.sprintf("%{@type}") } == "sprintf" + insist { subject.sprintf("%{@message}") } == subject["@message"] end it "should print deep fields" do - insist { @event.sprintf("%{j.k1}") } == "v" - insist { @event.sprintf("%{j.k2.0}") } == "w" + insist { subject.sprintf("%{j.k1}") } == "v" + insist { subject.sprintf("%{j.k2.0}") } == "w" end end context "#[]" do it "should fetch data" do - insist { @event["@type"] } == "sprintf" + insist { subject["@type"] } == "sprintf" end it "should fetch fields" do - insist { @event["a"] } == "b" - insist { @event['c\.d'] } == "e" + insist { subject["a"] } == "b" + insist { subject['c\.d'] } == "e" end it "should fetch deep fields" do - insist { @event["j.k1"] } == "v" - insist { @event["c.d"] } == "f" - insist { @event['f\.g.h'] } == "i" - insist { @event['j.k3\.4'] } == "m" - insist { @event['j.5'] } == 7 + insist { subject["j.k1"] } == "v" + insist { subject["c.d"] } == "f" + insist { subject['f\.g.h'] } == "i" + insist { subject['j.k3\.4'] } == "m" + insist { subject['j.5'] } == 7 end end context "#append" do - it "should append message with \\n" do - subject.append(LogStash::Event.new("@message" => "hello world")) - insist { subject.message } == "hello world\nhello world" - end - it "should concatenate tags" do - subject.append(LogStash::Event.new("@tags" => [ "tag2" ])) - insist { subject.tags } == [ "tag1", "tag2" ] + subject.append(LogStash::Event.new("tags" => [ "tag2" ])) + insist { subject["tags"] } == [ "tag1", "tag2" ] end context "when event field is nil" do it "should add single value as string" do - subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"})) + subject.append(LogStash::Event.new("field1" => "append1")) insist { subject[ "field1" ] } == "append1" end it "should add multi values as array" do - subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","append2" ]})) + subject.append(LogStash::Event.new("field1" => [ "append1","append2" ])) insist { subject[ "field1" ] } == [ "append1","append2" ] end end @@ -100,19 +86,19 @@ describe LogStash::Event do before { subject[ "field1" ] = "original1" } it "should append string to values, if different from current" do - subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"})) + subject.append(LogStash::Event.new("field1" => "append1")) insist { subject[ "field1" ] } == [ "original1", "append1" ] end it "should not change value, if appended value is equal current" do - subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"})) - insist { subject[ "field1" ] } == [ "original1" ] + subject.append(LogStash::Event.new("field1" => "original1")) + insist { subject[ "field1" ] } == "original1" end it "should concatenate values in an array" do - subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1" ]})) + subject.append(LogStash::Event.new("field1" => [ "append1" ])) insist { subject[ "field1" ] } == [ "original1", "append1" ] end it "should join array, removing duplicates" do - subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]})) + subject.append(LogStash::Event.new("field1" => [ "append1","original1" ])) insist { subject[ "field1" ] } == [ "original1", "append1" ] end end @@ -120,15 +106,15 @@ describe LogStash::Event do before { subject[ "field1" ] = [ "original1", "original2" ] } it "should append string values to array, if not present in array" do - subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"})) + subject.append(LogStash::Event.new("field1" => "append1")) insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ] end it "should not append string values, if the array already contains it" do - subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"})) + subject.append(LogStash::Event.new("field1" => "original1")) insist { subject[ "field1" ] } == [ "original1", "original2" ] end it "should join array, removing duplicates" do - subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]})) + subject.append(LogStash::Event.new("field1" => [ "append1","original1" ])) insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ] end end