Make event specs pass under event_v1

This required implementing Event#append. Some semantics have changed.
"@message" and "@tags" are no longer handled specially (mainly because
they aren't described by event v1 schema). This likely has broken the
multiline filter tests (intentionally). The multiline filter will need
to have custom merging of certain fields (like joining message with
newlines instead of putting it all in an array)

lib/logstash/event_v1.rb

@@ -35,7 +35,7 @@ module LogStash::EventV1
     @cancelled = false
 
     @data = data
-    @data["@timestamp"] = LogStash::Time.now if !@data.include?("@timestamp")
+    @data["@timestamp"] = ::Time.now if !@data.include?("@timestamp")
     @data["@version"] = "1" if !@data.include?("@version")
   end # def initialize
 
@@ -85,12 +85,24 @@ module LogStash::EventV1
   # field-related access
   public
   def [](key)
-    # TODO(sissel): Implement
+    if key[0] == '['
+      val = @data
+      key.gsub(/(?<=\[).+?(?=\])/).each do |tok|
+        if val.is_a? Array
+          val = val[tok.to_i]
+        else
+          val = val[tok]
+        end
+      end
+      return val
+    else
+      return @data[key]
+    end
   end # def []
   
   public
   def []=(key, value)
-    # TODO(sissel): Implement
+    @data[key] = value
   end # def []=
 
   public
@@ -104,7 +116,7 @@ module LogStash::EventV1
   end # def to_json
 
   def to_hash
-    raise DeprecatedMethod
+    return @data
   end # def to_hash
 
   public
@@ -120,8 +132,9 @@ module LogStash::EventV1
   # Append an event to this one.
   public
   def append(event)
-    raise NotImplementedError, "LogStash::EventV1#append needs implementing"
+    # non-destructively merge that event with ourselves.
-  end
+    LogStash::Util.hash_merge(@data, event.to_hash)
+  end # append
 
   # Remove a field. Returns the value of that field when deleted
   public
@@ -157,40 +170,30 @@ module LogStash::EventV1
 
       if key == "+%s"
         # Got %{+%s}, support for unix epoch time
-        if RUBY_ENGINE != "jruby"
+        next @data["@timestamp"].to_i
-          # This is really slow. See LOGSTASH-217
-          Time.parse(self.timestamp).to_i
-        else
-          datetime = @@date_parser.parseDateTime(self.timestamp)
-          (datetime.getMillis / 1000).to_i
-        end
       elsif key[0,1] == "+"
-        # We got a %{+TIMEFORMAT} so use joda to format it.
+        t = @data["@timestamp"]
-        if RUBY_ENGINE != "jruby"
+        next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).format(key[1 .. -1])
-          # This is really slow. See LOGSTASH-217
-          datetime = Date.parse(self.timestamp)
-          format = key[1 .. -1]
-          datetime.strftime(format)
-        else
-          datetime = @@date_parser.parseDateTime(self.timestamp)
-          format = key[1 .. -1]
-          datetime.toString(format) # return requested time format
-        end
       else
-        # Use an event field.
         value = self[key]
-
         case value
-        when nil
+          when nil
-          tok # leave the %{foo} if this field does not exist in this event.
+            tok # leave the %{foo} if this field does not exist in this event.
-        when Array
+          when Array
-          value.join(",") # Join by ',' if value is an array
+            value.join(",") # Join by ',' if value is an array
-        when Hash
+          when Hash
-          value.to_json # Convert hashes to json
+            value.to_json # Convert hashes to json
-        else
+          else
-          value # otherwise return the value
+            value # otherwise return the value
-        end
+        end # case value
-      end
+      end # 'key' checking
-    end
+    end # format.gsub...
   end # def sprintf
+
+  # Shims to remove after event v1 is the default.
+  def tags=(value); self["tags"] = value; end
+  def message=(value); self["message"] = value; end
+  def source=(value); self["source"] = value; end
+  def type=(value); self["type"] = value; end
+  def type; return self["type"]; end
 end # module LogStash::EventV1

lib/logstash/util.rb

@@ -32,4 +32,38 @@ module LogStash::Util
       LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
     end
   end # def set_thread_name
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array) 
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue |= svalue
+          else
+            dvalue = [dvalue] | svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+  end # def self.hash_merge
 end # module LogStash::Util

spec/event.rb

@@ -2,96 +2,82 @@ require "logstash/event"
 require "insist"
 
 describe LogStash::Event do
-  before :each do
+  subject do
-    @event = LogStash::Event.new
+    event = LogStash::Event.new
-    @event.timestamp = "2013-01-01T00:00:00.000Z"
+    event.timestamp = Time.at(1356998400) #"2013-01-01T00:00:00.000Z"
-    @event.type = "sprintf"
+    event.type = "sprintf"
-    @event.message = "hello world"
+    event.message = "hello world"
-    @event.tags = [ "tag1" ]
+    event.tags = [ "tag1" ]
-    @event.source = "/home/foo"
+    event.source = "/home/foo"
-    @event["@fields"] = { 
+    event.to_hash.merge!(
-        "a" => "b", 
+      "a" => "b", 
-        "c" => {
+      "c" => { "d" => "f", "e.f" => "g" }, 
-            "d" => "f",
+      "c.d" => "e",
-            "e.f" => "g"
+      "f.g" => { "h" => "i" },
-        }, 
+      "j" => { 
-        "c.d" => "e",
+        "k1" => "v", 
-        "f.g" => { 
+        "k2" => [ "w", "x" ],
-            "h" => "i" 
+        "k3.4" => "m",
-        },
+        5 => 6,
-        "j" => { 
+        "5" => 7
-            "k1" => "v", 
+      } 
-            "k2" => [
+    )
-                "w", 
+    next event
-                "x" 
-            ],
-            "k3.4" => "m",
-            5 => 6,
-            "5" => 7
-        } 
-    }
   end
 
-  subject { @event }
-
   context "#sprintf" do
     it "should report a unix timestamp for %{+%s}" do
-      insist { @event.sprintf("%{+%s}") } == "1356998400"
+      insist { subject.sprintf("%{+%s}") } == "1356998400"
     end
     
     it "should report a time with %{+format} syntax" do
-      insist { @event.sprintf("%{+YYYY}") } == "2013"
+      insist { subject.sprintf("%{+YYYY}") } == "2013"
-      insist { @event.sprintf("%{+MM}") } == "01"
+      insist { subject.sprintf("%{+MM}") } == "01"
-      insist { @event.sprintf("%{+HH}") } == "00"
+      insist { subject.sprintf("%{+HH}") } == "00"
     end
   
     it "should report fields with %{field} syntax" do
-      insist { @event.sprintf("%{@type}") } == "sprintf"
+      insist { subject.sprintf("%{@type}") } == "sprintf"
-      insist { @event.sprintf("%{@message}") } == subject["@message"]
+      insist { subject.sprintf("%{@message}") } == subject["@message"]
     end
     
     it "should print deep fields" do
-      insist { @event.sprintf("%{j.k1}") } == "v"
+      insist { subject.sprintf("%{j.k1}") } == "v"
-      insist { @event.sprintf("%{j.k2.0}") } == "w"
+      insist { subject.sprintf("%{j.k2.0}") } == "w"
     end
   end
   
   context "#[]" do
     it "should fetch data" do
-      insist { @event["@type"] } == "sprintf"
+      insist { subject["@type"] } == "sprintf"
     end
     it "should fetch fields" do
-      insist { @event["a"] } == "b"
+      insist { subject["a"] } == "b"
-      insist { @event['c\.d'] } == "e"
+      insist { subject['c\.d'] } == "e"
     end
     it "should fetch deep fields" do
-      insist { @event["j.k1"] } == "v"
+      insist { subject["j.k1"] } == "v"
-      insist { @event["c.d"] } == "f"
+      insist { subject["c.d"] } == "f"
-      insist { @event['f\.g.h'] } == "i"
+      insist { subject['f\.g.h'] } == "i"
-      insist { @event['j.k3\.4'] } == "m"
+      insist { subject['j.k3\.4'] } == "m"
-      insist { @event['j.5'] } == 7
+      insist { subject['j.5'] } == 7
 
     end
   end
 
   context "#append" do
-    it "should append message with \\n" do
-      subject.append(LogStash::Event.new("@message" => "hello world"))
-      insist { subject.message } == "hello world\nhello world"
-    end
-  
     it "should concatenate tags" do
-      subject.append(LogStash::Event.new("@tags" => [ "tag2" ]))
+      subject.append(LogStash::Event.new("tags" => [ "tag2" ]))
-      insist { subject.tags } == [ "tag1", "tag2" ]
+      insist { subject["tags"] } == [ "tag1", "tag2" ]
     end
   
     context "when event field is nil" do
       it "should add single value as string" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
+        subject.append(LogStash::Event.new("field1" => "append1"))
         insist { subject[ "field1" ] } == "append1"
       end
       it "should add multi values as array" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","append2" ]}))
+        subject.append(LogStash::Event.new("field1" => [ "append1","append2" ]))
         insist { subject[ "field1" ] } == [ "append1","append2" ]
       end
     end
@@ -100,19 +86,19 @@ describe LogStash::Event do
       before { subject[ "field1" ] = "original1" }
   
       it "should append string to values, if different from current" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
+        subject.append(LogStash::Event.new("field1" => "append1"))
         insist { subject[ "field1" ] } == [ "original1", "append1" ]
       end
       it "should not change value, if appended value is equal current" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"}))
+        subject.append(LogStash::Event.new("field1" => "original1"))
-        insist { subject[ "field1" ] } == [ "original1" ]
+        insist { subject[ "field1" ] } == "original1"
       end
       it "should concatenate values in an array" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1" ]}))
+        subject.append(LogStash::Event.new("field1" => [ "append1" ]))
         insist { subject[ "field1" ] } == [ "original1", "append1" ]
       end
       it "should join array, removing duplicates" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]}))
+        subject.append(LogStash::Event.new("field1" => [ "append1","original1" ]))
         insist { subject[ "field1" ] } == [ "original1", "append1" ]
       end
     end
@@ -120,15 +106,15 @@ describe LogStash::Event do
       before { subject[ "field1" ] = [ "original1", "original2" ] }
   
       it "should append string values to array, if not present in array" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => "append1"}))
+        subject.append(LogStash::Event.new("field1" => "append1"))
         insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
       end
       it "should not append string values, if the array already contains it" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => "original1"}))
+        subject.append(LogStash::Event.new("field1" => "original1"))
         insist { subject[ "field1" ] } == [ "original1", "original2" ]
       end
       it "should join array, removing duplicates" do
-        subject.append(LogStash::Event.new("@fields" => {"field1" => [ "append1","original1" ]}))
+        subject.append(LogStash::Event.new("field1" => [ "append1","original1" ]))
         insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
       end
     end