Support deep fields (foo.bar) in date filter (LOGSTASH-724)

lib/logstash/event.rb

@@ -133,7 +133,27 @@ class LogStash::Event
     # If the key isn't in fields and it starts with an "@" sign, get it out of data instead of fields
     if ! @data["@fields"].has_key?(key) and key.slice(0,1) == "@"
       return @data[key]
-    # Exists in @fields (returns value) or doesn't start with "@" (return null)
+    elsif key.include?(".")
+      value = nil
+      obj = self
+      # "." is what ES uses to access structured data, so adopt that
+      # idea here, too.  "foo.bar" will access key "bar" under hash "foo".
+      key.split('.').each do |segment|
+        # try to safely cast segment to integer for the 0 in foo.0.bar
+        begin
+          segment = Integer(segment)
+        rescue Exception
+          #not an int, do nothing, segment remains a string
+        end
+        if obj
+          value = obj[segment] rescue nil
+          obj = obj[segment] rescue nil
+        else
+          value = nil
+          break
+        end
+      end # key.split.each
+      return value
     else
       return @data["@fields"][key]
     end
@@ -252,26 +272,7 @@ class LogStash::Event
         end
       else
         # Use an event field.
-        value = nil
-        obj = self
-
-        # If the top-level value exists, use that and don't try
-        # to "look" into data structures.
-        if self[key]
-          value = self[key]
-        else
-          # "." is what ES uses to access structured data, so adopt that
-          # idea here, too.  "foo.bar" will access key "bar" under hash "foo".
-          key.split('.').each do |segment|
-            if obj
-              value = obj[segment] rescue nil
-              obj = obj[segment] rescue nil
-            else
-              value = nil
-              break
-            end
-          end # key.split.each
-        end # if self[key]
+        value = self[key]
 
         case value
         when nil

lib/logstash/filters/date.rb

@@ -182,9 +182,9 @@ class LogStash::Filters::Date < LogStash::Filters::Base
       @logger.debug("Date filter: type #{event.type}, looking for field #{field.inspect}",
                     :type => event.type, :field => field)
       # TODO(sissel): check event.message, too.
-      next unless event.fields.member?(field)
+      next unless event[field]
 
-      fieldvalues = event.fields[field]
+      fieldvalues = event[field]
       fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
       fieldvalues.each do |value|
         next if value.nil?

spec/event.rb

@@ -9,11 +9,16 @@ describe LogStash::Event do
     @event.message = "hello world"
     @event.tags = [ "tag1" ]
     @event.source = "/home/foo"
+    @event["@fields"] = { "j" => { "k1" => "v", "k2" => [ "w", "x" ] } }
   end
 
   subject { @event }
 
   context "#sprintf" do
+    it "should report a unix timestamp for %{+%s}" do
+      insist { @event.sprintf("%{+%s}") } == "1356998400"
+    end
+    
     it "should report a time with %{+format} syntax" do
       insist { @event.sprintf("%{+YYYY}") } == "2013"
       insist { @event.sprintf("%{+MM}") } == "01"
@@ -24,6 +29,11 @@ describe LogStash::Event do
       insist { @event.sprintf("%{@type}") } == "sprintf"
       insist { @event.sprintf("%{@message}") } == subject["@message"]
     end
+    
+    it "should print deep fields" do
+      insist { @event.sprintf("%{j.k1}") } == "v"
+      insist { @event.sprintf("%{j.k2.0}") } == "w"
+    end
   end
 
   context "#append" do

spec/filters/date.rb

@@ -199,5 +199,18 @@ describe LogStash::Filters::Date do
       insist { subject["@timestamp"] } == time
     end
   end
+  
+  describe "support deep field access" do
+    config <<-CONFIG
+      filter { 
+        date {
+          match => [ "data.deep", "ISO8601" ]
+        }
+      }
+    CONFIG
+    
+    sample({ "@fields" => { "data" => { "deep" => "2013-01-01T00:00:00.000Z" } } }) do
+      insist { subject["@timestamp"] } == "2013-01-01T00:00:00.000Z"
+    end
+  end
 end
-

spec/filters/json.rb

@@ -18,6 +18,9 @@ describe LogStash::Filters::Json do
       insist { subject["hello"] } == "world"
       insist { subject["list" ] } == [1,2,3]
       insist { subject["hash"] } == { "k" => "v" }
+
+      insist { subject["list.0" ] } == 1
+      insist { subject["hash.k"] } == "v"
     end
   end
 
@@ -33,8 +36,14 @@ describe LogStash::Filters::Json do
 
     sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do
       insist { subject["data"]["hello"] } == "world"
-      insist { subject["data"]["list" ] } == [1,2,3]
+      insist { subject["data"]["list"] } == [1,2,3]
       insist { subject["data"]["hash"] } == { "k" => "v" }
+      
+      insist { subject["data.hello"] } == "world"
+      insist { subject["data.list" ] } == [1,2,3]
+      insist { subject["data.list.0" ] } == 1
+      insist { subject["data.hash"] } == { "k" => "v" }
+      insist { subject["data.hash.k"] } ==  "v"
     end
   end