- Refine LogStash::Event to have less Hash feel.

- Add logging to filter/grok
- Make LogStash::Logger::Formatter only use awesome_inspect for non-strings

lib/logstash/event.rb

@@ -5,7 +5,12 @@ require "logstash/time"
 module LogStash; class Event
   def initialize(data)
     @cancelled = false
-    @data = data
+    @data = {
+      "source" => "unknown",
+      "tags" => [],
+      "fields" => {},
+    }.merge(data)
+
     if !@data.include?("timestamp")
       @data["timestamp"] = LogStash::Time.now.utc.to_iso8601
     end
@@ -15,10 +20,6 @@ module LogStash; class Event
     return Event.new(JSON.parse(json))
   end # def self.from_json
 
-  def to_json
-    return @data.to_json
-  end
-
   def cancel
     @cancelled = true
   end
@@ -31,31 +32,22 @@ module LogStash; class Event
     return "#{timestamp} #{source}: #{message}"
   end # def to_s
 
-  def [](key)
-    return @data[key]
-  end # def []
+  def timestamp; @data["timestamp"]; end # def timestamp
+  def timestamp=(val); @data["timestamp"] = val; end # def timestamp=
+  def source; @data["source"]; end # def source
+  def source=(val); @data["source"] = val; end # def source=
+  def message; @data["message"]; end # def message
+  def message=; @data["message"] = val; end # def message=
+  def tags; @data["tags"]; end # def tags
 
-  def []=(key, value)
-    @data[key] = value
-  end # def []=
+  # field-related access
+  def [](key); fields[key] end # def []
+  def []=(key, value); fields[key] = value end # def []=
+  def fields; return @data["fields"] end # def fields
+  
+  def to_json; return @data.to_json end # def to_json
 
-  def timestamp
-    @data["timestamp"]
-  end # def timestamp
+  def to_hash; return @data end # def to_hash
 
-  def source
-    @data["source"]
-  end # def source
-
-  def message
-    @data["message"]
-  end # def message
-
-  def to_hash
-    return @data
-  end # def to_hash
-
-  def include?(key)
-    return @data.include?(key)
-  end
+  def include?(key); return @data.include?(key) end
 end; end # class LogStash::Event

lib/logstash/filters/date.rb

@@ -11,10 +11,12 @@ class LogStash::Filters::Date
   #
   # filters:
   #   date:
-  #     tagname1:
+  #     <tagname>:
   #       <fieldname>: <format>
-  #     tagname2:
+  #     <tagname2>
   #       <fieldname>: <format>
+  #
+  # The format is whatever is supported by Ruby's DateTime.strptime
   def initialize(config = {})
     @config = config
     @tags = Hash.new { |h,k| h[k] = [] }
@@ -28,23 +30,23 @@ class LogStash::Filters::Date
   end # def register
 
   def filter(event)
-    return unless event.include?("tags")
-    event["tags"].each do |tag|
+    # TODO(sissel): crazy deep nesting here, refactor/redesign.
+    return if event.tags.empty?
+    event.tags.each do |tag|
       next unless @tags.include?(tag)
       @tags[tag].each do |tagconfig|
         tagconfig.each do |field, format|
-          #if event.include?(field) or (event["fields"].include?(field) rescue false)
-            #value = (event[field] or event["fields"][field])
-          if (event["fields"].include?(field) rescue false)
-            fieldvalue = event["fields"][field]
+          # TODO(sissel): check event.message, too.
+          if (event.fields.include?(field) rescue false)
+            fieldvalue = event.fields[field]
             #fieldvalue = [fieldvalue] if fieldvalue.is_a?(String)
             @logger.info fieldvalue
             fieldvalue.each do |value|
               #value = event["fields"][field]
               begin
                 time = DateTime.strptime(value, format)
-                event["timestamp"] = LogStash::Time.to_iso8601(time)
-                @logger.debug "Parsed #{value.inspect} as #{event["timestamp"]}"
+                event.timestamp = LogStash::Time.to_iso8601(time)
+                @logger.debug "Parsed #{value.inspect} as #{event.timestamp}"
               rescue => e
                 @logger.warn "Failed parsing date #{value.inspect} from field #{field} with format #{format.inspect}. Exception: #{e}"
               end
@@ -52,6 +54,6 @@ class LogStash::Filters::Date
           end # if this event has a field we expect to be a timestamp
         end # tagconfig.each
       end # @tags[tag].each
-    end # event["tags"].each
+    end # event.tags.each
   end # def filter
 end # class LogStash::Filters::Date

lib/logstash/filters/field.rb

@@ -21,10 +21,9 @@ class LogStash::Filters::Field
 
     @config.each do |condition|
       if data.instance_eval(condition)
-        return event
+        return # This event is OK, matches the condition.
       end
     end
     event.cancel
-    return event
   end
 end # class LogStash::Filters::Grok

lib/logstash/filters/grok.rb

@@ -1,10 +1,12 @@
 require "logstash/namespace"
+require "logstash/logging"
 
 gem "jls-grok", ">=0.2.3071"
 require "grok" # rubygem 'jls-grok'
 
 class LogStash::Filters::Grok
   def initialize(config = {})
+    @logger = LogStash::Logger.new(STDERR)
     @config = config
     @grokpiles = {}
   end # def initialize
@@ -12,6 +14,7 @@ class LogStash::Filters::Grok
   def register
     # TODO(sissel): Make patterns files come from the config
     @config.each do |tag, tagconfig|
+      @logger.debug("Grok tag #{tag}")
       pile = Grok::Pile.new
       pile.add_patterns_from_file("patterns/grok-patterns")
       pile.add_patterns_from_file("patterns/linux-syslog")
@@ -27,39 +30,41 @@ class LogStash::Filters::Grok
     message = event.message
     match = false
 
-    if event.include?("tags")
-      event["tags"].each do |tag|
+    if !event.tags.empty?
+      event.tags.each do |tag|
+        @logger.info @grokpiles.keys
         if @grokpiles.include?(tag)
           pile = @grokpiles[tag]
           grok, match = pile.match(message)
           break if match
         end # @grokpiles.include?(tag)
-      end # event["tags"].each
+      end # event.tags.each
     else 
       #pattern = @grok.discover(message)
       #@grok.compile(pattern)
       #match = @grok.match(message)
-      puts "No known tag for #{event.source} / #{event["tags"]}"
-      puts event.to_hash.inspect
+      @logger.info("No known tag for #{event.source} (tags: #{event.tags.inspect})")
+      @logger.debug(event.to_hash)
     end
 
     if match
-      event["fields"] = {}
       match.each_capture do |key, value|
         if key.include?(":")
           key = key.split(":")[1]
         end
 
-        if event["fields"][key].is_a?(String)
-          event["fields"][key] = [event["fields"][key]]
-        elsif event["fields"][key] == nil
-          event["fields"][key] = []
+        if event.fields[key].is_a?(String)
+          event.fields[key] = [event.fields[key]]
+        elsif event.fields[key] == nil
+          event.fields[key] = []
         end
 
-        event["fields"][key] << value
+        event.fields[key] << value
       end
     else
-      event["PARSEFAILURE"] = 1
+      # Tag this event if we can't parse it. We can use this later to
+      # reparse+reindex logs if we improve the patterns given .
+      event.tags << "grokparsefailure"
     end
-  end
+  end # def filter
 end # class LogStash::Filters::Grok

lib/logstash/inputs/amqp.rb

@@ -57,11 +57,7 @@ class LogStash::Inputs::Amqp
   end
 
   def receive(event)
-    if !event.include?("tags")
-      event["tags"] = @tags 
-    else
-      event["tags"] |= @tags # set union
-    end
+    event.tags |= @tags # set union
     @callback.call(event)
   end # def event
 end # class LogStash::Inputs::Amqp

lib/logstash/inputs/file.rb

@@ -40,7 +40,7 @@ class LogStash::Inputs::File
     event = LogStash::Event.new({
       "source" => @url.to_s,
       "message" => event,
-      "tags" => @tags,
+      "tags" => @tags.clone,
     })
     @callback.call(event)
   end # def event

lib/logstash/logging.rb

@@ -3,18 +3,21 @@ require "logger"
 require "ap"
 
 class LogStash::Logger < Logger
+  @@formatter = LogStash::Logger::Formatter.new
   def initialize(*args)
     super(*args)
-    @@formatter ||= LogStash::Logger::Formatter.new
     @formatter = @@formatter
   end
 end
 
+# Implement a custom Logger::Formatter that uses awesome_inspect on non-strings.
 class LogStash::Logger::Formatter < Logger::Formatter
-  # [:call, "INFO", Wed Oct 27 01:48:46 -0700 2010, nil, {"hello"=>12345}]e
   def call(level, timestamp, progname, object)
-    #TODO(sissel): implement
-    super(level, timestamp, progname, object.awesome_inspect)
+    if object.is_a?(String)
+      super(level, timestamp, progname, object)
+    else
+      super(level, timestamp, progname, object.awesome_inspect)
+    end
   end
 end # class LogStash::Logger::Formatter