- Add tests for filters/multiline

lib/logstash/event.rb

@@ -71,4 +71,19 @@ module LogStash; class Event
   end
 
   def include?(key); return @data.include?(key) end
+
+  # Append an event to this one.
+  def append(event)
+    self.message += "\n" + event.message 
+    self.tags |= event.tags
+
+    # Append all fields
+    event.fields.each do |name, value|
+      if event.fields.include?(name)
+        event.fields[name] |= value
+      else
+        event.fields[name] = value
+      end
+    end # event.fields.each
+  end
 end; end # class LogStash::Event

lib/logstash/filters/multiline.rb

@@ -61,18 +61,18 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
       @types[type] = typeconfig
 
       if !typeconfig.include?("pattern")
-        @logger.fatal(["'multiline' filter config for type #{type} is missing"
+        @logger.fatal(["'multiline' filter config for type #{type} is missing" \
                        " 'pattern' setting", typeconfig])
       end
 
       if !typeconfig.include?("what")
-        @logger.fatal(["'multiline' filter config for type #{type} is missing"
+        @logger.fatal(["'multiline' filter config for type #{type} is missing" \
                        " 'what' setting", typeconfig])
       end
 
       if !["next", "previous"].include?(typeconfig["what"])
-        @logger.fatal(["'multiline' filter config for type #{type} has invalid"
-                       " 'what' value. Must be 'next' or 'previous'",
+        @logger.fatal(["'multiline' filter config for type #{type} has " \
+                       "invalid 'what' value. Must be 'next' or 'previous'",
                        typeconfig])
       end
 
@@ -87,42 +87,70 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
 
   def filter(event)
     return unless @types.member?(event.type)
-    @types[event.type].each do |typeconfig|
-      match = typeconfig["pattern"].match(event.message)
-      key = [event.source, event.type]
-      pending = @pending[key]
+    typeconfig = @types[event.type]
+    #@types[event.type].each do |key, value|
+    #puts "#{event.type}: " + typeconfig.inspect
+    match = typeconfig["pattern"].match(event.message)
+    key = [event.source, event.type]
+    pending = @pending[key]
 
-      case typeconfig["what"]
-      when "prev"
-        if match
-          # previous previous line is part of this event.
-          # append it to the event and cancel it
+    case typeconfig["what"]
+    when "previous"
+      if match
+        # previous previous line is part of this event.
+        # append it to the event and cancel it
+        if pending
           pending.append(event)
-          event.cancel
         else
-          # this line is not part of the previous event
-          # if we have a pending event, it's done, send it.
-          # put the current event into pending
-          if pending
-            tmp = event.to_hash
-            event.overwrite(pending)
-            @pending[key]  = Event.new(tmp)
-          else
-            @pending[event.source]
-          end
-        end
-      when "next"
-        if match
-          # this line is part of a multiline event, the next
-          # line will be part, too, put it into pending.
-        else
-          # if we have something in pending, join it with this message
-          # and send it. otherwise, this is a new message and not part of
-          # multiline, send it.
+          @pending[key] = event
         end
+        event.cancel
       else
-        @logger.warn(["Unknown multiline 'what' value.", typeconfig])
-      end
-    end # @types[event.type].each
+        # this line is not part of the previous event
+        # if we have a pending event, it's done, send it.
+        # put the current event into pending
+        if pending
+          tmp = event.to_hash
+          event.overwrite(pending)
+          @pending[key] = LogStash::Event.new(tmp)
+        else
+          @pending[key] = event
+          event.cancel
+        end # if/else pending
+      end # if/else match
+    when "next"
+      if match
+        # this line is part of a multiline event, the next
+        # line will be part, too, put it into pending.
+        if pending
+          pending.append(event)
+        else
+          @pending[key] = event
+        end
+        event.cancel
+      else
+        # if we have something in pending, join it with this message
+        # and send it. otherwise, this is a new message and not part of
+        # multiline, send it.
+        if pending
+          pending.append(event)
+          event.overwrite(pending.to_hash)
+          @pending.delete(key)
+        end
+      end # if/else match
+    else
+      @logger.warn(["Unknown multiline 'what' value.", typeconfig])
+    end # case typeconfig["what"]
+    #end # @types[event.type].each
   end # def filter
+
+  # flush any pending messages
+  def flush(source, type)
+    key = [source, type]
+    if @pending[key]
+      event = @pending[key]
+      @pending.delete(key)
+    end
+    return event
+  end
 end # class LogStash::Filters::Date

test/filters/multiline.feature

@@ -0,0 +1,35 @@
+Feature: multiline filter
+  In order to ensure multiline filter is working
+  Events matching the multiline filter should be joined
+
+  Scenario: whitespace-leading lines (like java stack traces)
+    Given a multiline pattern of "^\s"
+      And a multiline what of "previous"
+    # We use quotes wrap lines here because cucumber will trim the whitespace
+    # otherwise
+    When the inputs are
+      |hello world|
+      |"   continued!"|
+      |one|
+      |two|
+      |"   two again"|
+    Then the event message should be
+      |hello world\n   continued!|
+      |one|
+      |two\n   two again|
+
+  Scenario: '...' continuation with next
+    Given a multiline pattern of "\.\.\.$"
+      And a multiline what of "next"
+    # We use quotes wrap lines here because cucumber will trim the whitespace
+    # otherwise
+    When the inputs are
+      |hello world... |
+      |"   continued!"|
+      |one|
+      |two...|
+      |"   two again"|
+    Then the event message should be
+      |hello world...\n   continued!|
+      |one|
+      |two...\n   two again|

test/filters/step_definitions/multiline.rb

@@ -0,0 +1,60 @@
+$:.unshift("#{File.dirname(__FILE__)}/../../../lib")
+require "logstash/event"
+require "logstash/filters/multiline"
+require "test/unit"
+
+include Test::Unit::Assertions
+
+Given /^a multiline pattern of "([^"]*)"$/ do |pattern|
+  @pattern = pattern
+end
+
+Given /^a multiline what of "([^"]*)"$/ do |what|
+  @what = what
+end
+
+When /^the inputs are$/ do |table|
+  @eventtype = "testtype"
+  @multiline = LogStash::Filters::Multiline.new({
+    @eventtype => {
+      "pattern" => @pattern,
+      "what" => @what
+    }
+  })
+  @multiline.register
+
+  @events = []
+  @source = "file:///test"
+  table.raw.each do |row|
+    # We are whitespace sensitive, and cucumber trims whitespace on
+    # table data sets, so let's hack around it with quotes.
+    message = row.first.gsub(/^"/, "").gsub(/"$/, "")
+    event = LogStash::Event.new({})
+    event.message = message
+    event.source = @source
+    event.type = @eventtype
+    @multiline.filter(event)
+
+    if !event.cancelled?
+      @events << event
+    end
+  end
+
+  lastevent = @multiline.flush(@source, @eventtype)
+  if lastevent
+    @events << lastevent
+  end
+end
+
+Then /^the event message should be$/ do |table|
+  # table is a Cucumber::Ast::Table
+  assert_equal(table.raw.length, @events.length,
+               "Should have #{table.raw.length} events, got #{@events.length}")
+  event_idx = 0
+  table.raw.each do |row|
+    message = row.first.gsub(/\\n/, "\n")
+    assert_equal(message, @events[event_idx].message, "Wrong message")
+    event_idx += 1
+  end
+end
+1