- add grok config 'break_on_match' defaults to previous behavior of

'true'. If false, grok will attempt to match all given patterns and
  produce new fields as they are matched.
- Added test coverage to ensure the above

lib/logstash/filters/grok.rb

@@ -35,7 +35,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
   # Specify a pattern to parse with. This will match the '@message' field.
   #
   # If you want to match other fields than @message, use the 'match' setting.
-  # Multiple patterns is fine. First match breaks.
+  # Multiple patterns is fine.
   config :pattern, :validate => :array
 
   # Specify a path to a directory with grok pattern files in it
@@ -72,6 +72,11 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
   # requested in: googlecode/issue/26
   config :drop_if_match, :validate => :boolean, :default => false
 
+  # Break on first match. The first successful match by grok will result in the
+  # filter being finished. If you want grok to try all patterns (maybe you are
+  # parsing different things), then set this to false.
+  config :break_on_match, :validate => :boolean, :default => true
+
   # If true, only store named captures from grok.
   config :named_captures_only, :validate => :boolean, :default => false
 
@@ -127,7 +132,8 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     @match.merge(@config).each do |field, patterns|
       # Skip known config names
       next if ["add_tag", "add_field", "type", "match", "patterns_dir",
-               "drop_if_match", "named_captures_only", "pattern" ].include?(field)
+               "drop_if_match", "named_captures_only", "pattern",
+               "break_on_match" ].include?(field)
       if !@patterns.include?(field)
         @patterns[field] = Grok::Pile.new 
         add_patterns_from_files(@patternfiles, @patterns[field])
@@ -156,18 +162,21 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     end
 
     @logger.debug(["Running grok filter", event])
+    done = false
     @patterns.each do |field, pile|
+      break if done
       if !event[field]
         @logger.debug(["Skipping match object, field not present", field,
                       event, event[field]])
         next
       end
 
-      @logger.debug(["Trying pattern for type #{event.type}", pile])
+      @logger.debug(["Trying pattern for type #{event.type}", { :pile => pile, :field => field }])
       (event[field].is_a?(Array) ? event[field] : [event[field]]).each do |fieldvalue|
         grok, match = pile.match(fieldvalue)
         next unless match
         matched = true
+        done = true if @break_on_match
 
         match.each_capture do |key, value|
           type_coerce = nil

test/logstash/filters/test_grok.rb

@@ -11,6 +11,11 @@ require "logstash/event"
 
 class TestFilterGrok < Test::Unit::TestCase
 
+  def setup
+    @typename = nil
+    @filter = nil
+  end
+
   def test_name(name)
     @typename = name.gsub(/[ ]/, "_")
   end
@@ -23,9 +28,10 @@ class TestFilterGrok < Test::Unit::TestCase
       end
     end
 
+    p :config => cfg
+    p :filter => @filter
     @filter = LogStash::Filters::Grok.new(cfg)
-    p :config => cfg, :id => @filter.object_id
-    p :fizzle => @filter.pattern
+
     @filter.register
     #p :newfilter => @filter
   end
@@ -133,7 +139,7 @@ class TestFilterGrok < Test::Unit::TestCase
 
     @filter.filter(event)
     assert_equal(expect.class, event.fields["foo"].first.class, "Expected field 'foo' to be of type #{expect.class.name} but got #{event.fields["foo"].first.class.name}")
-    assert_equal([expect], event.fields["foo"], "Expected field 'foo' to be [#{expect.inspect}], is #{event.fields["expect"].inspect}")
+    assert_equal([expect], event.fields["foo"], "Expected field 'foo' to be [#{expect.inspect}], is #{event.fields["foo"].inspect}")
   end # def test_grok_type_hinting_float
 
   def test_grok_inline_define
@@ -148,7 +154,7 @@ class TestFilterGrok < Test::Unit::TestCase
 
     @filter.filter(event)
     assert_equal(expect.class, event.fields["FIZZLE"].first.class, "Expected field 'FIZZLE' to be of type #{expect.class.name} but got #{event.fields["FIZZLE"].first.class.name}")
-    assert_equal([expect], event.fields["FIZZLE"], "Expected field 'FIZZLE' to be [#{expect.inspect}], is #{event.fields["expect"].inspect}")
+    assert_equal([expect], event.fields["FIZZLE"], "Expected field 'FIZZLE' to be [#{expect.inspect}], is #{event.fields["FIZZLE"].inspect}")
   end # def test_grok_type_hinting_float
 
   def test_grok_field_name_attribute
@@ -163,6 +169,30 @@ class TestFilterGrok < Test::Unit::TestCase
 
     @filter.filter(event)
     assert_equal(expect.class, event.fields["FIZZLE"].first.class, "Expected field 'FIZZLE' to be of type #{expect.class.name} but got #{event.fields["FIZZLE"].first.class.name}")
-    assert_equal([expect], event.fields["FIZZLE"], "Expected field 'FIZZLE' to be [#{expect.inspect}], is #{event.fields["expect"].inspect}")
-  end # def test_grok_type_hinting_float
-end
+    assert_equal([expect], event.fields["FIZZLE"], "Expected field 'FIZZLE' to be [#{expect.inspect}], is #{event.fields["FIZZLE"].inspect}")
+  end # def test_grok_field_name_attribute
+
+  def test_grok_field_name_and_pattern_coexisting
+    test_name "grok_field_name_attribute"
+    config "rum" => [ "%{FIZZLE=\\d+}" ], "pattern" => "%{WORD}", "break_on_match" => "false"
+    
+    event = LogStash::Event.new
+    event.type = @typename
+
+    expect = "1234"
+    event.fields["rum"] = "hello #{expect}"
+    event.message = "something fizzle"
+
+    @filter.filter(event)
+    assert_equal(expect.class, event.fields["FIZZLE"].first.class,
+                 "Expected field 'FIZZLE' to be of type #{expect.class.name} " \
+                 "but got #{event.fields["FIZZLE"].first.class.name}")
+    assert_equal([expect], event.fields["FIZZLE"],
+                 "Expected field 'FIZZLE' to be [#{expect.inspect}], is " \
+                 "#{event.fields["FIZZLE"].inspect}")
+
+    assert_equal(["something"], event.fields["WORD"],
+                 "Expected field 'WORD' to be ['something'], is " \
+                 "#{event.fields["WORD"].inspect}")
+  end # def test_grok_field_name_attribute
+end # class TestFilterGrok