support add_tag and add_field in filters/base

etc/agent.conf

@@ -9,6 +9,10 @@ filter {
   grok {
     type => "linux-syslog"
     pattern => ["%{SYSLOG_SUDO}", "%{SYSLOG_KERNEL}", "%{SYSLOGLINE}"]
+    add_tag => "test_tag1"
+    add_tag => ["test_tag2", "test_tag3"]
+    add_field => ["grok_filtered", "true"]
+    add_field => ["test_key", "test_value"]
   }
 }
 

lib/logstash/config/grammar.rb

@@ -4,7 +4,7 @@ require "rubygems"
 require "logstash/namespace"
 
 
-# line 146 "grammar.rl"
+# line 145 "grammar.rl"
 
 
 class LogStash::Config::Grammar
@@ -246,7 +246,7 @@ end
 self.logstash_config_en_main = 53;
 
 
-# line 155 "grammar.rl"
+# line 154 "grammar.rl"
     # END RAGEL DATA
 
     @tokenstack = Array.new
@@ -268,7 +268,7 @@ begin
 	cs = logstash_config_start
 end
 
-# line 169 "grammar.rl"
+# line 168 "grammar.rl"
     # END RAGEL INIT
 
     begin 
@@ -441,18 +441,17 @@ when 8 then
     name = @stack.pop
     #@components << { :name => name, :parameters => @parameters }
     @components << { name => @parameters }
-    @parameters = {}
+    @parameters = Hash.new { |h,k| h[k] = [] }
   		end
 when 9 then
 # line 78 "grammar.rl"
 		begin
 
-    #puts "current component: " + @stack.last
     @components = []
     @parameters = Hash.new { |h,k| h[k] = [] }
   		end
 when 10 then
-# line 84 "grammar.rl"
+# line 83 "grammar.rl"
 		begin
 
     name = @stack.pop
@@ -461,18 +460,18 @@ when 10 then
     #puts "Config component: #{name}"
   		end
 when 11 then
-# line 91 "grammar.rl"
+# line 90 "grammar.rl"
 		begin
  e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" 		end
 when 13 then
-# line 141 "grammar.rl"
+# line 140 "grammar.rl"
 		begin
  
             # Compute line and column of the cursor (p)
             $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
             # TODO(sissel): Note what we were expecting?
           		end
-# line 476 "grammar.rb"
+# line 475 "grammar.rb"
 			end # action switch
 		end
 	end
@@ -508,7 +507,7 @@ when 0 then
     #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
   		end
 when 10 then
-# line 84 "grammar.rl"
+# line 83 "grammar.rl"
 		begin
 
     name = @stack.pop
@@ -517,22 +516,22 @@ when 10 then
     #puts "Config component: #{name}"
   		end
 when 11 then
-# line 91 "grammar.rl"
+# line 90 "grammar.rl"
 		begin
  e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" 		end
 when 12 then
-# line 140 "grammar.rl"
+# line 139 "grammar.rl"
 		begin
  puts "END" 		end
 when 13 then
-# line 141 "grammar.rl"
+# line 140 "grammar.rl"
 		begin
  
             # Compute line and column of the cursor (p)
             $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
             # TODO(sissel): Note what we were expecting?
           		end
-# line 536 "grammar.rb"
+# line 535 "grammar.rb"
 		end # eof action switch
 	end
 	if _trigger_goto
@@ -546,7 +545,7 @@ end
 	end
 	end
 
-# line 174 "grammar.rl"
+# line 173 "grammar.rl"
       # END RAGEL EXEC
     rescue => e
       # Compute line and column of the cursor (p)

lib/logstash/config/grammar.rl

@@ -72,11 +72,10 @@ require "logstash/namespace"
     name = @stack.pop
     #@components << { :name => name, :parameters => @parameters }
     @components << { name => @parameters }
-    @parameters = {}
+    @parameters = Hash.new { |h,k| h[k] = [] }
   }
 
   action component_init {
-    #puts "current component: " + @stack.last
     @components = []
     @parameters = Hash.new { |h,k| h[k] = [] }
   }

lib/logstash/file/manager.rb

@@ -78,7 +78,7 @@ class LogStash::File::Manager
         e = LogStash::Event.new({
           "@message" => line,
           "@type" => config["type"],
-          "@tags" => config["tag"],
+          "@tags" => config["tag"].dup,
         })
         e.source = "file://#{@hostname}/#{path}"
         @output_queue << e.dup

lib/logstash/filters/base.rb

@@ -9,6 +9,8 @@ class LogStash::Filters::Base
 
   config_name "filter"
   config :type => :string
+  config :add_tag => nil
+  config :add_field => nil
 
   public
   def initialize(params)
@@ -34,4 +36,25 @@ class LogStash::Filters::Base
       @config[type] = typeconfig
     end
   end # def add_config
+
+  # a filter instance should call filter_matches from filter if the event
+  # matches the filter's conditions (right type, etc)
+  private
+  def filter_matched(event)
+    if @add_tag
+      @add_tag.each { |tag| event.tags << tag }
+    end
+    if @add_field
+      if @add_field.length % 2 != 0
+        @logger.warn("filter #{self.class}: add_field must be an even amount of fields [key1, val1, key2, val2, ...]")
+      else
+        combos = @add_field.length / 2
+        0.upto(combos - 1) do |i|
+          field, value = @add_field[2*i], @add_field[2*i+1]
+          event[field] ||= []
+          event[field] << value
+        end # 0.upto(combos)
+      end # if @add_field.length % 2
+    end # if @add_field
+  end # def filter_matched
 end # class LogStash::Filters::Base

lib/logstash/filters/grok.rb

@@ -51,6 +51,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     end
 
     if match
+      filter_matched(event)
       match.each_capture do |key, value|
         if key.include?(":")
           key = key.split(":")[1]