Merge pull request #225 from awheeler/master

Enhanced kv filter

lib/logstash/filters/kv.rb

@@ -28,6 +28,38 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     filter { kv { trim => "<>," } }
   config :trim, :validate => :string
 
+
+  # A string of characters to use as delimiters for parsing out key-value pairs.
+  #
+  # Example, to split out the args from a string such as
+  # '?pin=12345~0&d=123&e=foo@bar.com&oq=bobo&ss=12345':
+  # 
+  #     filter { kv { field_split => "&?" } }
+  config :field_split, :validate => :string, :default => ''
+
+
+  # A string of characters to use as delimiters for identifying key-value relations.
+  #
+  # Example, to identify key-values such as
+  # 'key1:value1 key2:value2':
+  # 
+  #     filter { kv { value_split => ":" } }
+  config :value_split, :validate => :string, :default => '='
+
+  # A string to prepend to all of the extracted keys
+  #
+  # Example, to prepend arg_ to all keys:
+  #
+  #     filter { kv { prefix => "arg_" } }
+  config :prefix, :validate => :string, :default => ''
+
+  # The name of the container to put all of the key-value pairs into 
+  #
+  # Example, to place all keys into container kv:
+  #
+  #     filter { kv { conatiner => "kv" } }
+  config :container, :validate => :string, :default => '@fields'
+
   def register
     @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
   end # def register
@@ -35,29 +67,43 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   def filter(event)
     return unless filter?(event)
 
+    kv_keys=Hash.new
+
     @fields.each do |fieldname|
       value = event[fieldname]
 
       case value
-        when String; parse(value, event)
-        when Array; value.each { |v| parse(v, event) }
+        when String; kv_keys = parse(value, event, kv_keys)
+        when Array; value.each { |v| kv_keys = parse(v, event, kv_keys) }
         else 
           @logger.warn("kv filter has no support for this type of data",
                        :type => value.type, :value => value)
       end # case value
     end
+    # If we have any keys, create/append the hash
+    if kv_keys.length > 0
+      if !event[@container].nil?
+        event[@container].merge!(kv_keys)
+      else
+        event[@container]= kv_keys
+      end
+    end
   end # def filter
 
   private
-  def parse(text, event)
-    #text.scan(/([^ =]+)=("[^"]+"|'[^']+'|[^ ]+)/) do |key, value|
-    text.scan(/([^ =]+)=(?:"([^"]+)"|'([^']+)'|([^ ]+))/) do |key, v1, v2, v3|
+  def parse(text, event, kv_keys)
+    if !event =~ /[@field_split]/
+      return kv_keys
+    end
+    scan_re = Regexp.new("([^ "+@field_split+@value_split+"]+)["+@value_split+"](?:\"([^\""+@field_split+"]+)\"|'([^'"+@field_split+"]+)'|([^ "+@field_split+"]+))")
+    text.scan(scan_re) do |key, v1, v2, v3|
       value = v1 || v2 || v3
       if !@trim.nil?
         value = value.gsub(@trim_re, "")
       end
-
-      event[key] = value
+      key = @prefix + key
+      kv_keys[key] = value
     end
+    return kv_keys
   end
 end # class LogStash::Filter::KV

spec/filters/kv.rb

@@ -19,6 +19,94 @@ describe LogStash::Filters::KV do
       insist { subject["baz"] } == "fizz"
       insist { subject["doublequoted"] } == "hello world"
       insist { subject["singlequoted"] } == "hello world"
+      insist {subject['@fields'].count } == 5
+    end
+
+  end
+
+  describe "test value_split" do
+    config <<-CONFIG
+      filter {
+        kv { value_split => ':' }
+      }
+    CONFIG
+
+    sample "hello:=world foo:bar baz=:fizz doublequoted:\"hello world\" singlequoted:'hello world'" do
+      insist { subject["hello"] } == "=world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz="] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist {subject['@fields'].count } == 5
+    end
+
+  end
+
+  describe "test field_split" do
+    config <<-CONFIG
+      filter {
+        kv { field_split => '?&' }
+      }
+    CONFIG
+
+    sample "?hello=world&foo=bar&baz=fizz&doublequoted=\"hello world\"&singlequoted='hello world'&ignoreme&foo12=bar12" do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist { subject["foo12"] } == "bar12"
+      insist {subject['@fields'].count } == 6
+    end
+
+  end
+
+  describe "test prefix" do
+    config <<-CONFIG
+      filter {
+        kv { prefix => '__' }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["__hello"] } == "world"
+      insist { subject["__foo"] } == "bar"
+      insist { subject["__baz"] } == "fizz"
+      insist { subject["__doublequoted"] } == "hello world"
+      insist { subject["__singlequoted"] } == "hello world"
+      insist {subject['@fields'].count } == 5
+    end
+
+  end
+
+  describe "test container" do
+    config <<-CONFIG
+      filter {
+        kv { container => 'kv' }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["kv"]["hello"] } == "world"
+      insist { subject["kv"]["foo"] } == "bar"
+      insist { subject["kv"]["baz"] } == "fizz"
+      insist { subject["kv"]["doublequoted"] } == "hello world"
+      insist { subject["kv"]["singlequoted"] } == "hello world"
+      insist {subject['@fields']["kv"].count } == 5
+    end
+
+  end
+
+  describe "test empty container" do
+    config <<-CONFIG
+      filter {
+        kv { container => 'kv' }
+      }
+    CONFIG
+
+    sample "hello:world:foo:bar:baz:fizz" do
+      insist { subject["kv"] } == nil
+      insist {subject['@fields'].count } == 0
     end
 
   end