From 56a590a47464764c0c54e0c165603449476dabc0 Mon Sep 17 00:00:00 2001 From: Richard Pijnenburg Date: Sun, 13 Jan 2013 21:58:38 +0100 Subject: [PATCH] Refactoring - Replace fields and container with source and target - Add tests for the new variables --- lib/logstash/filters/kv.rb | 43 +++++++++++++++-- spec/filters/kv.rb | 94 +++++++++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 7 deletions(-) diff --git a/lib/logstash/filters/kv.rb b/lib/logstash/filters/kv.rb index e05d76d33..63d69df61 100644 --- a/lib/logstash/filters/kv.rb +++ b/lib/logstash/filters/kv.rb @@ -27,7 +27,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base plugin_status "beta" # The fields to perform 'key=value' searching on - config :fields, :validate => :array, :default => ["@message"] + config :fields, :validate => :array # A string of characters to trim from the value. This is useful if your # values are wrapped in brackets or are terminated by comma (like postfix @@ -87,10 +87,42 @@ class LogStash::Filters::KV < LogStash::Filters::Base # Example, to place all keys into container kv: # # filter { kv { container => "kv" } } - config :container, :validate => :string, :default => '@fields' + config :container, :validate => :string, :deprecated => true + + # The fields to perform 'key=value' searching on + # + # Example, to use the @message field: + # + # filter { kv { source => "@message" } } + config :source, :validate => :string, :default => '@message' + + # The name of the container to put all of the key-value pairs into + # + # Example, to place all keys into container kv: + # + # filter { kv { container => "kv" } } + config :target, :validate => :string, :default => '@fields' def register @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil? + + #TODO(electrical): Remove this when removing the container variable + if @container + if @target + logger.error("'container' and 'target' are the same setting, but 'container' is deprecated. Please use only 'target'") + end + @target = @container + end + + #TODO(electrical): Remove this when removing the fields variable + if @source + if @fields + logger.error("'fields' and 'source' are the same setting, but 'fields' is deprecated. Please use only 'source'") + end + @fields=Array.new if @fields.nil? + @fields << @source + end + end # def register def filter(event) @@ -98,6 +130,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base kv_keys=Hash.new + #TODO(electrical): Remove this loop when we remove the fields variable @fields.each do |fieldname| value = event[fieldname] @@ -111,10 +144,10 @@ class LogStash::Filters::KV < LogStash::Filters::Base end # If we have any keys, create/append the hash if kv_keys.length > 0 - if !event[@container].nil? - event[@container].merge!(kv_keys) + if !event[@target].nil? + event[@target].merge!(kv_keys) else - event[@container] = kv_keys + event[@target]= kv_keys end filter_matched(event) end diff --git a/spec/filters/kv.rb b/spec/filters/kv.rb index 4773e6eab..5c32e46a5 100644 --- a/spec/filters/kv.rb +++ b/spec/filters/kv.rb @@ -106,7 +106,7 @@ describe LogStash::Filters::KV do end - describe "test container" do + describe "test container (deprecated test)" do config <<-CONFIG filter { kv { container => 'kv' } @@ -124,7 +124,7 @@ describe LogStash::Filters::KV do end - describe "test empty container" do + describe "test empty container (deprecated test)" do config <<-CONFIG filter { kv { container => 'kv' } @@ -215,4 +215,94 @@ describe LogStash::Filters::KV do end end end + + #New tests + describe "test target" do + config <<-CONFIG + filter { + kv { target => 'kv' } + } + CONFIG + + sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do + insist { subject["kv"]["hello"] } == "world" + insist { subject["kv"]["foo"] } == "bar" + insist { subject["kv"]["baz"] } == "fizz" + insist { subject["kv"]["doublequoted"] } == "hello world" + insist { subject["kv"]["singlequoted"] } == "hello world" + insist {subject['@fields']["kv"].count } == 5 + end + + end + + describe "test empty target" do + config <<-CONFIG + filter { + kv { target => 'kv' } + } + CONFIG + + sample "hello:world:foo:bar:baz:fizz" do + insist { subject["kv"] } == nil + insist {subject['@fields'].count } == 0 + end + end + + + describe "test data from specific sub source" do + config <<-CONFIG + filter { + kv { + source => "data" + } + } + CONFIG + sample({"@fields" => {"data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}}) do + insist { subject["hello"] } == "world" + insist { subject["foo"] } == "bar" + insist { subject["baz"] } == "fizz" + insist { subject["doublequoted"] } == "hello world" + insist { subject["singlequoted"] } == "hello world" + insist { subject['@fields'].count } == 6 + end + end + + describe "test data from specific top source" do + config <<-CONFIG + filter { + kv { + source => "@data" + } + } + CONFIG + sample({"@data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}) do + insist { subject["hello"] } == "world" + insist { subject["foo"] } == "bar" + insist { subject["baz"] } == "fizz" + insist { subject["doublequoted"] } == "hello world" + insist { subject["singlequoted"] } == "hello world" + insist { subject['@fields'].count } == 5 + end + end + + + describe "test data from specific sub source and target" do + config <<-CONFIG + filter { + kv { + source => "data" + target => "kv" + } + } + CONFIG + sample({"@fields" => {"data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}}) do + insist { subject["kv"]["hello"] } == "world" + insist { subject["kv"]["foo"] } == "bar" + insist { subject["kv"]["baz"] } == "fizz" + insist { subject["kv"]["doublequoted"] } == "hello world" + insist { subject["kv"]["singlequoted"] } == "hello world" + insist { subject['@fields']["kv"].count } == 5 + end + end + end