diff --git a/lib/logstash/event_v1.rb b/lib/logstash/event_v1.rb index c06a34822..17607f585 100644 --- a/lib/logstash/event_v1.rb +++ b/lib/logstash/event_v1.rb @@ -208,4 +208,10 @@ module LogStash::EventV1 def type=(value); self["type"] = value; end def type; return self["type"]; end def fields; return self.to_hash; end + + def tag(value) + # Generalize this method for more usability + self["tags"] ||= [] + self["tags"] << value unless self["tags"].include?(value) + end end # module LogStash::EventV1 diff --git a/lib/logstash/filters/json.rb b/lib/logstash/filters/json.rb index 2bb064479..77c42040b 100644 --- a/lib/logstash/filters/json.rb +++ b/lib/logstash/filters/json.rb @@ -23,9 +23,10 @@ class LogStash::Filters::Json < LogStash::Filters::Base # The above would parse the xml from the @message field config :source, :validate => :string, :required => true - # Define target for placing the data + # Define target for placing the data. If this setting is omitted, + # the json data will be stored at the root of the event. # - # for example if you want the data to be put in the 'doc' field: + # For example if you want the data to be put in the 'doc' field: # # filter { # json { @@ -35,14 +36,13 @@ class LogStash::Filters::Json < LogStash::Filters::Base # # json in the value of the source field will be expanded into a # datastructure in the "target" field. - # Note: if the "target" field already exists, it will be overridden - config :target, :validate => :string, :required => true + # + # Note: if the "target" field already exists, it will be overwritten. + config :target, :validate => :string public def register - # Nothing to do here - end # def register public @@ -51,29 +51,26 @@ class LogStash::Filters::Json < LogStash::Filters::Base @logger.debug("Running json filter", :event => event) - matches = 0 + return unless event[@source] - key = @source - dest = @target - - return unless event[key] - if event[key].is_a?(String) - event[key] = [event[key]] + if @target.nil? + # Default is to write to the root of the event. + dest = event.to_hash + else + dest = event[@target] ||= {} end - if event[key].length > 1 - @logger.warn("JSON filter only works on single fields (not lists)", - :key => key, :value => event[key]) - return - end - - raw = event[key].first + raw = event[@source] begin - event[dest] = JSON.parse(raw) + # TODO(sissel): Note, this will not successfully handle json lists + # like your text is '[ 1,2,3 ]' JSON.parse gives you an array (correctly) + # which won't merge into a hash. If someone needs this, we can fix it + # later. + dest.merge!(JSON.parse(raw)) filter_matched(event) rescue => e - event.tags << "_jsonparsefailure" - @logger.warn("Trouble parsing json", :key => key, :raw => raw, + event.tag("_jsonparsefailure") + @logger.warn("Trouble parsing json", :source => @source, :raw => raw, :exception => e) return end diff --git a/spec/filters/json.rb b/spec/filters/json.rb index 5f419900a..d83e09d8b 100644 --- a/spec/filters/json.rb +++ b/spec/filters/json.rb @@ -4,73 +4,12 @@ require "logstash/filters/json" describe LogStash::Filters::Json do extend LogStash::RSpec - describe "parse @message into @fields ( deprecated check )" do + describe "parse message into the event" do config <<-CONFIG filter { json { # Parse @message as JSON, store the results in the 'data' field' - "@message" => "@fields" - } - } - CONFIG - - sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do - insist { subject["hello"] } == "world" - insist { subject["list" ] } == [1,2,3] - insist { subject["hash"] } == { "k" => "v" } - - insist { subject["list.0" ] } == 1 - insist { subject["hash.k"] } == "v" - end - end - - describe "parse @message into a target field ( deprecated check )" do - config <<-CONFIG - filter { - json { - # Parse @message as JSON, store the results in the 'data' field' - "@message" => "data" - } - } - CONFIG - - sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do - insist { subject["data"]["hello"] } == "world" - insist { subject["data"]["list"] } == [1,2,3] - insist { subject["data"]["hash"] } == { "k" => "v" } - - insist { subject["data.hello"] } == "world" - insist { subject["data.list" ] } == [1,2,3] - insist { subject["data.list.0" ] } == 1 - insist { subject["data.hash"] } == { "k" => "v" } - insist { subject["data.hash.k"] } == "v" - end - end - - describe "tag invalid json ( deprecated check )" do - config <<-CONFIG - filter { - json { - # Parse @message as JSON, store the results in the 'data' field' - "@message" => "data" - } - } - CONFIG - - sample "invalid json" do - insist { subject.tags }.include?("_jsonparsefailure") - end - end - - ## New tests - - describe "parse @message into @fields" do - config <<-CONFIG - filter { - json { - # Parse @message as JSON, store the results in the 'data' field' - source => "@message" - target => "@fields" + source => "message" } } CONFIG @@ -86,14 +25,15 @@ describe LogStash::Filters::Json do config <<-CONFIG filter { json { - # Parse @message as JSON, store the results in the 'data' field' - source => "@message" + # Parse message as JSON, store the results in the 'data' field' + source => "message" target => "data" } } CONFIG sample '{ "hello": "world", "list": [ 1, 2, 3 ], "hash": { "k": "v" } }' do + puts subject.to_json insist { subject["data"]["hello"] } == "world" insist { subject["data"]["list" ] } == [1,2,3] insist { subject["data"]["hash"] } == { "k" => "v" } @@ -104,8 +44,8 @@ describe LogStash::Filters::Json do config <<-CONFIG filter { json { - # Parse @message as JSON, store the results in the 'data' field' - source => "@message" + # Parse message as JSON, store the results in the 'data' field' + source => "message" target => "data" } }