Merge pull request #710 from logstash/json-codec-fix

Fix json codec

lib/logstash/codecs/json.rb

@@ -1,4 +1,5 @@
 require "logstash/codecs/base"
+require "logstash/codecs/line"
 require "json"
 
 # This codec will encode and decode JSON.
@@ -18,21 +19,23 @@ class LogStash::Codecs::JSON < LogStash::Codecs::Base
   # For nxlog users, you'll want to set this to "CP1252"
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
+  public
+  def initialize(params={})
+    super(params)
+    @lines = LogStash::Codecs::Line.new
+    @lines.charset = @charset
+  end
+  
   public
   def decode(data)
-    data.force_encoding(@charset)
-    if @charset != "UTF-8"
-      # The user has declared the character encoding of this data is
-      # something other than UTF-8. Let's convert it (as cleanly as possible)
-      # into UTF-8 so we can use it with JSON, etc.
-      data = data.encode("UTF-8", :invalid => :replace, :undef => :replace)
-    end
 
-    begin
-      yield LogStash::Event.new(JSON.parse(data))
-    rescue JSON::ParserError => e
-      @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
-      yield LogStash::Event.new("message" => data)
+    @lines.decode(data) do |event|
+      begin
+        yield LogStash::Event.new(JSON.parse(event["message"]))
+      rescue JSON::ParserError => e
+        @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
+        yield LogStash::Event.new("message" => data)
+      end
     end
   end # def decode
 

spec/codecs/json.rb

@@ -9,8 +9,21 @@ describe LogStash::Codecs::JSON do
 
   context "#decode" do
     it "should return an event from json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(data.to_json+"\n") do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event["foo"] } == data["foo"]
+        insist { event["baz"] } == data["baz"]
+        insist { event["bah"] } == data["bah"]
+      end
+    end
+    
+    it "should return an event from json data when a newline is recieved" do
       data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
       subject.decode(data.to_json) do |event|
+        insist {false}
+      end
+      subject.decode("\n") do |event|
         insist { event.is_a? LogStash::Event }
         insist { event["foo"] } == data["foo"]
         insist { event["baz"] } == data["baz"]