main message codec support and headers transcoding

lib/logstash/inputs/imap.rb

@@ -13,6 +13,8 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   milestone 1
   ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
 
+  default :codec, "plain"
+
   config :host, :validate => :string, :required => true
   config :port, :validate => :number
 
@@ -91,7 +93,8 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
       message = part.decoded
     end
 
-    event = LogStash::Event.new("message" => message)
+    @codec.decode(message) do |event|
+      # event = LogStash::Event.new("message" => message)
 
       # Use the 'Date' field as the timestamp
       event["@timestamp"] = mail.date.to_time.gmtime
@@ -109,7 +112,7 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
         # Details at:
         #   https://github.com/mikel/mail/blob/master/README.md#encodings
         #   http://tools.ietf.org/html/rfc2047#section-2
-      value = header.decoded
+        value = transcode_to_utf8(header.decoded)
 
         # Assume we already processed the 'date' above.
         next if name == "Date"
@@ -124,8 +127,8 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
       end # mail.header_fields.each
 
       decorate(event)
-
-    return event
+      event
+    end
   end # def handle
 
   public
@@ -133,4 +136,13 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
     $stdin.close
     finished
   end # def teardown
+
+  private
+
+  # transcode_to_utf8 is meant for headers transcoding.
+  # the mail gem will set the correct encoding on header strings decoding
+  # and we want to transcode it to utf8
+  def transcode_to_utf8(s)
+    s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
+  end
 end # class LogStash::Inputs::IMAP