filter/date fix locale config

Closes #1664

lib/logstash/filters/date.rb

@@ -38,11 +38,12 @@ class LogStash::Filters::Date < LogStash::Filters::Base
   # For example, `America/Los_Angeles` or `Europe/France` are valid IDs.
   config :timezone, :validate => :string
 
-  # Specify a locale to be used for date parsing. If this is not specified, the
-  # platform default will be used.
+  # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
+  # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
+  # If not specified, the platform default will be used.
   #
-  # The locale is mostly necessary to be set for parsing month names and
-  # weekday names.
+  # The locale is mostly necessary to be set for parsing month names (pattern with MMM) and
+  # weekday names (pattern with EEE).
   #
   config :locale, :validate => :string
 
@@ -98,16 +99,6 @@ class LogStash::Filters::Date < LogStash::Filters::Base
     @parsers = Hash.new { |h,k| h[k] = [] }
   end # def initialize
 
-  private
-  def parseLocale(localeString)
-    return nil if localeString == nil
-    matches = localeString.match(/(?<lang>.+?)(?:_(?<country>.+?))?(?:_(?<variant>.+))?/)
-    lang = matches['lang'] == nil ? "" : matches['lang'].strip()
-    country = matches['country'] == nil ? "" : matches['country'].strip()
-    variant = matches['variant'] == nil ? "" : matches['variant'].strip()
-    return lang.length > 0 ? java.util.Locale.new(lang, country, variant) : nil
-  end
-
   public
   def register
     require "java"
@@ -116,8 +107,15 @@ class LogStash::Filters::Date < LogStash::Filters::Base
         :plugin => "filter", :type => "date",
         :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
     end
-    # TODO(sissel): Need a way of capturing regexp configs better.
-    locale = parseLocale(@config["locale"][0]) if @config["locale"] != nil and @config["locale"][0] != nil
+
+    locale = nil
+    if @locale
+      if @locale.include? '_'
+        @logger.warn("Date filter now use BCP47 format for locale, replacing underscore with dash")
+        @locale.gsub!('_','-')
+      end
+      locale = java.util.Locale.forLanguageTag(@locale)
+    end
     setupMatcher(@config["match"].shift, locale, @config["match"] )
   end
 

spec/filters/date.rb

@@ -356,4 +356,52 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
       insist { subject["@timestamp"].year } == Time.now.year
     end
   end
+
+  describe "Supporting locale only" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "dd MMMM yyyy" ]
+          locale => "fr"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "14 juillet 1789" do
+      insist { subject["@timestamp"].time } == Time.iso8601("1789-07-14T00:00:00.000Z").utc
+    end
+  end
+
+  describe "Supporting locale+country in BCP47" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "dd MMMM yyyy" ]
+          locale => "fr-FR"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "14 juillet 1789" do
+      insist { subject["@timestamp"].time } == Time.iso8601("1789-07-14T00:00:00.000Z").utc
+    end
+  end
+
+  describe "Supporting locale+country in POSIX (internally replace '_' by '-')" do
+    config <<-CONFIG
+      filter {
+        date {
+          match => [ "message", "dd MMMM yyyy" ]
+          locale => "fr_FR"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    sample "14 juillet 1789" do
+      insist { subject["@timestamp"].time } == Time.iso8601("1789-07-14T00:00:00.000Z").utc
+    end
+  end
 end