github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-25 07:07:54 -04:00
Code Issues Projects Releases Packages Wiki Activity

Support almost-ISO8601 patterns in date filter ISO8601 config

Browse source
This commit is contained in:
Wiibaa 2013-11-11 08:20:23 +01:00 • committed by Suyog Rao
parent dafb3f1478
commit 18049e138d
2 changed files with 41 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

47
lib/logstash/filters/date.rb
View file

@ -123,31 +123,42 @@ class LogStash::Filters::Date < LogStash::Filters::Base
def setupMatcher(field, locale, value) def setupMatcher(field, locale, value)
value.each do |format| value.each do |format|
parsers = []
case format case format
when "ISO8601" when "ISO8601"
joda_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser iso_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
if @timezone
iso_parser = iso_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
else
iso_parser = iso_parser.withOffsetParsed
end
parsers << lambda { |date| iso_parser.parseDateTime(date) }
#Fall back solution of almost ISO8601 date-time
almostISOparsers = [
org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").getParser(),
org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").getParser()
].to_java(org.joda.time.format.DateTimeParser)
joda_parser = org.joda.time.format.DateTimeFormatterBuilder.new.append( nil, almostISOparsers ).toFormatter()
if @timezone if @timezone
joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
else else
joda_parser = joda_parser.withOffsetParsed joda_parser = joda_parser.withOffsetParsed
end end
parser = lambda { |date| joda_parser.parseMillis(date) } parsers << lambda { |date| joda_parser.parseMillis(date) }
when "UNIX" # unix epoch when "UNIX" # unix epoch
parser = lambda do |date| parsers << lambda do |date|
raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric) raise "Invalid UNIX epoch value '#{date}'" unless /^\d+(?:\.\d+)?$/ === date || date.is_a?(Numeric)
(date.to_f * 1000).to_i (date.to_f * 1000).to_i
end end
when "UNIX_MS" # unix epoch in ms when "UNIX_MS" # unix epoch in ms
parser = lambda do |date| parsers << lambda do |date|
raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric) raise "Invalid UNIX epoch value '#{date}'" unless /^\d+$/ === date || date.is_a?(Numeric)
date.to_i date.to_i
end end
when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance) parsers << lambda do |date|
parser = lambda do |date|
# Skip leading "@" if it is present (common in tai64n times) # Skip leading "@" if it is present (common in tai64n times)
date = date[1..-1] if date[0, 1] == "@" date = date[1..-1] if date[0, 1] == "@"
#return joda_instant.call((date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)).to_java.toDateTime
return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000) return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
end end
else else
@ -160,13 +171,13 @@ class LogStash::Filters::Date < LogStash::Filters::Base
if (locale != nil) if (locale != nil)
joda_parser = joda_parser.withLocale(locale) joda_parser = joda_parser.withLocale(locale)
end end
parser = lambda { |date| joda_parser.parseMillis(date) } parsers << lambda { |date| joda_parser.parseMillis(date) }
end end
@logger.debug("Adding type with date config", :type => @type, @logger.debug("Adding type with date config", :type => @type,
:field => field, :format => format) :field => field, :format => format)
@parsers[field] << { @parsers[field] << {
:parser => parser, :parser => parsers,
:format => format :format => format
} }
end end
@ -192,14 +203,16 @@ class LogStash::Filters::Date < LogStash::Filters::Base
success = false success = false
last_exception = RuntimeError.new "Unknown" last_exception = RuntimeError.new "Unknown"
fieldparsers.each do |parserconfig| fieldparsers.each do |parserconfig|
parser = parserconfig[:parser] parserconfig[:parser].each do |parser|
begin begin
epochmillis = parser.call(value) epochmillis = parser.call(value)
success = true success = true
break # success break # success
rescue StandardError, JavaException => e rescue StandardError, JavaException => e
last_exception = e last_exception = e
end end
end # parserconfig[:parser].each
break if success
end # fieldparsers.each end # fieldparsers.each
raise last_exception unless success raise last_exception unless success

11
spec/filters/date.rb
View file

@ -27,6 +27,7 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
date { date {
match => [ "mydate", "ISO8601" ] match => [ "mydate", "ISO8601" ]
locale => "en" locale => "en"
timezone => "UTC"
} }
} }
CONFIG CONFIG
@ -48,6 +49,16 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
"2001-09-05T16:36:36.123+0700" => "2001-09-05T09:36:36.123Z", "2001-09-05T16:36:36.123+0700" => "2001-09-05T09:36:36.123Z",
"2001-11-06T20:45:45.123-0000" => "2001-11-06T20:45:45.123Z", "2001-11-06T20:45:45.123-0000" => "2001-11-06T20:45:45.123Z",
"2001-12-07T23:54:54.123Z" => "2001-12-07T23:54:54.123Z", "2001-12-07T23:54:54.123Z" => "2001-12-07T23:54:54.123Z",
#Almost ISO8601 support, with timezone
"2001-11-06 20:45:45.123-0000" => "2001-11-06T20:45:45.123Z",
"2001-12-07 23:54:54.123Z" => "2001-12-07T23:54:54.123Z",
#Almost ISO8601 support, without timezone
"2001-11-06 20:45:45.123" => "2001-11-06T20:45:45.123Z",
} }
times.each do |input, output| times.each do |input, output|