Allow configuration of whether the grok filter tags events on failure

This commit is contained in:
Neil Prosser 2013-01-22 16:48:58 +00:00
parent 4d60fa24c4
commit 44ef1e5beb
2 changed files with 25 additions and 2 deletions

View file

@ -202,6 +202,10 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
# containing that one value. # containing that one value.
config :singles, :validate => :boolean, :default => false config :singles, :validate => :boolean, :default => false
# If true, ensure the '_grokparsefailure' tag is present when there has been no
# successful match
config :tag_on_failure, :validate => :boolean, :default => true
# TODO(sissel): Add this feature? # TODO(sissel): Add this feature?
# When disabled, any pattern that matches the entire string will not be set. # When disabled, any pattern that matches the entire string will not be set.
# This is useful if you have named patterns like COMBINEDAPACHELOG that will # This is useful if you have named patterns like COMBINEDAPACHELOG that will
@ -276,7 +280,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
# Skip known config names # Skip known config names
next if (RESERVED + ["match", "patterns_dir", next if (RESERVED + ["match", "patterns_dir",
"drop_if_match", "named_captures_only", "pattern", "drop_if_match", "named_captures_only", "pattern",
"keep_empty_captures", "break_on_match", "singles"]).include?(field) "keep_empty_captures", "break_on_match", "singles", "tag_on_failure"]).include?(field)
patterns = [patterns] if patterns.is_a?(String) patterns = [patterns] if patterns.is_a?(String)
if !@patterns.include?(field) if !@patterns.include?(field)
@ -388,7 +392,7 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
end # event[field] end # event[field]
end # patterns.each end # patterns.each
if !matched if !matched && @tag_on_failure
# Tag this event if we can't parse it. We can use this later to # Tag this event if we can't parse it. We can use this later to
# reparse+reindex logs if we improve the patterns given . # reparse+reindex logs if we improve the patterns given .
event.tags << "_grokparsefailure" unless event.tags.include?("_grokparsefailure") event.tags << "_grokparsefailure" unless event.tags.include?("_grokparsefailure")

View file

@ -235,4 +235,23 @@ describe LogStash::Filters::Grok do
insist { subject.tags }.include?("one_point_oh") insist { subject.tags }.include?("one_point_oh")
end end
end end
describe "tagging on failure" do
config <<-CONFIG
filter {
grok {
pattern => "matchme %{NUMBER:fancy}"
tag_on_failure => false
}
}
CONFIG
sample "matchme 1234" do
reject { subject["@tags"] }.include?("_grokparsefailure")
end
sample "this will not be matched" do
reject { subject["@tags"] }.include?("_grokparsefailure")
end
end
end end