Merge pull request #432 from piavlo/feature/base-filter-changes

enhanced base filter logic
This commit is contained in:
Jordan Sissel 2013-05-28 23:51:06 -07:00
commit deaee4b272
6 changed files with 385 additions and 71 deletions

View file

@ -8,21 +8,40 @@ class LogStash::Filters::Base < LogStash::Plugin
config_name "filter"
# Note that all of the specified routing options (type,tags.exclude_tags,include_fields,exclude_fields)
# must be met in order for the event to be handled by the filter.
# The type to act on. If a type is given, then this filter will only
# act on messages with the same type. See any input plugin's "type"
# attribute for more.
# Optional.
config :type, :validate => :string, :default => ""
# Only handle events with all of these tags. Note that if you specify
# a type, the event must also match that type.
# Only handle events with all/any (controlled by include_any config option) of these tags.
# Optional.
# TODO(piavlo): sould we rename/alias this to include_tags for clearness and consistency?
config :tags, :validate => :array, :default => []
# Only handle events without any of these tags. Note this check is
# additional to type and tags.
# Only handle events without all/any (controlled by exclude_any config option) of these tags.
# Optional.
config :exclude_tags, :validate => :array, :default => []
# Only handle events with all/any (controlled by include_any config option) of these fields.
# Optional.
config :include_fields, :validate => :array, :default => []
# Only handle events without all/any (controlled by exclude_any config option) of these fields.
# Optional.
config :exclude_fields, :validate => :array, :default => []
# Should all or any of the specified tags/include_fields be present for event to
# be handled. Defaults to all.
config :include_any, :validate => :boolean, :default => false
# Should all or any of the specified exclude_tags/exclude_fields be missing for event to
# be handled. Defaults to all.
config :exclude_any, :validate => :boolean, :default => true
# If this filter is successful, add arbitrary tags to the event.
# Tags can be dynamic and include parts of the event using the %{field}
# syntax. Example:
@ -52,20 +71,36 @@ class LogStash::Filters::Base < LogStash::Plugin
config :remove_tag, :validate => :array, :default => []
# If this filter is successful, add any arbitrary fields to this event.
# Tags can be dynamic and include parts of the event using the %{field}
# Example:
#
# filter {
# %PLUGIN% {
# add_field => [ "sample", "Hello world, from %{@source}" ]
# add_field => [ "foo_%{somefield}", "Hello world, from %{@source}" ]
# }
# }
#
# On success, the %PLUGIN% plugin will then add field 'sample' with the
# value above and the %{@source} piece replaced with that value from the
# event.
# If the event has field "somefield" == "hello" this filter, on success,
# would add field "foo_hello" if it is present, with the
# value above and the %{@source} piece replaced with that value from the
# event.
config :add_field, :validate => :hash, :default => {}
RESERVED = ["type", "tags", "add_tag", "remove_tag", "add_field", "exclude_tags"]
# If this filter is successful, remove arbitrary fields from this event.
# Fields names can be dynamic and include parts of the event using the %{field}
# Example:
#
# filter {
# %PLUGIN% {
# remove_field => [ "foo_%{somefield}" ]
# }
# }
#
# If the event has field "somefield" == "hello" this filter, on success,
# would remove the field with name "foo_hello" if it is present
config :remove_field, :validate => :array, :default => []
RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field", "include_any", "exclude_any"]
public
def initialize(params)
@ -98,33 +133,38 @@ class LogStash::Filters::Base < LogStash::Plugin
# matches the filter's conditions (right type, etc)
protected
def filter_matched(event)
(@add_field or {}).each do |field, value|
event[field] ||= []
if value.is_a?(Array)
event[field] += value
else
@add_field.each do |field, value|
field = event.sprintf(field)
value = event.sprintf(value)
if event.include?(field)
event[field] = [event[field]] if !event[field].is_a?(Array)
event[field] << event.sprintf(value)
event[field] << value
else
event[field] = value
end
@logger.debug? and @logger.debug("filters/#{self.class.name}: adding " \
"value to field", :field => field,
:value => value)
@logger.debug? and @logger.debug("filters/#{self.class.name}: adding value to field",
:field => field, :value => value)
end
@remove_field.each do |field|
field = event.sprintf(field)
@logger.debug? and @logger.debug("filters/#{self.class.name}: removing field",
:field => field)
event.remove(field)
end
(@add_tag or []).each do |tag|
@add_tag.each do |tag|
tag = event.sprintf(tag)
@logger.debug? and @logger.debug("filters/#{self.class.name}: adding tag",
:tag => tag)
event.tags << event.sprintf(tag)
#event.tags |= [ event.sprintf(tag) ]
event.tags << tag
end
if @remove_tag
remove_tags = @remove_tag.map do |tag|
event.sprintf(tag)
end
@logger.debug? and @logger.debug("filters/#{self.class.name}: removing tags",
:tags => (event.tags & remove_tags))
event.tags -= remove_tags
@remove_tag.each do |tag|
tag = event.sprintf(tag)
@logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag",
:tag => tag)
event.tags.delete(tag)
end
end # def filter_matched
@ -132,21 +172,39 @@ class LogStash::Filters::Base < LogStash::Plugin
def filter?(event)
if !@type.empty?
if event.type != @type
@logger.debug(["Skipping event because type doesn't match #{@type}", event])
@logger.debug? and @logger.debug(["Skipping event because type doesn't match #{@type}", event])
return false
end
end
# TODO(piavlo): It would much nicer to set this in the "raising" register method somehow?
include_method = @include_any ? :any? : :all?
exclude_method = @exclude_any ? :any? : :all?
if !@tags.empty?
if (event.tags & @tags).size != @tags.size
@logger.debug(["Skipping event because tags don't match #{@tags.inspect}", event])
if !@tags.send(include_method) {|tag| event.tags.include?(tag)}
@logger.debug? and @logger.debug(["Skipping event because tags don't match #{@tags.inspect}", event])
return false
end
end
if !@exclude_tags.empty?
if (diff_tags = (event.tags & @exclude_tags)).size != 0
@logger.debug(["Skipping event because tags contains excluded tags: #{diff_tags.inspect}", event])
if @exclude_tags.send(exclude_method) {|tag| event.tags.include?(tag)}
@logger.debug? and @logger.debug(["Skipping event because tags contains excluded tags: #{exclude_tags.inspect}", event])
return false
end
end
if !@include_fields.empty?
if !@include_fields.send(include_method) {|field| event.include?(field)}
@logger.debug? and @logger.debug(["Skipping event because fields don't match #{@include_fields.inspect}", event])
return false
end
end
if !@exclude_fields.empty?
if @exclude_fields.send(exclude_method) {|field| event.include?(field)}
@logger.debug? and @logger.debug(["Skipping event because fields contain excluded fields #{@exclude_fields.inspect}", event])
return false
end
end

View file

@ -31,7 +31,9 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
# remove => [ "client" ] # Removes the 'client' field
# }
# }
config :remove, :validate => :array
#
# This option is deprecated, instead use remove_field option available in all filters.
config :remove, :validate => :array, :deprecated => true
# Replace a field with a new value. The new value can include %{foo} strings
# to help you build a new value from other parts of the event.

View file

@ -121,7 +121,7 @@ describe LogStash::Filters::Grep do
sample ({"@fields" => {"str" => "test"}}) do
reject { subject }.nil?
insist { subject["new_field"]} == ["new_value"]
insist { subject["new_field"]} == "new_value"
end
end
@ -137,7 +137,7 @@ describe LogStash::Filters::Grep do
sample ({"@type" => "grepper", "@fields" => {"str" => "test"}}) do
reject { subject }.nil?
insist { subject["new_field"]} == [subject.type]
insist { subject["new_field"]} == subject.type
end
end

View file

@ -132,7 +132,7 @@ describe LogStash::Filters::Grok do
sample "matchme 1234" do
reject { subject["@tags"] }.include?("_grokparsefailure")
insist { subject["new_field"] } == ["1234"]
insist { subject["new_field"] } == "1234"
end
sample "this will not be matched" do

View file

@ -167,7 +167,7 @@ describe LogStash::Filters::KV do
sample "hello=world" do
insist { subject["hello"] } == "world"
insist { subject["whoa"] } == [ "fancypants" ]
insist { subject["whoa"] } == "fancypants"
end
end
@ -179,7 +179,7 @@ describe LogStash::Filters::KV do
CONFIG
sample "this is not key value" do
reject { subject["whoa"] } == [ "fancypants" ]
reject { subject["whoa"] } == "fancypants"
end
end
end

View file

@ -30,11 +30,11 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop"}) do
sample({"@type" => "noop"}) do
insist { subject.tags } == ["test"]
end
sample({"@type" => "not_noop"}) do
sample({"@type" => "not_noop"}) do
insist { subject.tags } == []
end
end
@ -50,12 +50,12 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop"}) do
insist { subject.tags} == []
sample({"@type" => "noop"}) do
insist { subject.tags } == []
end
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags} == ["t1", "t2", "test"]
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags } == ["t1", "t2", "test"]
end
end
@ -70,20 +70,20 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop"}) do
insist { subject.tags} == []
sample({"@type" => "noop"}) do
insist { subject.tags } == []
end
sample({"@type" => "noop", "@tags" => ["t1"]}) do
insist { subject.tags} == ["t1"]
sample({"@type" => "noop", "@tags" => ["t1"]}) do
insist { subject.tags } == ["t1"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags} == ["t1", "t2", "test"]
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags } == ["t1", "t2", "test"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t3"]}) do
insist { subject.tags} == ["t1", "t2", "t3", "test"]
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t3"]}) do
insist { subject.tags } == ["t1", "t2", "t3", "test"]
end
end
@ -99,16 +99,16 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop"}) do
insist { subject.tags} == []
sample({"@type" => "noop"}) do
insist { subject.tags } == []
end
sample({"@type" => "noop", "@tags" => ["t1"]}) do
insist { subject.tags} == ["t1", "test"]
sample({"@type" => "noop", "@tags" => ["t1"]}) do
insist { subject.tags } == ["t1", "test"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags} == ["t1", "t2"]
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags } == ["t1", "t2"]
end
end
@ -124,16 +124,16 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t4"]}) do
insist { subject.tags} == ["t1", "t2", "t4"]
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t4"]}) do
insist { subject.tags } == ["t1", "t2", "t4"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t3", "t4"]}) do
insist { subject.tags} == ["t1", "t3", "t4"]
sample({"@type" => "noop", "@tags" => ["t1", "t3", "t4"]}) do
insist { subject.tags } == ["t1", "t3", "t4"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t4", "t5"]}) do
insist { subject.tags} == ["t1", "t4", "t5", "test"]
sample({"@type" => "noop", "@tags" => ["t1", "t4", "t5"]}) do
insist { subject.tags } == ["t1", "t4", "t5", "test"]
end
end
@ -148,16 +148,270 @@ describe LogStash::Filters::NOOP do
}
CONFIG
sample({"@type" => "noop", "@tags" => ["t4"]}) do
insist { subject.tags} == ["t4"]
sample({"@type" => "noop", "@tags" => ["t4"]}) do
insist { subject.tags } == ["t4"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t3"]}) do
insist { subject.tags} == ["t1"]
sample({"@type" => "noop", "@tags" => ["t1", "t2", "t3"]}) do
insist { subject.tags } == ["t1"]
end
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags} == ["t1"]
sample({"@type" => "noop", "@tags" => ["t1", "t2"]}) do
insist { subject.tags } == ["t1"]
end
end
describe "remove_tag with dynamic value" do
config <<-CONFIG
filter {
noop {
type => "noop"
tags => ["t1"]
remove_tag => ["%{blackhole}"]
}
}
CONFIG
sample({"@type" => "noop", "@tags" => ["t1", "goaway", "t3"], "@fields" => {"blackhole" => "goaway"}}) do
insist { subject.tags } == ["t1", "t3"]
end
end
describe "remove_field" do
config <<-CONFIG
filter {
noop {
type => "noop"
remove_field => ["t2", "t3"]
}
}
CONFIG
sample({"@type" => "noop", "@fields" => {"t4" => "four"}}) do
insist { subject["@fields"] }.include?("t4")
end
sample({"@type" => "noop", "@fields" => {"t1" => "one", "t2" => "two", "t3" => "three"}}) do
insist { subject["@fields"] }.include?("t1")
reject { subject["@fields"] }.include?("t2")
reject { subject["@fields"] }.include?("t3")
end
sample({"@type" => "noop", "@fields" => {"t1" => "one", "t2" => "two"}}) do
insist { subject["@fields"] }.include?("t1")
reject { subject["@fields"] }.include?("t2")
end
end
describe "remove_field with dynamic value in field name" do
config <<-CONFIG
filter {
noop {
type => "noop"
remove_field => ["%{blackhole}"]
}
}
CONFIG
sample({"@type" => "noop", "@fields" => {"blackhole" => "go", "go" => "away"}}) do
insist { subject["@fields"] }.include?("blackhole")
reject { subject["@fields"] }.include?("go")
end
end
describe "checking AND include_any logic on tags filter" do
config <<-CONFIG
filter {
noop {
tags => ["two", "three", "four"]
include_any => false
add_tag => ["match"]
}
}
CONFIG
sample({"@tags" => ["one", "two", "three", "four", "five"]}) do
insist { subject.tags }.include?("match")
end
sample({"@tags" => ["one", "two", "four", "five"]}) do
reject { subject.tags }.include?("match")
end
sample({}) do
reject { subject.tags }.include?("match")
end
end
describe "checking OR include_any logic on tags filter" do
config <<-CONFIG
filter {
noop {
tags => ["two", "three", "four"]
include_any => true
add_tag => ["match"]
}
}
CONFIG
sample({"@tags" => ["one1", "two2", "three", "four4", "five5"]}) do
insist { subject.tags }.include?("match")
end
sample({"@tags" => ["one1", "two2", "three3", "four4", "five5"]}) do
reject { subject.tags }.include?("match")
end
sample({}) do
reject { subject.tags }.include?("match")
end
end
describe "checking AND include_any logic on include_fields filter" do
config <<-CONFIG
filter {
noop {
include_fields => ["two", "two", "three", "three", "four", "four"]
include_any => false
add_tag => ["match"]
}
}
CONFIG
sample({"@fields" => {"one" => "1", "two" => "2", "three" => "3", "four" => "4", "five" => "5"}}) do
insist { subject.tags }.include?("match")
end
sample({"@fields" => {"one" => "1", "two" => "2", "four" => "4", "five" => "5"}}) do
reject { subject.tags }.include?("match")
end
sample({}) do
reject { subject.tags }.include?("match")
end
end
describe "checking OR include_any logic on include_fields filter" do
config <<-CONFIG
filter {
noop {
include_fields => ["two", "two", "three", "three", "four", "four"]
include_any => true
add_tag => ["match"]
}
}
CONFIG
sample({"@fields" => {"one1" => "1", "two2" => "2", "three" => "3", "four4" => "4", "five5" => "5"}}) do
insist { subject.tags }.include?("match")
end
sample({"@fields" => {"one1" => "1", "two2" => "2", "three3" => "3", "four4" => "4", "five5" => "5"}}) do
reject { subject.tags }.include?("match")
end
sample({}) do
reject { subject.tags }.include?("match")
end
end
describe "checking AND exclude_any logic on exclude_tags filter" do
config <<-CONFIG
filter {
noop {
exclude_tags => ["two", "three", "four"]
exclude_any => false
add_tag => ["match"]
}
}
CONFIG
sample({"@tags" => ["one", "two", "three", "four", "five"]}) do
reject { subject.tags }.include?("match")
end
sample({"@tags" => ["one", "two", "four", "five"]}) do
insist { subject.tags }.include?("match")
end
sample({}) do
insist { subject.tags }.include?("match")
end
end
describe "checking OR exclude_any logic on exclude_tags filter" do
config <<-CONFIG
filter {
noop {
exclude_tags => ["two", "three", "four"]
exclude_any => true
add_tag => ["match"]
}
}
CONFIG
sample({"@tags" => ["one", "two", "three", "four", "five"]}) do
reject { subject.tags }.include?("match")
end
sample({"@tags" => ["one1", "two2", "three", "four4", "five5"]}) do
reject { subject.tags }.include?("match")
end
sample({"@tags" => ["one1", "two2", "three3", "four4", "five5"]}) do
insist { subject.tags }.include?("match")
end
sample({}) do
insist { subject.tags }.include?("match")
end
end
describe "checking AND exclude_any logic on exclude_fields filter" do
config <<-CONFIG
filter {
noop {
exclude_fields => ["two", "two", "three", "three", "four", "four"]
exclude_any => false
add_tag => ["match"]
}
}
CONFIG
sample({"@fields" => {"one" => "1", "two" => "2", "three" => "3", "four" => "4", "five" => "5"}}) do
reject { subject.tags }.include?("match")
end
sample({"@fields" => {"one" => "1", "two" => "2", "four" => "4", "five" => "5"}}) do
insist { subject.tags }.include?("match")
end
sample({}) do
insist { subject.tags }.include?("match")
end
end
describe "checking OR exclude_any logic on exclude_fields filter" do
config <<-CONFIG
filter {
noop {
exclude_fields => ["two", "two", "three", "three", "four", "four"]
exclude_any => true
add_tag => ["match"]
}
}
CONFIG
sample({"@fields" => {"one1" => "1", "two2" => "2", "three" => "3", "four4" => "4", "five5" => "5"}}) do
reject { subject.tags }.include?("match")
end
sample({"@fields" => {"one1" => "1", "two2" => "2", "three3" => "3", "four4" => "4", "five5" => "5"}}) do
insist { subject.tags }.include?("match")
end
sample({}) do
insist { subject.tags }.include?("match")
end
end
end