mirror of
https://github.com/elastic/logstash.git
synced 2025-04-24 14:47:19 -04:00
- try to fix prune specs, but I think there are actually bugs found by
the tests that need fixing now.
This commit is contained in:
Jordan Sissel
parent
4c00fc3e32
commit
e58845f34d
2 changed files with 48 additions and 44 deletions
|
|
@ -8,7 +8,8 @@ class LogStash::Filters::Prune < LogStash::Filters::Base
|
|||
config_name "prune"
|
||||
plugin_status "experimental"
|
||||
|
||||
# Trigger whether configation fields and values should be interpolated for dynamic values.
|
||||
# Trigger whether configation fields and values should be interpolated for
|
||||
# dynamic values.
|
||||
# Probably adds some performance overhead. Defaults to false.
|
||||
config :interpolate, :validate => :boolean, :default => false
|
||||
|
||||
|
|
@ -78,21 +79,24 @@ class LogStash::Filters::Prune < LogStash::Filters::Base
|
|||
def filter(event)
|
||||
return unless filter?(event)
|
||||
|
||||
# We need to collect fields which needs to be remove ,and only in the end actually remove it
|
||||
# since then interpolation mode you can get unexpected results as fields with dynamic values will not match
|
||||
# since the fields to which they refer have already been removed.
|
||||
hash = event.to_hash
|
||||
|
||||
# We need to collect fields which needs to be remove ,and only in the end
|
||||
# actually remove it since then interpolation mode you can get unexpected
|
||||
# results as fields with dynamic values will not match since the fields to
|
||||
# which they refer have already been removed.
|
||||
fields_to_remove = []
|
||||
|
||||
unless @whitelist_names.empty?
|
||||
@whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate
|
||||
event.fields.each_key do |field|
|
||||
hash.each_key do |field|
|
||||
fields_to_remove << field unless field.match(@whitelist_names_regexp)
|
||||
end
|
||||
end
|
||||
|
||||
unless @blacklist_names.empty?
|
||||
@blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate
|
||||
event.fields.each_key do |field|
|
||||
hash.each_key do |field|
|
||||
fields_to_remove << field if field.match(@blacklist_names_regexp)
|
||||
end
|
||||
end
|
||||
|
|
@ -102,14 +106,14 @@ class LogStash::Filters::Prune < LogStash::Filters::Base
|
|||
key = event.sprintf(key)
|
||||
value = Regexp.new(event.sprintf(value))
|
||||
end
|
||||
if event.fields[key]
|
||||
if event.fields[key].is_a?(Array)
|
||||
subvalues_to_remove = event.fields[key].find_all{|x| not x.match(value)}
|
||||
if hash[key]
|
||||
if hash[key].is_a?(Array)
|
||||
subvalues_to_remove = hash[key].find_all{|x| not x.match(value)}
|
||||
unless subvalues_to_remove.empty?
|
||||
fields_to_remove << (subvalues_to_remove.length == event.fields[key].length ? key : { :key => key, :values => subvalues_to_remove })
|
||||
fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove })
|
||||
end
|
||||
else
|
||||
fields_to_remove << key if not event.fields[key].match(value)
|
||||
fields_to_remove << key if not hash[key].match(value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -119,23 +123,23 @@ class LogStash::Filters::Prune < LogStash::Filters::Base
|
|||
key = event.sprintf(key)
|
||||
value = Regexp.new(event.sprintf(value))
|
||||
end
|
||||
if event.fields[key]
|
||||
if event.fields[key].is_a?(Array)
|
||||
subvalues_to_remove = event.fields[key].find_all{|x| x.match(value)}
|
||||
if hash[key]
|
||||
if hash[key].is_a?(Array)
|
||||
subvalues_to_remove = hash[key].find_all{|x| x.match(value)}
|
||||
unless subvalues_to_remove.empty?
|
||||
fields_to_remove << (subvalues_to_remove.length == event.fields[key].length ? key : { :key => key, :values => subvalues_to_remove })
|
||||
fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove })
|
||||
end
|
||||
else
|
||||
fields_to_remove << key if event.fields[key].match(value)
|
||||
fields_to_remove << key if hash[key].match(value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
fields_to_remove.each do |field|
|
||||
if field.is_a?(Hash)
|
||||
event.fields[field[:key]] = event.fields[field[:key]] - field[:values]
|
||||
hash[field[:key]] = hash[field[:key]] - field[:values]
|
||||
else
|
||||
event.remove(field)
|
||||
hash.delete(field)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -22,7 +22,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == "Borat"
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == "Borat Sagdiyev"
|
||||
|
|
@ -45,7 +45,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -55,7 +55,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == "Borat"
|
||||
insist { subject["lastname"] } == nil
|
||||
insist { subject["fullname"] } == nil
|
||||
|
|
@ -79,7 +79,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -89,7 +89,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == "Borat"
|
||||
insist { subject["lastname"] } == nil
|
||||
insist { subject["fullname"] } == nil
|
||||
|
|
@ -112,7 +112,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -122,7 +122,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == nil
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == "Borat Sagdiyev"
|
||||
|
|
@ -146,7 +146,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -156,7 +156,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == nil
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == "Borat Sagdiyev"
|
||||
|
|
@ -183,7 +183,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -193,7 +193,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == "Borat"
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == nil
|
||||
|
|
@ -221,7 +221,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -231,7 +231,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == "Borat"
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == "Borat Sagdiyev"
|
||||
|
|
@ -258,7 +258,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -268,7 +268,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == nil
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == "Borat Sagdiyev"
|
||||
|
|
@ -296,7 +296,7 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"firstname" => "Borat",
|
||||
"lastname" => "Sagdiyev",
|
||||
"fullname" => "Borat Sagdiyev",
|
||||
|
|
@ -306,7 +306,7 @@ describe LogStash::Filters::Prune do
|
|||
"status" => "200",
|
||||
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
||||
"%{hmm}" => "doh"
|
||||
} do
|
||||
) do
|
||||
insist { subject["firstname"] } == nil
|
||||
insist { subject["lastname"] } == "Sagdiyev"
|
||||
insist { subject["fullname"] } == nil
|
||||
|
|
@ -331,12 +331,12 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"blah" => "foo",
|
||||
"xxx" => [ "1 2 3", "3 4 5" ],
|
||||
"status" => [ "100", "200", "300", "400", "500" ],
|
||||
"error" => [ "This is foolish" , "Need smthing smart too" ]
|
||||
} do
|
||||
) do
|
||||
insist { subject["blah"] } == "foo"
|
||||
insist { subject["error"] } == nil
|
||||
insist { subject["xxx"] } == [ "1 2 3", "3 4 5" ]
|
||||
|
|
@ -356,12 +356,12 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"blah" => "foo",
|
||||
"xxx" => [ "1 2 3", "3 4 5" ],
|
||||
"status" => [ "100", "200", "300", "400", "500" ],
|
||||
"error" => [ "This is foolish", "Need smthing smart too" ]
|
||||
} do
|
||||
) do
|
||||
insist { subject["blah"] } == "foo"
|
||||
insist { subject["error"] } == [ "This is foolish", "Need smthing smart too" ]
|
||||
insist { subject["xxx"] } == nil
|
||||
|
|
@ -382,12 +382,12 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"blah" => "foo",
|
||||
"xxx" => [ "1 2 3", "3 4 5" ],
|
||||
"status" => [ "100", "200", "300", "400", "500" ],
|
||||
"error" => [ "This is foolish" , "Need smthing smart too" ]
|
||||
} do
|
||||
) do
|
||||
insist { subject["blah"] } == "foo"
|
||||
insist { subject["error"] } == [ "This is foolish" ]
|
||||
insist { subject["xxx"] } == [ "1 2 3", "3 4 5" ]
|
||||
|
|
@ -408,12 +408,12 @@ describe LogStash::Filters::Prune do
|
|||
}
|
||||
CONFIG
|
||||
|
||||
sample "@fields" => {
|
||||
sample(
|
||||
"blah" => "foo",
|
||||
"xxx" => [ "1 2 3", "3 4 5" ],
|
||||
"status" => [ "100", "200", "300", "400", "500" ],
|
||||
"error" => [ "This is foolish" , "Need smthing smart too" ]
|
||||
} do
|
||||
) do
|
||||
insist { subject["blah"] } == "foo"
|
||||
insist { subject["error"] } == [ "Need smthing smart too" ]
|
||||
insist { subject["xxx"] } == nil
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue