From 51a24e8de1990c9138f5b1cf0485782ac9c84aec Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Fri, 6 Sep 2013 20:19:02 -0700 Subject: [PATCH] - Add "not in" operator based on feedback from this logstash 1.2.0 writeup: http://tobrunet.ch/2013/09/logstash-1-2-0-upgrade-notes-included/ --- CHANGELOG | 2 +- docs/configuration.md | 18 +++- lib/logstash/config/config_ast.rb | 7 ++ lib/logstash/config/grammar.rb | 135 ++++++++++++++++++++++++++-- lib/logstash/config/grammar.treetop | 10 +++ spec/conditionals/test.rb | 23 +++++ 6 files changed, 186 insertions(+), 9 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3e57af826..6fccef7f5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,7 @@ - deprecation: Using deprecated plugin settings can now advise you on a corrective path to take. One example is the 'type' setting on filters and outputs will now advise you to use conditionals and give an example. - + - conditionals: The "not in" operator is now supported. ## inputs - feature: pipe: reopen the pipe and retry on any error. (#619, Jonathan Van diff --git a/docs/configuration.md b/docs/configuration.md index 35e1edbe6..9ff6ce7e9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -195,8 +195,8 @@ What's an expression? Comparison tests, boolean logic, etc! The following comparison operators are supported: * equality, etc: == != < > <= >= -* regexp: =~ !~ -* inclusion: in +* regexp: =~ !~ +* inclusion: in, not in The following boolean operators are supported: @@ -207,7 +207,8 @@ The following unary operators are supported: * ! Expressions may contain expressions. Expressions may be negated with `!`. -Expressions may be grouped with parentheses `(...)`. +Expressions may be grouped with parentheses `(...)`. Expressions can be long +and complex. For example, if we want to remove the field `secret` if the field `action` has a value of `login`: @@ -242,6 +243,17 @@ How about telling nagios of any http event that has a status code of 5xx? } } +You can also do multiple expressions in a single condition: + + output { + # Send production errors to pagerduty + if [loglevel] == "ERROR" and [deployment] == "production" { + pagerduty { + ... + } + } + } + ## Further Reading For more information, see [the plugin docs index](index) diff --git a/lib/logstash/config/config_ast.rb b/lib/logstash/config/config_ast.rb index 6acee525d..e959f3ad5 100644 --- a/lib/logstash/config/config_ast.rb +++ b/lib/logstash/config/config_ast.rb @@ -289,6 +289,13 @@ module LogStash; module Config; module AST end end + module NotInExpression + def compile + item, list = recursive_select(LogStash::Config::AST::RValue) + return "(x = #{list.compile}; x.respond_to?(:include?) && !x.include?(#{item.compile}))" + end + end + class MethodCall < Node def compile arguments = recursive_inject { |e| [String, Number, Selector, Array, MethodCall].any? { |c| e.is_a?(c) } } diff --git a/lib/logstash/config/grammar.rb b/lib/logstash/config/grammar.rb index 155ba5805..3a4605314 100644 --- a/lib/logstash/config/grammar.rb +++ b/lib/logstash/config/grammar.rb @@ -2448,23 +2448,29 @@ module LogStashConfig r0 = r8 r0.extend(LogStash::Config::AST::Expression) else - r9 = _nt_compare_expression + r9 = _nt_not_in_expression if r9 r0 = r9 r0.extend(LogStash::Config::AST::Expression) else - r10 = _nt_regexp_expression + r10 = _nt_compare_expression if r10 r0 = r10 r0.extend(LogStash::Config::AST::Expression) else - r11 = _nt_rvalue + r11 = _nt_regexp_expression if r11 r0 = r11 r0.extend(LogStash::Config::AST::Expression) else - @index = i0 - r0 = nil + r12 = _nt_rvalue + if r12 + r0 = r12 + r0.extend(LogStash::Config::AST::Expression) + else + @index = i0 + r0 = nil + end end end end @@ -2677,6 +2683,71 @@ module LogStashConfig r0 end + module NotInExpression0 + def rvalue1 + elements[0] + end + + def _1 + elements[1] + end + + def not_in_operator + elements[2] + end + + def _2 + elements[3] + end + + def rvalue2 + elements[4] + end + end + + def _nt_not_in_expression + start_index = index + if node_cache[:not_in_expression].has_key?(index) + cached = node_cache[:not_in_expression][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + r1 = _nt_rvalue + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + r3 = _nt_not_in_operator + s0 << r3 + if r3 + r4 = _nt__ + s0 << r4 + if r4 + r5 = _nt_rvalue + s0 << r5 + end + end + end + end + if s0.last + r0 = instantiate_node(LogStash::Config::AST::NotInExpression,input, i0...index, s0) + r0.extend(NotInExpression0) + else + @index = i0 + r0 = nil + end + + node_cache[:not_in_expression][start_index] = r0 + + r0 + end + def _nt_in_operator start_index = index if node_cache[:in_operator].has_key?(index) @@ -2701,6 +2772,60 @@ module LogStashConfig r0 end + module NotInOperator0 + def _ + elements[1] + end + + end + + def _nt_not_in_operator + start_index = index + if node_cache[:not_in_operator].has_key?(index) + cached = node_cache[:not_in_operator][index] + if cached + cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0, s0 = index, [] + if has_terminal?("not ", false, index) + r1 = instantiate_node(SyntaxNode,input, index...(index + 4)) + @index += 4 + else + terminal_parse_failure("not ") + r1 = nil + end + s0 << r1 + if r1 + r2 = _nt__ + s0 << r2 + if r2 + if has_terminal?("in", false, index) + r3 = instantiate_node(SyntaxNode,input, index...(index + 2)) + @index += 2 + else + terminal_parse_failure("in") + r3 = nil + end + s0 << r3 + end + end + if s0.last + r0 = instantiate_node(SyntaxNode,input, i0...index, s0) + r0.extend(NotInOperator0) + else + @index = i0 + r0 = nil + end + + node_cache[:not_in_operator][start_index] = r0 + + r0 + end + def _nt_rvalue start_index = index if node_cache[:rvalue].has_key?(index) diff --git a/lib/logstash/config/grammar.treetop b/lib/logstash/config/grammar.treetop index 2c07442e2..328eabb2d 100644 --- a/lib/logstash/config/grammar.treetop +++ b/lib/logstash/config/grammar.treetop @@ -154,6 +154,7 @@ grammar LogStashConfig ("(" _ condition _ ")") / negative_expression / in_expression + / not_in_expression / compare_expression / regexp_expression / rvalue @@ -172,10 +173,19 @@ grammar LogStashConfig end + rule not_in_expression + rvalue _ not_in_operator _ rvalue + + end + rule in_operator "in" end + rule not_in_operator + "not " _ "in" + end + rule rvalue string / number / selector / array / method_call / regexp end diff --git a/spec/conditionals/test.rb b/spec/conditionals/test.rb index 01a760b7d..800bfaa1e 100644 --- a/spec/conditionals/test.rb +++ b/spec/conditionals/test.rb @@ -154,6 +154,29 @@ describe "conditionals" do end end + describe "the 'not in' operator" do + config <<-CONFIG + filter { + if "foo" not in "baz" { mutate { add_tag => "baz" } } + if "foo" not in "foo" { mutate { add_tag => "foo" } } + if !("foo" not in "foo") { mutate { add_tag => "notfoo" } } + if "foo" not in [somelist] { mutate { add_tag => "notsomelist" } } + if "one" not in [somelist] { mutate { add_tag => "somelist" } } + } + CONFIG + + sample("foo" => "foo", "somelist" => [ "one", "two" ], "foobar" => "foobar", "greeting" => "hello world", "tags" => [ "fancypantsy" ]) do + # verify the original exists + insist { subject["tags"] }.include?("fancypantsy") + + insist { subject["tags"] }.include?("baz") + reject { subject["tags"] }.include?("foo") + insist { subject["tags"] }.include?("notfoo") + insist { subject["tags"] }.include?("notsomelist") + reject { subject["tags"] }.include?("somelist") + end + end + describe "operators" do conditional "[message] == 'sample'" do sample("sample") { insist { subject["tags"] }.include?("success") }