- Add "not in" operator based on feedback from this logstash 1.2.0

writeup:
http://tobrunet.ch/2013/09/logstash-1-2-0-upgrade-notes-included/

CHANGELOG

@@ -6,7 +6,7 @@
   - deprecation: Using deprecated plugin settings can now advise you on a
     corrective path to take. One example is the 'type' setting on filters and
     outputs will now advise you to use conditionals and give an example.
-
+  - conditionals: The "not in" operator is now supported.
 
   ## inputs
   - feature: pipe: reopen the pipe and retry on any error. (#619, Jonathan Van

docs/configuration.md

@@ -195,8 +195,8 @@ What's an expression? Comparison tests, boolean logic, etc!
 The following comparison operators  are supported:
 
 * equality, etc: ==  !=  <  >  <=  >= 
-* regexp: =~  !~ 
-* inclusion: in
+* regexp: =~ !~ 
+* inclusion: in, not in
 
 The following boolean operators are supported:
 
@@ -207,7 +207,8 @@ The following unary operators are supported:
 * !
 
 Expressions may contain expressions. Expressions may be negated with `!`.
-Expressions may be grouped with parentheses `(...)`.
+Expressions may be grouped with parentheses `(...)`. Expressions can be long
+and complex.
 
 For example, if we want to remove the field `secret` if the field
 `action` has a value of `login`:
@@ -242,6 +243,17 @@ How about telling nagios of any http event that has a status code of 5xx?
       }
     }
 
+You can also do multiple expressions in a single condition:
+
+    output {
+      # Send production errors to pagerduty
+      if [loglevel] == "ERROR" and [deployment] == "production" {
+        pagerduty {
+          ...
+        }
+      }
+    }
+
 ## Further Reading
 
 For more information, see [the plugin docs index](index)

lib/logstash/config/config_ast.rb

@@ -289,6 +289,13 @@ module LogStash; module Config; module AST
     end
   end
 
+  module NotInExpression
+    def compile
+      item, list = recursive_select(LogStash::Config::AST::RValue)
+      return "(x = #{list.compile}; x.respond_to?(:include?) && !x.include?(#{item.compile}))"
+    end
+  end
+
   class MethodCall < Node
     def compile
       arguments = recursive_inject { |e| [String, Number, Selector, Array, MethodCall].any? { |c| e.is_a?(c) } }

lib/logstash/config/grammar.rb

@@ -2448,23 +2448,29 @@ module LogStashConfig
           r0 = r8
           r0.extend(LogStash::Config::AST::Expression)
         else
-          r9 = _nt_compare_expression
+          r9 = _nt_not_in_expression
           if r9
             r0 = r9
             r0.extend(LogStash::Config::AST::Expression)
           else
-            r10 = _nt_regexp_expression
+            r10 = _nt_compare_expression
             if r10
               r0 = r10
               r0.extend(LogStash::Config::AST::Expression)
             else
-              r11 = _nt_rvalue
+              r11 = _nt_regexp_expression
               if r11
                 r0 = r11
                 r0.extend(LogStash::Config::AST::Expression)
               else
-                @index = i0
-                r0 = nil
+                r12 = _nt_rvalue
+                if r12
+                  r0 = r12
+                  r0.extend(LogStash::Config::AST::Expression)
+                else
+                  @index = i0
+                  r0 = nil
+                end
               end
             end
           end
@@ -2677,6 +2683,71 @@ module LogStashConfig
     r0
   end
 
+  module NotInExpression0
+    def rvalue1
+      elements[0]
+    end
+
+    def _1
+      elements[1]
+    end
+
+    def not_in_operator
+      elements[2]
+    end
+
+    def _2
+      elements[3]
+    end
+
+    def rvalue2
+      elements[4]
+    end
+  end
+
+  def _nt_not_in_expression
+    start_index = index
+    if node_cache[:not_in_expression].has_key?(index)
+      cached = node_cache[:not_in_expression][index]
+      if cached
+        cached = SyntaxNode.new(input, index...(index + 1)) if cached == true
+        @index = cached.interval.end
+      end
+      return cached
+    end
+
+    i0, s0 = index, []
+    r1 = _nt_rvalue
+    s0 << r1
+    if r1
+      r2 = _nt__
+      s0 << r2
+      if r2
+        r3 = _nt_not_in_operator
+        s0 << r3
+        if r3
+          r4 = _nt__
+          s0 << r4
+          if r4
+            r5 = _nt_rvalue
+            s0 << r5
+          end
+        end
+      end
+    end
+    if s0.last
+      r0 = instantiate_node(LogStash::Config::AST::NotInExpression,input, i0...index, s0)
+      r0.extend(NotInExpression0)
+    else
+      @index = i0
+      r0 = nil
+    end
+
+    node_cache[:not_in_expression][start_index] = r0
+
+    r0
+  end
+
   def _nt_in_operator
     start_index = index
     if node_cache[:in_operator].has_key?(index)
@@ -2701,6 +2772,60 @@ module LogStashConfig
     r0
   end
 
+  module NotInOperator0
+    def _
+      elements[1]
+    end
+
+  end
+
+  def _nt_not_in_operator
+    start_index = index
+    if node_cache[:not_in_operator].has_key?(index)
+      cached = node_cache[:not_in_operator][index]
+      if cached
+        cached = SyntaxNode.new(input, index...(index + 1)) if cached == true
+        @index = cached.interval.end
+      end
+      return cached
+    end
+
+    i0, s0 = index, []
+    if has_terminal?("not ", false, index)
+      r1 = instantiate_node(SyntaxNode,input, index...(index + 4))
+      @index += 4
+    else
+      terminal_parse_failure("not ")
+      r1 = nil
+    end
+    s0 << r1
+    if r1
+      r2 = _nt__
+      s0 << r2
+      if r2
+        if has_terminal?("in", false, index)
+          r3 = instantiate_node(SyntaxNode,input, index...(index + 2))
+          @index += 2
+        else
+          terminal_parse_failure("in")
+          r3 = nil
+        end
+        s0 << r3
+      end
+    end
+    if s0.last
+      r0 = instantiate_node(SyntaxNode,input, i0...index, s0)
+      r0.extend(NotInOperator0)
+    else
+      @index = i0
+      r0 = nil
+    end
+
+    node_cache[:not_in_operator][start_index] = r0
+
+    r0
+  end
+
   def _nt_rvalue
     start_index = index
     if node_cache[:rvalue].has_key?(index)

lib/logstash/config/grammar.treetop

@@ -154,6 +154,7 @@ grammar LogStashConfig
         ("(" _ condition _ ")")
       / negative_expression
       / in_expression
+      / not_in_expression
       / compare_expression
       / regexp_expression
       / rvalue
@@ -172,10 +173,19 @@ grammar LogStashConfig
     <LogStash::Config::AST::InExpression>
   end
 
+  rule not_in_expression
+    rvalue _ not_in_operator _ rvalue
+    <LogStash::Config::AST::NotInExpression>
+  end
+
   rule in_operator
     "in"
   end
 
+  rule not_in_operator
+    "not " _ "in"
+  end
+
   rule rvalue
     string / number / selector / array / method_call / regexp
   end

spec/conditionals/test.rb

@@ -154,6 +154,29 @@ describe "conditionals" do
     end
   end
 
+  describe "the 'not in' operator" do
+    config <<-CONFIG
+      filter {
+        if "foo" not in "baz" { mutate { add_tag => "baz" } }
+        if "foo" not in "foo" { mutate { add_tag => "foo" } }
+        if !("foo" not in "foo") { mutate { add_tag => "notfoo" } }
+        if "foo" not in [somelist] { mutate { add_tag => "notsomelist" } } 
+        if "one" not in [somelist] { mutate { add_tag => "somelist" } }
+      }
+    CONFIG
+
+    sample("foo" => "foo", "somelist" => [ "one", "two" ], "foobar" => "foobar", "greeting" => "hello world", "tags" => [ "fancypantsy" ]) do
+      # verify the original exists
+      insist { subject["tags"] }.include?("fancypantsy")
+
+      insist { subject["tags"] }.include?("baz")
+      reject { subject["tags"] }.include?("foo")
+      insist { subject["tags"] }.include?("notfoo")
+      insist { subject["tags"] }.include?("notsomelist")
+      reject { subject["tags"] }.include?("somelist")
+    end
+  end
+
   describe "operators" do
     conditional "[message] == 'sample'" do
       sample("sample") { insist { subject["tags"] }.include?("success") }