Merge pull request #62 from crankycoder/master

- Add some testing/debugging libs
- Add ffi-rbzmq in prep for some ZMQ plugin work
- Add flags and other improvements to the test runner

Gemfile

@@ -31,6 +31,10 @@ gem "gmetric", "~> 0.1.3" # outputs/ganglia, # License: MIT
 gem "xmpp4r", "~> 0.5" # outputs/xmpp, # License: As-Is
 gem "gelfd", "~> 0.1.0" #inputs/gelf, # License: Apache 2.0
 
+gem "ffi-rzmq"
+gem "ruby-debug"
+gem "mocha"
+
 # For testing/dev
 group :development do
   gem "rake"

Gemfile.lock

@@ -8,6 +8,8 @@ GEM
     bunny (0.7.8)
     cabin (0.1.3)
       json
+    columnize (0.3.4)
+    ffi-rzmq (0.8.2)
     filewatch (0.3.0)
     gelf (1.1.3)
       json
@@ -20,16 +22,28 @@ GEM
       bouncy-castle-java
     json (1.6.1)
     json (1.6.1-java)
+    linecache (0.46)
+      rbx-require-relative (> 0.0.4)
+    metaclass (0.0.1)
     minitest (2.6.1)
     mizuno (0.4.0)
       rack (>= 1.0.0)
+    mocha (0.10.0)
+      metaclass (~> 0.0.1)
     mongo (1.4.0)
       bson (= 1.4.0)
     rack (1.3.4)
     rack-protection (1.1.4)
       rack
     rake (0.9.2)
+    rbx-require-relative (0.0.5)
     redis (2.2.2)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    ruby-debug-base (0.10.4-java)
     sass (3.1.10)
     sinatra (1.3.1)
       rack (~> 1.3, >= 1.3.4)
@@ -49,6 +63,7 @@ DEPENDENCIES
   awesome_print
   bunny
   cabin (= 0.1.3)
+  ffi-rzmq
   filewatch (~> 0.3.0)
   gelf
   gelfd (~> 0.1.0)
@@ -60,10 +75,12 @@ DEPENDENCIES
   json
   minitest
   mizuno
+  mocha
   mongo
   rack
   rake
   redis
+  ruby-debug
   sass
   sinatra
   statsd-ruby (~> 0.3.0)

lib/logstash/filters/grok.rb

@@ -149,7 +149,6 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
 
       if !@patterns.include?(field)
         @patterns[field] = Grok::Pile.new
-        @patterns[field].logger = @logger
 
         add_patterns_from_files(@patternfiles, @patterns[field])
       end

lib/logstash/test.rb

@@ -4,8 +4,24 @@ $:.unshift "#{File.dirname(__FILE__)}/../lib"
 $:.unshift "#{File.dirname(__FILE__)}/../test"
 require "logstash/namespace"
 require "logstash/loadlibs"
+require "logstash/logging"
+require "ruby-debug"
 
 class LogStash::Test
+    public
+    def initialize
+        log_to(STDERR)
+       
+        # initialize to an empty pluginpath
+        @verbose = 0
+        @plugin_paths = []
+    end
+
+  public
+  def log_to(target)
+    @logger = LogStash::Logger.new(target)
+  end # def log_to
+
   def check_lib(lib, provider, is=:optional, message=nil)
     optional = (is == :optional)
     begin
@@ -68,10 +84,64 @@ class LogStash::Test
   def run_tests(args)
     require "logstash_test_runner"
     return MiniTest::Unit.new.run(args)
-    #return Test::Unit::AutoRunner.run
   end # def run_tests
 
+
+  # Parse options.
+  private
+  def extend_pluginpath(args)
+    # strip out the pluginpath argument if it exists and 
+    # extend the LOAD_PATH for the ruby runtime
+    opts = OptionParser.new
+
+    opts.on("-v", "Increase verbosity") do
+      @verbose += 1
+
+      if @verbose >= 3  # Uber debugging.
+          @logger.level = :debug
+          $DEBUG = true
+      elsif @verbose == 2 # logstash debug logs
+          @logger.level = :debug
+      elsif @verbose == 1 # logstash info logs
+          @logger.level = :info
+      else # Default log level
+          @logger.level = :warn
+      end
+
+    end
+
+    # Step one is to add test flags.
+    opts.on("--pluginpath PLUGINPATH", 
+            "Load plugins and test from a pluginpath") do |path|
+      @plugin_paths << path
+
+      @plugin_paths.each do |p|
+        @logger.debug("Adding to ruby load path", :path => p)
+
+        runner = PluginTestRunner.new p
+        $:.unshift p
+        runner.load_tests()
+
+        puts "Added to ruby load :path = [#{p}]"
+      end
+    end # --pluginpath PLUGINPATH
+
+
+    begin
+      remainder = opts.parse(args)
+
+    rescue OptionParser::InvalidOption => e
+      @logger.info("Invalid option", :exception => e)
+      raise e
+    end
+    return remainder
+  end # def extend_pluginpath
+
+  public
   def run(args)
+
+    args = extend_pluginpath(args)
+
     @success = true
     @thread = Thread.new do
       report_ruby_version
@@ -95,3 +165,28 @@ class LogStash::Test
     return @success ? 0 : 1
   end # def wait
 end # class LogStash::Test
+
+class PluginTestRunner
+    def initialize(rootpath)
+        @rootpath = rootpath
+    end
+
+    def _discover_tests()
+        glob_path = File.join(@rootpath, "**", "test_*.rb")
+        puts "Searching [#{glob_path}]"
+        Dir.glob(glob_path).each do|f|
+            yield f
+        end
+    end
+
+    def load_tests()
+        _discover_tests() do |path|
+            path_parts = path.split(File::SEPARATOR).map {|x| x=="" ? File::SEPARATOR : x}
+            test_module = File.join(path_parts.slice(1, path_parts.length + 1))
+            test_module = test_module.sub(".rb", '')
+            puts "Loading test module: #{test_module}"
+            require test_module
+            puts "Loaded : [#{test_module}]"
+        end
+    end
+end

logstash.gemspec

@@ -47,6 +47,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "uuidtools" # for naming amqp queues, License ???
   spec.add_dependency "xmpp4r", "~> 0.5" # outputs/xmpp, # License: As-Is
 
+  spec.add_dependency("ffi-rzmq")
+  spec.add_dependency("ruby-debug")
+  spec.add_dependency("mocha")
+
   spec.files = files
   spec.require_paths << "lib"
   spec.bindir = "bin"

test/logstash/filters/test_date.rb

@@ -38,7 +38,9 @@ describe LogStash::Filters::Date do
       "2001-09-05T16:36:36+0000"         => "2001-09-05T16:36:36.000Z",
       "2001-11-06T20:45:45-0000"         => "2001-11-06T20:45:45.000Z",
       "2001-12-07T23:54:54Z"             => "2001-12-07T23:54:54.000Z",
-      "2001-01-01T00:00:00.123"          => "2001-01-01T08:00:00.123Z",
+
+      # TODO: This test assumes PDT
+      #"2001-01-01T00:00:00.123"          => "2001-01-01T08:00:00.123Z",
 
       "2010-05-03T08:18:18.123+00:00"    => "2010-05-03T08:18:18.123Z",
       "2004-07-04T12:27:27.123-04:00"    => "2004-07-04T16:27:27.123Z",
@@ -64,6 +66,9 @@ describe LogStash::Filters::Date do
     now = Time.now
     now += now.gmt_offset
     year = now.year
+    require 'java'
+    skip("this test assumes pacific time - won't work anywhere else")
+
     times = {
       "Nov 24 01:29:01" => "#{year}-11-24T09:29:01.000Z",
     }
@@ -84,6 +89,8 @@ describe LogStash::Filters::Date do
     start = Time.now
     gmt_now = start + start.gmt_offset
     year = gmt_now.year
+
+    skip("This test assumes PDT")
     input = "Nov 24 01:29:01" 
     output = "#{year}-11-24T09:29:01.000Z"