Merge branch 'master' of github.com:logstash/logstash

Conflicts:
	etc/agent.conf

Rakefile

@@ -8,33 +8,32 @@ file "lib/logstash/config/grammar.rb" => ["lib/logstash/config/grammar.rl"] do
   sh "make -C lib/logstash/config grammar.rb"
 end
 
-task :compile => "lib/logstash/config/grammar.rb" do |t|
-  # Taken from 'jrubyc' 
-  #  Currently this code is commented out because jruby emits this:
-  #     Failure during compilation of file logstash/web/helpers/require_param.rb:
-  #       java.lang.RuntimeException: java.io.FileNotFoundException: File path
-  #       /home/jls/projects/logstash/logstash/web/helpers/require_param.rb
-  #       does not start with parent path /home/jls/projects/logstash/lib
-  #
-  #     org/jruby/util/JavaNameMangler.java:105:in `mangleFilenameForClasspath'
-  #     org/jruby/util/JavaNameMangler.java:32:in `mangleFilenameForClasspath'
-  require 'jruby/jrubyc'
-  #args = [ "-p", "net.logstash" ]
-  args = ["-d", "build"]
-  args += Dir.glob("**/*.rb")
-  status = JRuby::Compiler::compile_argv(args)
-  if (status != 0)
-    puts "Compilation FAILED: #{status} error(s) encountered"
-    exit status
-  end
+# Taken from 'jrubyc' 
+#  Currently this code is commented out because jruby emits this:
+#     Failure during compilation of file logstash/web/helpers/require_param.rb:
+#       java.lang.RuntimeException: java.io.FileNotFoundException: File path
+#       /home/jls/projects/logstash/logstash/web/helpers/require_param.rb
+#       does not start with parent path /home/jls/projects/logstash/lib
+#
+#     org/jruby/util/JavaNameMangler.java:105:in `mangleFilenameForClasspath'
+#     org/jruby/util/JavaNameMangler.java:32:in `mangleFilenameForClasspath'
+#require 'jruby/jrubyc'
+##args = [ "-p", "net.logstash" ]
+#args = ["-d", "build"]
+#args += Dir.glob("**/*.rb")
+#status = JRuby::Compiler::compile_argv(args)
+#if (status != 0)
+  #puts "Compilation FAILED: #{status} error(s) encountered"
+  #exit status
+#end
 
-  #mkdir_p "build"
-  #sh "rm -rf lib/net"
-  #Dir.chdir("lib") do
-    #args = Dir.glob("**/*.rb")
-    ##sh "jrubyc", "-d", "../build" *args
-    #sh "jrubyc", *args
-  #end
+task :compile => "lib/logstash/config/grammar.rb" do |t|
+  mkdir_p "build"
+  sh "rm -rf lib/net"
+  Dir.chdir("lib") do
+    args = Dir.glob("**/*.rb")
+    sh "jrubyc", "-t", "../build", *args
+  end
 end
 
 VERSIONS = {
@@ -45,7 +44,7 @@ VERSIONS = {
 
 namespace :vendor do
   file "vendor/jar" do |t|
-    mkdir_p mkdir(t.name)
+    mkdir_p t.name
   end
 
   # Download jruby.jar
@@ -119,15 +118,24 @@ namespace :package do
         end
       end
 
-      # We compile stuff to lib/net/logstash/...
-      Dir.glob("lib/**/*.class").each do |file|
-        target = File.join("build-jar", file.gsub("lib/", ""))
-        #target = File.join("build-jar", file)
+      # We compile stuff to build/...
+      Dir.glob("build/**/*.class").each do |file|
+        target = File.join("build-jar", file.gsub("build/", ""))
         mkdir_p File.dirname(target)
         puts "=> Copying #{file} => #{target}"
         File.copy(file, target)
       end
 
+      # Copy gems to the root of the build-jar dir
+      # TODO(sissel): Figure out how to package the gems. Maybe see how warbler
+      # does it.
+      #Dir.glob("vendor/bundle/jruby/1.8/gems/**/*") do |file|
+        #target = File.join("build-jar", file.gsub("build/", ""))
+        #mkdir_p File.dirname(target)
+        #puts "=> Copying #{file} => #{target}"
+        #File.copy(file, target)
+      #end
+
       output = "logstash-#{LOGSTASH_VERSION}.jar"
       sh "jar -cfe #{output} logstash.agent -C build-jar ."
       sh "jar -uf #{output} patterns/"

etc/agent.conf

@@ -0,0 +1,24 @@
+input {
+  file {
+    path => [ "/var/log/messages", "/var/log/*.log", "/tmp/zlog" ]
+    type => "linux-syslog"
+  }
+}
+
+filter {
+  grok {
+    type => "linux-syslog"
+    pattern => "%{SYSLOGLINE}"
+  }
+
+  date {
+    type => "linux-syslog"
+    timestamp => "MMM dd HH:mm:ss"
+    timestamp8601 => ISO8601
+  }
+}
+output {
+  stdout {
+    debug => true
+  }
+}

lib/logstash/agent.rb

@@ -1,19 +1,18 @@
 #TODO(sissel): Maybe this will help jruby jar issues?
 #$: << File.join(File.dirname(__FILE__), "../"
 
+require "java"
+require "logstash/config/file"
 require "logstash/filters"
 require "logstash/inputs"
 require "logstash/logging"
 require "logstash/multiqueue"
 require "logstash/namespace"
 require "logstash/outputs"
-require "logstash/config/file"
+require "logstash/util"
 require "optparse"
-require "java"
 require "uri"
 
-JThread = java.lang.Thread
-
 # TODO(sissel): only enable this if we are in debug mode.
 # JRuby.objectspace=true
 
@@ -201,14 +200,22 @@ class LogStash::Agent
       STDERR.reopen(devnull)
     end
 
-    if @verbose > 0
+    if @verbose >= 3  # Uber debugging.
       @logger.level = Logger::DEBUG
+      $DEBUG = true
+    elsif @verbose == 2 # logstash debug logs
+      @logger.level = Logger::DEBUG
+    elsif @verbose == 1 # logstash info logs
+      @logger.level = Logger::INFO
+    else # Default log level
+      @logger.level = Logger::WARN
     end
   end # def configure
 
   public
   def run
-    JThread.currentThread().setName(self.class.name)
+    LogStash::Util::set_thread_name(self.class.name)
+
     ok = parse_options
     if !ok
       raise "Option parsing failed. See error log."
@@ -270,7 +277,7 @@ class LogStash::Agent
       1.times do |n|
         @logger.info("Starting filter worker thread #{n}")
         @threads["filter|worker|#{n}"] = Thread.new do
-          JThread.currentThread().setName("filter|worker|#{n}")
+          LogStash::Util::set_thread_name("filter|worker|#{n}")
           @filters.each do |filter|
             filter.logger = @logger
             filter.register
@@ -303,7 +310,7 @@ class LogStash::Agent
       @threads["outputs/#{output.to_s}"] = Thread.new(queue) do |queue|
         output.register
         begin
-          JThread.currentThread().setName("output/#{output.to_s}")
+          LogStash::Util::set_thread_name("output/#{output.to_s}")
           output.logger = @logger
 
           while event = queue.pop do

lib/logstash/config/mixin.rb

@@ -115,7 +115,7 @@ module LogStash::Config::Mixin
 
     def validate(params)
       @plugin_name = [ancestors[1].config_name, config_name].join("/")
-      @logger = LogStash::Logger.new(STDERR)
+      @logger = LogStash::Logger.new(STDOUT)
       is_valid = true
 
       is_valid &&= validate_check_invalid_parameter_names(params)

lib/logstash/file/manager.rb

@@ -1,5 +1,7 @@
 require "filewatch/tail" # rubygem 'filewatch'
 require "logstash/namespace"
+require "logstash/logging"
+require "logstash/util"
 require "set"
 require "socket" # for Socket.gethostname
 
@@ -16,7 +18,7 @@ class LogStash::File::Manager
     @file_threads = {}
     @main_thread = nil
     @output_queue = nil
-    @logger = Logger.new(STDOUT)
+    @logger = LogStash::Logger.new(STDOUT)
     @hostname = Socket.gethostname
   end # def initialize
 
@@ -57,7 +59,7 @@ class LogStash::File::Manager
 
   private
   def watcher
-    JThread.currentThread().setName(self.class.name)
+    LogStash::Util::set_thread_name(self.class.name)
     @buffers = Hash.new { |h,k| h[k] = BufferedTokenizer.new }
     begin
       @tail.subscribe do |path, data|

lib/logstash/filters/base.rb

@@ -14,7 +14,7 @@ class LogStash::Filters::Base
 
   public
   def initialize(params)
-    @logger = LogStash::Logger.new(STDERR)
+    @logger = LogStash::Logger.new(STDOUT)
     config_init(params)
   end # def initialize
 

lib/logstash/filters/grok.rb

@@ -15,7 +15,13 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     attr_accessor :patterns_dir
   end
 
-  self.patterns_dir = ["#{File.dirname(__FILE__)}/../../../patterns/*"]
+  # Detect if we are running from a jarfile, pick the right path.
+  if __FILE__ =~ /file:\/.*\.jar!.*/
+    self.patterns_dir = ["#{File.dirname(__FILE__)}/../../patterns/*"]
+  else
+    self.patterns_dir = ["#{File.dirname(__FILE__)}/../../../patterns/*"]
+  end
+
   flag("--patterns-path PATH", "Colon-delimited path of patterns to load") do |val|
     @patterns_dir += val.split(":")
   end
@@ -33,13 +39,20 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
     @pile = Grok::Pile.new
     @logger.info("Grok patterns paths: #{self.class.patterns_dir.inspect}")
     self.class.patterns_dir.each do |path|
+      # Can't read relative paths from jars, try to normalize away '../'
+      while path =~ /file:\/.*\.jar!.*\/\.\.\//
+        # replace /foo/bar/../baz => /foo/baz
+        path.gsub!(/[^\/]+\/\.\.\//, "")
+        @logger.debug "In-jar path to read: #{path}"
+      end
+
       if File.directory?(path)
         path = File.join(path, "*")
       end
 
       Dir.glob(path).each do |file|
         @logger.info("Grok loading patterns from #{file}")
-        @pile.add_patterns_from_file(file)
+        add_patterns_from_file(file)
       end
     end
 
@@ -109,4 +122,20 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
 
     @logger.debug(["Event now: ", event.to_hash])
   end # def filter
+
+  private
+  def add_patterns_from_file(file)
+    # Check if the file path is a jar, if so, we'll have to read it ourselves
+    # since libgrok won't know what to do with it.
+    if file =~ /file:\/.*\.jar!.*/
+      File.new(file).each do |line|
+        next if line =~ /^(?:\s*#|\s*$)/
+        name, pattern = line.split(/\s+/, 2)
+        @logger.debug "Adding pattern '#{name}' from file #{file}"
+        @pile.add_pattern(name, pattern)
+      end
+    else
+      @pile.add_patterns_from_file(file)
+    end
+  end # def add_patterns
 end # class LogStash::Filters::Grok

lib/logstash/inputs/base.rb

@@ -22,7 +22,7 @@ class LogStash::Inputs::Base
 
   public
   def initialize(params)
-    @logger = LogStash::Logger.new(STDERR)
+    @logger = LogStash::Logger.new(STDOUT)
     config_init(params)
 
     @tags ||= []

lib/logstash/util.rb

@@ -0,0 +1,7 @@
+require "logstash/namespace"
+
+module LogStash::Util
+  def self.set_thread_name(name)
+    java.lang.Thread.currentThread.setName(name)
+  end # def set_thread_name
+end # module LogStash::Util