Merge branch 'master' of https://github.com/fetep/logstash into fetep-pull/4

etc/logstash-nagios.yaml

@@ -0,0 +1,19 @@
+--- 
+configname: nagios
+# Example config that filters already-parsed logs (grok filter at least) for
+# certain patterns and sends the results to Nagios.
+inputs:
+  all:
+  - amqp:///topic/parsedlogs
+filters:
+- grep:
+    java:
+      - match:
+          JAVASTACKTRACEPART: .*
+        add_fields:
+          nagios_host: localhost
+          nagios_service: Java Exceptions
+          nagios_annotation: "Java exception"
+outputs:
+- stdout:///
+- nagios:///var/lib/nagios3/rw/nagios.cmd

etc/logstash-parser.yaml

@@ -1,13 +1,13 @@
 # Example config that parses rawlogs with grok and puts them on another AMQP topic
 inputs:
+  all:
   - amqp://localhost/topic/rawlogs
 outputs:
-  - amqp://localhost/topic/parsedlogs
   - stdout:///
 filters:
-  grok:
+- grok:
     linux-syslog: # for logs tagged 'linux-syslog'
-      timestamp: 
+      timestamp:
         key: date
         format: %b %e %H:%M:%S
       patterns:

etc/logstash-reader.yaml

@@ -1,5 +1,6 @@
 # Example config that reads parsed logs from AMQP and prints to stdout
 inputs:
+  all:
   - amqp://localhost/topic/parsedlogs
 #filters:
   #field:

lib/logstash/filters/date.rb

@@ -40,6 +40,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
         fieldvalue = event.fields[field]
         fieldvalue = [fieldvalue] if fieldvalue.is_a?(String)
         fieldvalue.each do |value|
+          next if value == ""
           begin
             case format
               when "ISO8601"

lib/logstash/filters/grep.rb

@@ -33,6 +33,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
     config = @config[event.type]
     if not config
       @logger.debug("grep: skipping type #{event.type} from #{event.source}")
+      event.cancel
       return
     end
 
@@ -54,6 +55,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
           next unless re.match(value)
           @logger.debug("grep matched on field #{field}")
           match_count += 1
+          break
         end
       end # match["match"].each
 

lib/logstash/inputs/beanstalk.rb

@@ -0,0 +1,33 @@
+require "logstash/inputs/base"
+require "em-jack"
+
+class LogStash::Inputs::Beanstalk < LogStash::Inputs::Base
+  def initialize(url, type, config={}, &block)
+    super
+
+    if @url.path == "" or @url.path == "/"
+      raise "must specify a tube for beanstalk output"
+    end
+  end
+
+  def register
+    tube = @url.path[1..-1] # Skip leading '/'
+    port = @url.port || 11300
+    @beanstalk = EMJack::Connection.new(:host => @url.host,
+                                        :port => port,
+                                        :tube => tube)
+    @beanstalk.each_job do |job|
+      begin
+        event = LogStash::Event.from_json(job.body)
+      rescue => e
+        @logger.warn(["Trouble parsing beanstalk job",
+                     {:error => e.message, :body => job.body,
+                      :backtrace => e.backtrace}])
+        @beanstalk.bury(job, 0)
+      end
+
+      receive(event)
+      @beanstalk.delete(job)
+    end # @beanstalk.each_job
+  end # def register
+end # class LogStash::Inputs::Beanstalk

lib/logstash/outputs/beanstalk.rb

@@ -0,0 +1,25 @@
+require "logstash/outputs/base"
+require "em-jack"
+
+class LogStash::Outputs::Beanstalk < LogStash::Outputs::Base
+  def initialize(url, config={}, &block)
+    super
+
+    @ttr = @urlopts["ttr"] || 300;
+    if @url.path == "" or @url.path == "/"
+      raise "must specify a tube for beanstalk output"
+    end
+  end
+
+  def register
+    tube = @url.path[1..-1] # Skip leading '/'
+    port = @url.port || 11300
+    @beanstalk = EMJack::Connection.new(:host => @url.host,
+                                        :port => port,
+                                        :tube => tube)
+  end # def register
+
+  def receive(event)
+    @beanstalk.put(event.to_json, :ttr => @ttr)
+  end # def receive
+end # class LogStash::Outputs::Beanstalk

lib/logstash/outputs/nagios.rb

@@ -0,0 +1,72 @@
+require "logstash/outputs/base"
+
+class LogStash::Outputs::Nagios < LogStash::Outputs::Base
+  NAGIOS_CRITICAL = 2
+  NAGIOS_WARN = 1
+
+  def initialize(url, config={}, &block)
+    super
+
+    if @url.path == "" or @url.path == "/"
+      @cmdfile = "/var/lib/nagios3/rw/nagios.cmd"
+    else
+      @cmdfile = @url.path
+    end
+  end
+
+  def register
+    # nothing to do
+  end # def register
+
+  def receive(event)
+    if !File.exists?(@cmdfile)
+      @logger.warn(["Skipping nagios output; command file is missing",
+                   {"cmdfile" => @cmdfile, "missed_event" => event}])
+      return
+    end
+
+    # TODO(petef): if nagios_host/nagios_service both have more than one
+    # value, send multiple alerts. They will have to match up together by
+    # array indexes (host/service combos) and the arrays must be the same
+    # length.
+
+    host = event.fields["nagios_host"]
+    if !host
+      @logger.warn(["Skipping nagios output; nagios_host field is missing",
+                   {"missed_event" => event}])
+      return
+    end
+
+    service = event.fields["nagios_service"]
+    if !service
+      @logger.warn(["Skipping nagios output; nagios_service field is missing",
+                   {"missed_event" => event}])
+      return
+    end
+
+    annotation = event.fields["nagios_annotation"]
+    level = NAGIOS_CRITICAL
+    if event.fields["nagios_level"] and event.fields["nagios_level"][0].downcase == "warn"
+      level = NAGIOS_WARN
+    end
+
+    cmd = "[#{Time.now.to_i}] PROCESS_SERVICE_CHECK_RESULT;#{host[0]};#{service[0]};#{level};"
+    if annotation
+      cmd += "#{annotation[0]}: "
+    end
+    cmd += "#{event.source}: "
+    # In the multi-line case, escape the newlines for the nagios command file
+    cmd += event.message.gsub("\n", "\\n")
+
+    @logger.debug({"cmdfile" => @cmdfile, "nagios_command" => cmd})
+    begin
+      File.open(@cmdfile, "a") do |f|
+        f.puts cmd
+      end
+    rescue
+      @logger.warn(["Skipping nagios output; error writing to command file",
+                   {"error" => $!, "cmdfile" => @cmdfile,
+                    "missed_event" => event}])
+    end
+  end # def event
+end # class LogStash::Outputs::Nagios

logstash-lite.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |spec|
   spec.version = "0.2.#{rev}"
   spec.summary = "logstash - log and event management (lite install, no dependencies)"
   spec.description = "scalable log and event management (search, archive, pipeline)"
+  spec.license = "Apache License (2.0)"
   spec.add_dependency("eventmachine-tail")
   spec.add_dependency("json")
   #spec.add_dependency("awesome_print")
@@ -28,7 +29,7 @@ Gem::Specification.new do |spec|
   # For amqp://
   spec.add_dependency("amqp")
   spec.add_dependency("uuidtools")
-  
+
   # For the web interface
   #spec.add_dependency("async_sinatra")
   #spec.add_dependency("rack")
@@ -42,8 +43,8 @@ Gem::Specification.new do |spec|
   spec.executables << "logstash-web"
   spec.executables << "logstash-test"
 
-  spec.author = "Jordan Sissel"
-  spec.email = "jls@semicomplete.com"
+  spec.authors = ["Jordan Sissel", "Pete Fritchman"]
+  spec.email = ["jls@semicomplete.com", "petef@databits.net"]
   spec.homepage = "http://code.google.com/p/logstash/"
 end
 

logstash.gemspec

@@ -11,6 +11,8 @@ Gem::Specification.new do |spec|
   spec.version = "0.2.#{rev}"
   spec.summary = "logstash - log and event management"
   spec.description = "scalable log and event management (search, archive, pipeline)"
+  spec.license = "Apache License (2.0)"
+
   spec.add_dependency("eventmachine-tail")
   spec.add_dependency("json")
   #spec.add_dependency("awesome_print")
@@ -28,13 +30,16 @@ Gem::Specification.new do |spec|
   # For amqp://
   spec.add_dependency("amqp")
   spec.add_dependency("uuidtools")
-  
+
   # For the web interface
   spec.add_dependency("async_sinatra")
   spec.add_dependency("rack")
   spec.add_dependency("thin")
   spec.add_dependency("haml")
 
+  # For beanstalk://
+  #spec.add_dependency("em-jack")
+
   spec.files = files
   spec.require_paths << "lib"
   spec.bindir = "bin"
@@ -42,8 +47,8 @@ Gem::Specification.new do |spec|
   spec.executables << "logstash-web"
   spec.executables << "logstash-test"
 
-  spec.author = "Jordan Sissel"
-  spec.email = "jls@semicomplete.com"
+  spec.authors = ["Jordan Sissel", "Pete Fritchman"]
+  spec.email = ["jls@semicomplete.com", "petef@databits.net"]
   spec.homepage = "http://code.google.com/p/logstash/"
 end