Merge tag 'v1.1.4' of git://github.com/logstash/logstash

CHANGELOG

@@ -1,7 +1,21 @@
-1.1.3 (October 22 2012)
+1.1.4 (October 28, 2012)
+ ## Overview of this release:
+ - bug fixes mostly
+
+ ## filters
+ - date: Fix crashing on date filter failures. Wrote test to cover this case.
+   (LOGSTASH-641)
+ - grok: Improve QUOTEDSTRING pattern to avoid some more 'watchdog timeout' problems
+
+ ## outputs
+ - nagios_nsca: Allow check status to be set from the event (#228, patch by
+   Tomas Doran)
+ - elasticsearch_http: Fix OpenSSL::X509::StoreError (LOGSTASH-642)
+
+1.1.3 (October 22, 2012)
  - rebuilt 1.1.2 for java 5 and 6
 
-1.1.2 (October 22 2012)
+1.1.2 (October 22, 2012)
  ## Overview of this release:
   * New input plugins: lumberjack, sqs, relp
   * New output plugins: exec, sqs

CONTRIBUTORS

@@ -57,6 +57,8 @@ Contributors:
 * goblin
 * Mike Worth (MikeWorth)
 * Nic Williams (drnic)
+* Tomas Doran (bobtfish)
+* zuazo
 
 Note: If you've sent me patches, bug reports, or otherwise contributed to
 logstash, and you aren't on the list above and want to be, please let me know

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    logstash (1.1.3-java)
+    logstash (1.1.4-java)
       addressable (= 2.2.6)
       aws-sdk
       bunny
@@ -10,7 +10,7 @@ PATH
       ffi
       ffi-rzmq (= 0.9.3)
       filewatch (= 0.5.0)
-      ftw (~> 0.0.20)
+      ftw (~> 0.0.22)
       gelf (= 1.3.2)
       gelfd (= 0.2.0)
       geoip (>= 1.1.0)
@@ -69,7 +69,7 @@ GEM
     ffi-rzmq (0.9.3)
       ffi
     filewatch (0.5.0)
-    ftw (0.0.20)
+    ftw (0.0.22)
       addressable (= 2.2.6)
       backports (= 2.3.0)
       cabin (> 0)
@@ -82,7 +82,7 @@ GEM
     geoip (1.2.0)
     gmetric (0.1.3)
     haml (3.1.7)
-    heroku (2.32.14)
+    heroku (2.33.0)
       heroku-api (~> 0.3.5)
       launchy (>= 0.3.2)
       netrc (~> 0.7.7)
@@ -156,8 +156,8 @@ GEM
     rspec-mocks (2.11.3)
     rubyzip (0.9.9)
     sass (3.2.1)
-    shoulda (3.3.1)
-      shoulda-context (~> 1.0)
+    shoulda (3.3.2)
+      shoulda-context (~> 1.0.1)
       shoulda-matchers (~> 1.4.1)
     shoulda-context (1.0.1)
     shoulda-matchers (1.4.1)
@@ -181,7 +181,6 @@ GEM
 
 PLATFORMS
   java
-  ruby
 
 DEPENDENCIES
   insist (= 0.0.7)

Makefile

@@ -79,6 +79,8 @@ copy-ruby-files: | build/ruby
 	| (cd lib; cpio -p --make-directories ../build/ruby)
 	$(QUIET)find ./test -name '*.rb' | sed -e 's,^\./test/,,' \
 	| (cd test; cpio -p --make-directories ../build/ruby)
+	$(QUIET)find ./spec -name '*.rb' | sed -e 's,^\./spec/,,' \
+	| (cd spec; cpio -p --make-directories ../build/ruby)
 
 vendor:
 	$(QUIET)mkdir $@

docs/flags.md

@@ -15,7 +15,7 @@ The logstash agent has the following flags (also try using the '--help' flag)
 <dd> Log to a given path. Default is to log to stdout </dd>
 <dt> -v </dt>
 <dd> Increase verbosity. There are multiple levels of verbosity available with
-'-vvv' currently being the highest </dd>
+'-vv' currently being the highest </dd>
 <dt> --pluginpath PLUGIN_PATH </dt>
 <dd> A colon-delimted path to find other logstash plugins in </dd>
 </dl>

docs/tutorials/getting-started-simple.md

@@ -113,7 +113,7 @@ Your output may look a little different.
 The reason we're going about it this way is to make absolutely sure that we have all the bits working before adding more complexity.
 
 If you are unable to get these steps working, you likely have something interfering with multicast traffic. This has been known to happen when connected to VPNs for instance.
-For best results, test on a Linux VM or system with less complicated networking. If in doubt, rerun the command with the options `-vvv` and paste the output to Github Gist or Pastie.
+For best results, test on a Linux VM or system with less complicated networking. If in doubt, rerun the command with the options `-vv` and paste the output to Github Gist or Pastie.
 Hop on the logstash IRC channel or mailing list and ask for help with that output as reference.
 
 Obviously this is fairly useless this way. Let's add the final step and test with the builtin logstash web ui:

lib/logstash/outputs/nagios_nsca.rb

@@ -25,6 +25,9 @@ class LogStash::Outputs::NagiosNsca < LogStash::Outputs::Base
   config_name "nagios_nsca"
   plugin_status "experimental"
 
+  # The status to send to nagios. Should be 0 = OK, 1 = WARNING, 2 = CRITICAL, 3 = UNKNOWN
+  config :nagios_status, :validate => :string, :required => true
+
   # The nagios host or IP to send logs to. It should have a NSCA daemon running.
   config :host, :validate => :string, :default => "localhost"
 
@@ -81,11 +84,21 @@ class LogStash::Outputs::NagiosNsca < LogStash::Outputs::Base
     msg.gsub!("\n", "<br/>")
     msg.gsub!("'", "&#146;")
 
+    status = event.sprintf(@nagios_status)
+    if status.to_i != status
+      msg = "status '#{status}' is not numeric"
+      status = 2
+    else
+      if status > 3 || status < 0
+         msg "status must be > 0 and <= 3, not #{status}"
+         status = 2
+      end
+    end
+
     # build the command
     # syntax: echo '<server>!<nagios_service>!<status>!<text>'  | \
     #           /usr/sbin/send_nsca -H <nagios_host> -d '!' -c <nsca_config>"
-    # TODO: make nagios status configurable ; defaults to 1 = 'WARNING' for now.
-    cmd = %(echo '#{nagios_host}~#{nagios_service}~1~#{msg}' |)
+    cmd = %(echo '#{nagios_host}~#{nagios_service}~#{status}~#{msg}' |)
     cmd << %( #{@send_nsca_bin} -H #{@host} -p #{@port} -d '~')
     cmd << %( -c #{@send_nsca_config}) if @send_nsca_config
     cmd << %( 2>/dev/null >/dev/null)

lib/logstash/version.rb

@@ -1,5 +1,5 @@
 # The version of logstash.
-LOGSTASH_VERSION = "1.1.3"
+LOGSTASH_VERSION = "1.1.4"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

logstash.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "stud"
 
   # Web dependencies
-  gem.add_runtime_dependency "ftw", ["~> 0.0.20"]
+  gem.add_runtime_dependency "ftw", ["~> 0.0.22"]
   gem.add_runtime_dependency "haml"
   gem.add_runtime_dependency "rack"
   gem.add_runtime_dependency "sass"

patterns/grok-patterns

@@ -14,7 +14,7 @@ SPACE \s*
 DATA .*?
 GREEDYDATA .*
 #QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
-QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|(?>'(?>\\.|[^\\']+)+')|(?>`(?>\\.|[^\\`]+)+`)))
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
 UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
 
 # Networking

spec/examples/syslog.rb

@@ -0,0 +1,51 @@
+require "test_utils"
+
+describe "parse syslog" do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grok {
+          type => "syslog"
+          singles => true
+          pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ]
+          add_field => [ "received_at", "%{@timestamp}" ]
+          add_field => [ "received_from", "%{@source_host}" ]
+      }
+      syslog_pri {
+          type => "syslog"
+      }
+      date {
+          type => "syslog"
+          syslog_timestamp => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+      }
+      mutate {
+          type => "syslog"
+          exclude_tags => "_grokparsefailure"
+          replace => [ "@source_host", "%{syslog_hostname}" ]
+          replace => [ "@message", "%{syslog_message}" ]
+      }
+      mutate {
+          type => "syslog"
+          remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
+      }
+    }
+  CONFIG
+
+  sample("@message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]", "@type" => "syslog") do
+    insist { subject.type } == "syslog"
+    reject { subject.tags }.include?("_grokparsefailure")
+    insist { subject["syslog_pri"] } == "164"
+    #insist { subject.timestamp } == "2012-10-26T15:19:25.000Z"
+    puts subject.to_hash
+  end
+
+  # Single digit day
+  sample("@message" => "<164>Oct  6 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]", "@type" => "syslog") do
+    insist { subject.type } == "syslog"
+    reject { subject.tags }.include?("_grokparsefailure")
+    insist { subject["syslog_pri"] } == "164"
+    #insist { subject.timestamp } == "2012-10-26T15:19:25.000Z"
+    puts subject.to_hash
+  end
+end

spec/inputs/generator.rb

@@ -0,0 +1,35 @@
+require "test_utils"
+
+describe "inputs/generator" do
+  extend LogStash::RSpec
+
+  class Shiftcount
+    def initialize; @count = 0; end
+    def <<(arg); @count += 1 end
+    attr_reader :count
+  end
+
+  describe "generate events" do
+    event_count = 100000 + rand(50)
+
+    config <<-CONFIG
+      input {
+        generator {
+          type => "blah"
+          count => #{event_count}
+        }
+      }
+    CONFIG
+
+    input do |plugins|
+      sequence = 0
+      generator = plugins.first
+      output = Shiftcount.new
+      generator.register
+      start = Time.now
+      generator.run(output)
+      duration = Time.now - start
+      puts "Rate: #{event_count / duration}"
+    end # input
+  end
+end