Merge branch 'logstash-master'

CHANGELOG

@@ -1,9 +1,27 @@
+1.0.7 ( ????? )
+  @ TODO: logstash-web needs to support elasticsearch cluster name
+  @ TODO: GELF 'dynamic' level/facility? (LOGSTASH-83)
+  @ TODO: Catch input exceptions and handle them sanely (LOGSTASH-84)
+
+1.0.6 (May 11, 2011)
+  * Remove 'sigar' from monolithic jar packaging. This removes a boatload of
+    unnecessary warning messages on startup whenever you use elasticsearch
+    output or logstash-web.
+    Issue: https://logstash.jira.com/browse/LOGSTASH-79
+
+1.0.5 (May 10, 2011)
+  * fix queues when durable is set to true
+
+1.0.4 (May 9, 2011)
+  * Fix bugs in syslog input
+
 1.0.2 (May 8, 2011)
   * Fix default-value handling for configs when the validation type is
     'password'
 
 1.0.1 (May 7, 2011)
   * Fix password auth for amqp and stomp (Reported by Luke Macken)
+  * Fix default elasticsearch target for logstash-web (Reported by Donald Gordon)
 
 1.0.0 (May 6, 2011)
   * First major release.

Rakefile

@@ -1,9 +1,8 @@
 require 'tempfile'
 require 'ftools' # fails in 1.9.2
 
-# TODO(sissel): load the gemspec and parse the version from it instead.
-LOGSTASH_VERSION = "1.0.4" 
-
+require File.join(File.dirname(__FILE__), "VERSION")  # For LOGSTASH_VERSION
+  
 # Compile config grammar (ragel -> ruby)
 file "lib/logstash/config/grammar.rb" => ["lib/logstash/config/grammar.rl"] do
   sh "make -C lib/logstash/config grammar.rb"
@@ -30,22 +29,35 @@ end
 
 task :clean do
   sh "rm -rf .bundle"
-  sh "rm -rf build-jar-thin"
-  sh "rm -rf build-jar"
+  #sh "rm -rf build-jar-thin"
+  #sh "rm -rf build-jar"
   sh "rm -rf build"
   sh "rm -rf vendor"
 end
 
 task :compile => "lib/logstash/config/grammar.rb" do |t|
-  mkdir_p "build"
-  sh "rm -rf lib/net"
+  target = "build/ruby"
+  mkdir_p target if !File.directory?(target)
+  #sh "rm -rf lib/net"
   Dir.chdir("lib") do
-    args = Dir.glob("**/*.rb")
-    sh "jrubyc", "-t", "../build", *args
+    rel_target = File.join("..", target)
+    sh "jrubyc", "-t", rel_target, "logstash/runner"
+    files = Dir.glob("**/*.rb")
+    files.each do |file|
+      d = File.join(rel_target, File.dirname(file))
+      mkdir_p d if !File.directory?(d)
+      cp file, File.join(d, File.basename(file))
+    end
   end
+
   Dir.chdir("test") do
-    args = Dir.glob("**/*.rb")
-    sh "jrubyc", "-t", "../build", *args
+    rel_target = File.join("..", target)
+    files = Dir.glob("**/*.rb")
+    files.each do |file|
+      d = File.join(rel_target, File.dirname(file))
+      mkdir_p d if !File.directory?(d)
+      cp file, File.join(d, File.basename(file))
+    end
   end
 end
 
@@ -62,7 +74,7 @@ VERSIONS = {
 
 namespace :vendor do
   file "vendor/jar" do |t|
-    mkdir_p t.name
+    mkdir_p t.name if !File.directory?(t.name)
   end
 
   # Download jruby.jar
@@ -126,13 +138,20 @@ namespace :package do
     end # package:monolith:tar
 
     task :jar => monolith_deps do
-      mkdir_p "build-jar"
+      builddir = "build/monolith-jar"
+      mkdir_p builddir if !File.directory?(builddir)
 
       # Unpack all the 3rdparty jars and any jars in gems
       Dir.glob("vendor/{bundle,jar}/**/*.jar").each do |jar|
-        puts "=> Unpacking #{jar} into build-jar/"
-        Dir.chdir("build-jar") do 
-          sh "jar xf ../#{jar}"
+        if jar =~ /sigar.*\.jar$/
+          puts "=> Skipping #{jar} (sigar not needed)"
+          next
+        end
+
+        puts "=> Unpacking #{jar} into #{builddir}/"
+        relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." })
+        Dir.chdir(builddir) do 
+          sh "jar xf #{relative_path}/#{jar}"
         end
       end
 
@@ -148,12 +167,11 @@ namespace :package do
       # Purge any extra files we don't need in META-INF (like manifests and
       # jar signatures)
       ["INDEX.LIST", "MANIFEST.MF", "ECLIPSEF.RSA", "ECLIPSEF.SF"].each do |file|
-        File.delete(File.join("build-jar", "META-INF", file)) rescue nil
+        File.delete(File.join(builddir, "META-INF", file)) rescue nil
       end
-      #FileUtils.rm_r(File.join("build-jar", "META-INF")) rescue nil
 
       output = "logstash-#{LOGSTASH_VERSION}-monolithic.jar"
-      sh "jar cfe #{output} logstash.runner -C build-jar ."
+      sh "jar cfe #{output} logstash.runner -C #{builddir} ."
 
       jar_update_args = []
 
@@ -165,11 +183,11 @@ namespace :package do
       gem_dirs = %w{bin doc gems specifications}
       gem_root = File.join(%w{vendor bundle jruby 1.8})
       # for each dir, build args: -C vendor/bundle/jruby/1.8 bin, etc
-      gem_jar_args = gem_dirs.collect { |dir| ["-C", gem_root, dir ] }.flatten
+      gem_jar_args = gem_dirs.collect { |d| ["-C", gem_root, d ] }.flatten
       jar_update_args += gem_jar_args
 
       # Add compiled our compiled ruby code
-      jar_update_args += %w{ -C build . }
+      jar_update_args += %w{ -C build/ruby . }
 
       # Add web stuff
       jar_update_args += %w{ -C lib logstash/web/public }
@@ -188,14 +206,15 @@ namespace :package do
   end # namespace monolith
 
   task :jar => [ "vendor:jruby", "vendor:gems", "compile" ] do
-    builddir = "build-jar-thin"
-    mkdir_p builddir
+    builddir = "build/thin-jar"
+    mkdir_p builddir if !File.directory?(builddir)
 
     # Unpack jruby
+    relative_path = File.join(builddir.split(File::SEPARATOR).collect { |a| ".." })
     Dir.glob("vendor/jar/jruby-complete-1.6.0.jar").each do |jar|
       puts "=> Unpacking #{jar} into #{builddir}/"
       Dir.chdir(builddir) do 
-        sh "jar xf ../#{jar}"
+        sh "jar xf #{relative_path}/#{jar}"
       end
     end
 
@@ -258,9 +277,13 @@ task :doccopy => [:require_output_env] do
 
   Dir.glob("docs/**/*").each do |doc|
     dir = File.join(ENV["output"], File.dirname(doc).gsub(/docs\/?/, ""))
-    mkdir_p dir
-    puts "Copy #{doc} => #{dir}"
-    cp(doc, dir)
+    mkdir_p dir if !File.directory?(dir)
+    if File.directory?(doc)
+      mkdir_p doc
+    else
+      puts "Copy #{doc} => #{dir}"
+      cp(doc, dir)
+    end
   end
 end
 

VERSION.rb

@@ -0,0 +1 @@
+LOGSTASH_VERSION = "1.0.6"

docs/extending/example-add-a-new-filter.md

@@ -0,0 +1,117 @@
+---
+title: How to extend - logstash
+layout: content_right
+---
+# Add a new filter
+
+This document shows you how to add a new filter to logstash.
+
+For a general overview of how to add a new plugin, see [the extending
+logstash](.) overview.
+
+## Write code.
+
+Let's write a 'hello world' filter. This filter will replace the 'message' in
+the event with "Hello world!"
+
+First, logstash expects plugins in a certain directory structure: logstash/TYPE/PLUGIN_NAME.rb
+
+Since we're creating a filter, let's mkdir this:
+
+    mkdir -p logstash/filters/
+    cd logstash/filters
+
+Now add the code:
+
+    # Call this file 'foo.rb' (in logstash/filters, as above)
+    require "logstash/filters/base"
+    require "logstash/namespace"
+
+    class LogStash::Filters::Foo < LogStash::Filters::Base
+
+      # Setting the config_name here is required. This is how you
+      # configure this filter from your logstash config.
+      #
+      # filter {
+      #   foo { ... }
+      # }
+      config_name "foo"
+
+      # Replace the message with this value.
+      config :message, :validate => :string
+
+      public
+      def register
+        # nothing to do
+      end # def register
+
+      public
+      def filter(event)
+        if @message
+          # Replace the event message with our message as configured in the
+          # config file.
+          # If no message is specified, do nothing.
+          event.message = @message
+        end
+      end # def filter
+    end # class LogStash::Filters::Foo
+
+## Add it to your configuration
+
+For this simple example, let's just use stdin input and stdout output.
+The config file looks like this:
+
+    input { 
+      stdin { type => "foo" } 
+    }
+    filter {
+      foo {
+        type => "foo"
+        message => "Hello world!"
+      }
+    }
+    output {
+      stdout { }
+    }
+
+Call this file 'example.conf'
+
+## Tell logstash about it.
+
+Depending on how you installed logstash, you have a few ways of including this
+plugin.
+
+You can use the agent flag --pluginpath flag to specify where the root of your
+plugin tree is. In our case, it's the current directory.
+
+    % logstash --pluginpath . -f example.conf
+
+If you use the monolith jar release of logstash, you have an additional option
+- you can include the plugin right in the jar file.
+
+    % jar -uf jar -uf logstash-1.0.5-monolithic.jar logstash/filters/foo.rb
+
+    # Verify it's in the right location in the jar!
+    % jar tf logstash-1.0.5-monolithic.jar | grep foo.rb
+    logstash/filters/foo.rb
+
+    % java -jar logstash-1.0.5-monolithic.jar agent -f example.conf
+
+## Example running
+
+In the example below, I typed in "the quick brown fox" after running the java
+command.
+
+    % java -jar logstash-1.0.5-monolithic.jar agent -f example.conf
+    the quick brown fox   
+    2011-05-12T01:05:09.495000Z stdin://snack.home/: Hello world!
+
+The output is the standard logstash stdout output, but in this case our "the
+quick brown fox" message was replaced with "Hello world!"
+
+All done! :)
+
+
+
+
+

docs/extending/index.md

@@ -6,8 +6,6 @@ layout: content_right
 
 You can add your own input, output, or filter plugins to logstash.
 
-DOCS - TBD
-
 If you're looking to extend logstash today, please look at the existing plugins.
 
 Good examples include:
@@ -38,3 +36,9 @@ Outputs have two methods: register and receive.
 
 * 'register' is called per plugin instantiation. Do any of your initialization here.
 * 'receive' is called when an event gets pushed to your output
+
+## Example: new filter
+
+Learn by example how to [add a new filter to logstash](example-add-a-new-filter)
+
+

docs/installation.md

@@ -33,17 +33,26 @@ CLASSPATH environment variable to include any elasticsearch jar files.
 Using this method to download logstash will install all ruby dependencies.
 
 * You must have jruby already
+* If you use elasticsearch, you'll have to it and its jars add that to the java
+  classpath. ( See below for web interface notes 
+* If you use grok, you'll need libgrok installed.
 
 ### web interface
 
 * You have elasticsearch already
 * You'll need to know the path to your elasticsearch lib directory.
 
-    % CLASSPATH=elasticsearch-0.16.0/lib/*.jar logstash-web
-    >> Thin web server (v1.2.7 codename No Hup)
-    >> Maximum connections set to 1024
-    >> Listening on 0.0.0.0:9292, CTRL+C to stop
+    % CLASSPATH=$(ls /opt/elasticsearch/lib/*.jar | tr '\n' ':')  logstash-web
+   Thin web server (v1.2.7 codename No Hup)
+   Maximum connections set to 1024
+   Listening on 0.0.0.0:9292, CTRL+C to stop
+
+For the above, replace '/opt/elasticsearch/lib' with wherever you downloaded
+and unpacked elasticsearch.
 
 ### agent
 
     % logstash -f youragent.conf
+
+    # Or if you need elasticsearch:
+    % CLASSPATH=$(ls /opt/elasticsearch/lib/*.jar | tr '\n' ':') logstash -f youragent.conf

docs/learn.md

@@ -35,8 +35,8 @@ for such things, that works for me, too.)
 
 logstash releases come in a few flavors.
 
-* [Monolithic jar](http://semicomplete.com/files/logstash/logstash-1.0.4-monolithic.jar)
-* [rubygem](https://github.com/downloads/logstash/releases/logstash-1.0.4.gem)
+* [Monolithic jar](http://semicomplete.com/files/logstash/logstash-1.0.6-monolithic.jar)
+* [rubygem](https://github.com/downloads/logstash/releases/logstash-1.0.6.gem)
 * [`gem install logstash`](http://rubygems.org/gems/logstash)
 
 ## What's next?

lib/logstash/inputs/amqp.rb

@@ -68,7 +68,7 @@ class LogStash::Inputs::Amqp < LogStash::Inputs::Base
       begin
         @bunny.start
 
-        @queue = @bunny.queue(@name)
+        @queue = @bunny.queue(@name, :durable => @durable)
         exchange = @bunny.exchange(@name, :type => @exchange_type.to_sym, :durable => @durable)
         @queue.bind(exchange)
 

lib/logstash/outputs/gelf.rb

@@ -18,36 +18,66 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
   # The GELF chunksize
   config :chunksize, :validate => :number, :default => 1420
 
-  # The GELF message level
-  config :level, :validate => :number, :default => 1
+  # The GELF message level. Dynamic values like %{level} are permitted here;
+  # useful if you want to parse the 'log level' from an event and use that
+  # as the gelf level/severity.
+  #
+  # Values here can be integers [0..7] inclusive or any of 
+  # "debug", "info", "warn", "error", "fatal", "unknown" (case insensitive).
+  # Single-character versions of these are also valid, "d", "i", "w", "e", "f",
+  # "u"
+  config :level, :validate => :string, :default => "INFO"
 
-  # The GELF facility.
+  # The GELF facility. Dynamic values like %{foo} are permitted here; this
+  # is useful if you need to use a value from the event as the facility name.
   config :facility, :validate => :string, :default => "logstash-gelf"
 
   public
   def register
     require "gelf" # rubygem 'gelf'
     option_hash = Hash.new
-    option_hash['level'] = @level
-    option_hash['facility'] = @facility
+    #option_hash['level'] = @level
+    #option_hash['facility'] = @facility
 
-    @gelf = GELF::Notifier.new(@host, @port, @chunksize, option_hash)
+    #@gelf = GELF::Notifier.new(@host, @port, @chunksize, option_hash)
+    @gelf = GELF::Notifier.new(@host, @port, @chunksize)
+
+    # This sets the 'log level' of gelf; since we're forwarding messages, we'll
+    # want to forward *all* messages, so set level to 0 so all messages get
+    # shipped
+    @gelf.level = 0
+
+    @level_map = {
+      "debug" => 7, "d" => 7,
+      "info" => 6, "i" => 6,
+      "warn" => 5, "w" => 5,
+      "error" => 4, "e" => 4,
+      "fatal" => 3, "f" => 3,
+      "unknown" => 1, "u" => 1,
+    }
   end # def register
 
   public
   def receive(event)
+    # We have to make our own hash here because GELF expects a hash
+    # with a specific format.
     m = Hash.new
     m["short_message"] = (event.fields["message"] or event.message)
     m["full_message"] = (event.message)
     m["host"] = event["@source_host"]
     m["file"] = event["@source_path"]
-    m["level"] = 1
 
     event.fields.each do |name, value|
       next if value == nil or value.empty?
       m["#{name}"] = value
     end
+
+    # Allow 'INFO' 'I' or number. for 'level'
+    level = event.sprintf(@level.to_s)
+    m["level"] = (@level_map[level.downcase] || level).to_i
+    m["facility"] = event.sprintf(@facility)
     m["timestamp"] = event.timestamp
+
     @gelf.notify!(m)
   end # def receive
 end # class LogStash::Outputs::Gelf

logstash.gemspec

@@ -1,3 +1,5 @@
+require File.join(File.dirname(__FILE__), "VERSION")  # For LOGSTASH_VERSION
+
 Gem::Specification.new do |spec|
   files = []
   paths = %w{lib examples etc patterns}
@@ -16,7 +18,7 @@ Gem::Specification.new do |spec|
   #rev = %x{svn info}.split("\n").grep(/Revision:/).first.split(" ").last.to_i
   rev = Time.now.strftime("%Y%m%d%H%M%S")
   spec.name = "logstash"
-  spec.version = "1.0.4"
+  spec.version = LOGSTASH_VERSION
   spec.summary = "logstash - log and event management"
   spec.description = "scalable log and event management (search, archive, pipeline)"
   spec.license = "Apache License (2.0)"