Merge branch 'master' of github.com:logstash/logstash

CHANGELOG

@@ -1,18 +1,18 @@
 1.2.0 (???)
   # general
-  - THE LOGSTASH EVENT SCHEMA HAS CHANGED.
-      TODO(sissel): Document what this means, etc.
+  - The logstash json schema has changed. It is smaller now.
   - The old logstash web ui has been replaced by Kibana 3. Kibana is a far
     superior search and analytics interface.
   - TODO(sissel): document new conditionals feature (LOGSTASH-661)
   - TODO(sissel): document new field reference syntax (LOGSTASH-1153)
-  - ElasticSearch version 0.90.2 is included.
+  - ElasticSearch version 0.90.3 is included.
   - Many deprecated features have been removed.
     TODO(sissel): Document what these were.
   - 'type' is no longer a required setting on inputs.
   - feature: codecs. Used to implement decoding of events for inputs and
-    encoding of events for outputs.
-      TODO(nickethier): Document how to use and how to hack.
+    encoding of events for outputs. Codecs allow us to separate transport
+    (like tcp, redis, rabbitmq) from serialization (gzip text, json, msgpack,
+    etc).
 
   ## inputs
   - bugfix: gelf: work around gelf parser errors (#476, patch by Chris McCoy)
@@ -32,12 +32,14 @@
     stored as single values in most cases instead of the old behavior of being
     captured as an array of values.
   - removed: the multiline filter is gone. See the multiline codec instead.
+  - new: json_encoder filter (Ralph Meijer)
 
   ## outputs
   - feature: irc: add messages_per_second tunable (LOGSTASH-962)
   - bugfix: emails: restored initial really useful documentation
   - improvement: emails: allow @message, @source, @... in match (LOGSTASH-826,
     LOGSTASH-823) 
+  - feature: mongodb: replica sets are supported (Mathias Gug)
 
 1.1.13 (May 28, 2013)
   ## general

Makefile

@@ -17,7 +17,7 @@ ELASTICSEARCH=vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION)
 GEOIP=vendor/geoip/GeoLiteCity.dat
 GEOIP_URL=http://logstash.objects.dreamhost.com/maxmind/GeoLiteCity-2013-01-18.dat.gz
 KIBANA_URL=https://github.com/elasticsearch/kibana/archive/master.tar.gz
-PLUGIN_FILES=$(shell git ls-files | egrep '^lib/logstash/(inputs|outputs|filters)/[^/]+$$' | egrep -v '/(base|threadable).rb$$|/inputs/ganglia/')
+PLUGIN_FILES=$(shell git ls-files | egrep '^lib/logstash/(inputs|outputs|filters|codecs)/[^/]+$$' | egrep -v '/(base|threadable).rb$$|/inputs/ganglia/')
 QUIET=@
 
 WGET=$(shell which wget 2>/dev/null)
@@ -276,7 +276,7 @@ docgen: $(addprefix build/docs/,$(subst lib/logstash/,,$(subst .rb,.html,$(PLUGI
 build/docs: build
 	-$(QUIET)mkdir $@
 
-build/docs/inputs build/docs/filters build/docs/outputs: | build/docs
+build/docs/inputs build/docs/filters build/docs/outputs build/docs/codecs: | build/docs
 	-$(QUIET)mkdir $@
 
 # bluecloth gem doesn't work on jruby. Use ruby.
@@ -292,6 +292,9 @@ build/docs/outputs/%.html: lib/logstash/outputs/%.rb docs/docgen.rb docs/plugin-
 	$(QUIET)ruby docs/docgen.rb -o build/docs $<
 	$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
 	$(QUIET)sed -i -re 's/%ELASTICSEARCH_VERSION%/$(ELASTICSEARCH_VERSION)/g' $@
+build/docs/codecs/%.html: lib/logstash/codecs/%.rb docs/docgen.rb docs/plugin-doc.html.erb | build/docs/codecs
+	$(QUIET)ruby docs/docgen.rb -o build/docs $<
+	$(QUIET)sed -i -re 's/%VERSION%/$(VERSION)/g' $@
 
 build/docs/%: docs/% lib/logstash/version.rb Makefile
 	@echo "Copying $< (to $@)"

docs/docgen.rb

@@ -8,6 +8,7 @@ $: << File.join(File.dirname(__FILE__), "..", "lib")
 
 require "logstash/config/mixin"
 require "logstash/inputs/base"
+require "logstash/codecs/base"
 require "logstash/filters/base"
 require "logstash/outputs/base"
 require "logstash/version"
@@ -18,7 +19,7 @@ class LogStashConfigDocGenerator
   def initialize
     @rules = {
       COMMENT_RE => lambda { |m| add_comment(m[1]) },
-      /^ *class.*< *LogStash::(Outputs|Filters|Inputs)::(Base|Threadable)/ => \
+      /^ *class.*< *LogStash::(Outputs|Filters|Inputs|Codecs)::(Base|Threadable)/ => \
         lambda { |m| set_class_description },
       /^ *config +[^=].*/ => lambda { |m| add_config(m[0]) },
       /^ *milestone .*/ => lambda { |m| set_milestone(m[0]) },
@@ -184,6 +185,8 @@ class LogStashConfigDocGenerator
       section = "filter"
     elsif klass.ancestors.include?(LogStash::Outputs::Base)
       section = "output"
+    elsif klass.ancestors.include?(LogStash::Codecs::Base)
+      section = "codec"
     end
 
     template_file = File.join(File.dirname(__FILE__), "plugin-doc.html.erb")
@@ -199,6 +202,9 @@ class LogStashConfigDocGenerator
     klassname = LogStash::Config::Registry.registry[@name].to_s
     name = @name
 
+    synopsis_file = File.join(File.dirname(__FILE__), "plugin-synopsis.html.erb")
+    synopsis = ERB.new(File.new(synopsis_file).read, nil, "-").result(binding)
+
     if settings[:output]
       dir = File.join(settings[:output], section + "s")
       path = File.join(dir, "#{name}.html")

docs/generate_index.rb

@@ -17,6 +17,7 @@ end # def plugins
 basedir = ARGV[0]
 docs = {
   "inputs" => plugins(File.join(basedir, "inputs/*.html")),
+  "codecs" => plugins(File.join(basedir, "codecs/*.html")),
   "filters" => plugins(File.join(basedir, "filters/*.html")),
   "outputs" => plugins(File.join(basedir, "outputs/*.html")),
 }

docs/plugin-doc.html.erb

@@ -29,33 +29,19 @@ This plugin provides the following flags:
 
 This is what it might look like in your config file:
 
-<pre><code><%= section %> {
-  <%= name %> {
-<% sorted_attributes.each do |name, config|
-     next if config[:deprecated]
-     if config[:validate].is_a?(Array) 
-       annotation = "string, one of #{config[:validate].inspect}"
-     elsif config[:validate] == :path
-       annotation = "a valid filesystem path"
-     else 
-       annotation = "#{config[:validate]}"
-     end
-
-     if name.is_a?(Regexp)
-       name = "/" + name.to_s.gsub(/^\(\?-mix:/, "").gsub(/\)$/, "") + "/"
-     end
-     if config[:required]
-       annotation += " (required)"
-     else
-       annotation += " (optional)"
-     end
-     annotation += ", default: #{config[:default].inspect}" if config.include?(:default)
--%>
-    <a href="#<%= name %>"><%= name %></a> => ... # <%= annotation %>
-<% end -%>
+<pre><code><% if section == "codec" -%>
+# with an input plugin:
+# you can also use this codec with an output.
+input { 
+  file { 
+    codec =&gt; <%= synopsis.split("\n").map { |l| "  #{l}" }.join("\n") %>
   }
 }
-</code></pre>
+<% else -%>
+<%= section %> {
+  <%= synopsis %>
+}
+<% end -%></code></pre>
 
 <h3> Details </h3>
 

docs/plugin-synopsis.html.erb

@@ -0,0 +1,24 @@
+<%= name %> {
+<% sorted_attributes.each do |name, config|
+   next if config[:deprecated]
+   if config[:validate].is_a?(Array) 
+     annotation = "string, one of #{config[:validate].inspect}"
+   elsif config[:validate] == :path
+     annotation = "a valid filesystem path"
+   else 
+     annotation = "#{config[:validate]}"
+   end
+
+   if name.is_a?(Regexp)
+     name = "/" + name.to_s.gsub(/^\(\?-mix:/, "").gsub(/\)$/, "") + "/"
+   end
+   if config[:required]
+     annotation += " (required)"
+   else
+     annotation += " (optional)"
+   end
+   annotation += ", default: #{config[:default].inspect}" if config.include?(:default)
+-%>
+    <a href="#<%= name %>"><%= name %></a> => ... # <%= annotation %>
+<% end -%>
+}

lib/logstash/agent.rb

@@ -112,7 +112,7 @@ class LogStash::Agent < Clamp::Command
     return 1
   rescue => e
     puts I18n.t("oops", :error => e)
-    puts e.backtrace if @logger.debug?
+    puts e.backtrace if @logger.debug? || $DEBUGLIST.include?("stacktrace")
     return 1
   ensure
     Stud::untrap("INT", trap_id) unless trap_id.nil?
@@ -187,7 +187,7 @@ class LogStash::Agent < Clamp::Command
       @logger.level = :debug
     else
       # Old support for the -v and -vv stuff.
-      if verbosity?.any?
+      if verbosity? && verbosity?.any?
         # this is an array with length of how many times the flag is given
         if verbosity?.length == 1
           @logger.level = :info

lib/logstash/codecs/msgpack.rb

@@ -1,7 +1,7 @@
 require "logstash/codecs/base"
 
 class LogStash::Codecs::Msgpack < LogStash::Codecs::Base
-  config_name "json"
+  config_name "msgpack"
 
   milestone 1
 

lib/logstash/config/config_ast.rb

@@ -206,6 +206,11 @@ module LogStash; module Config; module AST
       return text_value[1...-1].inspect
     end
   end
+  class RegExp < Value
+    def compile
+      return text_value
+    end
+  end
   class Number < Value
     def compile
       return text_value
@@ -267,12 +272,24 @@ module LogStash; module Config; module AST
       # Hack for compiling 'in' support.
       # This really belongs elsewhere, I think.
       cmp = recursive_select(LogStash::Config::AST::ComparisonOperator)
-      if cmp.count == 1 && cmp.first.text_value == "in"
-        # item 'in' list
-        # technically anything that responds to #include? is accepted.
-        item, list = recursive_select(LogStash::Config::AST::RValue)
-        return "(x = #{list.compile}; x.respond_to?(:include?) && x.include?(#{item.compile}))"
-        #return "#{list.compile}.include?(#{item.compile})"
+      if cmp.count == 1 
+        operator = cmp.first.text_value
+        if operator == "in"
+          # item 'in' list
+          # technically anything that responds to #include? is accepted.
+          item, list = recursive_select(LogStash::Config::AST::RValue)
+          return "(x = #{list.compile}; x.respond_to?(:include?) && x.include?(#{item.compile}))"
+          #return "#{list.compile}.include?(#{item.compile})"
+        elsif ["=~", "!~"].include?(operator)
+          item, regexp = recursive_select(LogStash::Config::AST::RValue)
+          if regexp.is_a?(LogStash::Config::AST::String)
+            regexp = "/#{regexp.text_value[1..-1]}/"
+          else
+            regexp = regexp.compile
+          end
+          ::File.write("/tmp/x",  "(#{item.compile} #{operator} #{regexp})")
+          return "(#{item.compile} #{operator} #{regexp})"
+        end
       end
       return "(#{super})"
     end

lib/logstash/config/grammar.rb

@@ -1172,6 +1172,118 @@ module LogStashConfig
     r0
   end
 
+  module Regexp0
+  end
+
+  module Regexp1
+  end
+
+  def _nt_regexp
+    start_index = index
+    if node_cache[:regexp].has_key?(index)
+      cached = node_cache[:regexp][index]
+      if cached
+        cached = SyntaxNode.new(input, index...(index + 1)) if cached == true
+        @index = cached.interval.end
+      end
+      return cached
+    end
+
+    i0, s0 = index, []
+    if has_terminal?('/', false, index)
+      r1 = instantiate_node(SyntaxNode,input, index...(index + 1))
+      @index += 1
+    else
+      terminal_parse_failure('/')
+      r1 = nil
+    end
+    s0 << r1
+    if r1
+      s2, i2 = [], index
+      loop do
+        i3 = index
+        if has_terminal?('\/', false, index)
+          r4 = instantiate_node(SyntaxNode,input, index...(index + 2))
+          @index += 2
+        else
+          terminal_parse_failure('\/')
+          r4 = nil
+        end
+        if r4
+          r3 = r4
+        else
+          i5, s5 = index, []
+          i6 = index
+          if has_terminal?('/', false, index)
+            r7 = instantiate_node(SyntaxNode,input, index...(index + 1))
+            @index += 1
+          else
+            terminal_parse_failure('/')
+            r7 = nil
+          end
+          if r7
+            r6 = nil
+          else
+            @index = i6
+            r6 = instantiate_node(SyntaxNode,input, index...index)
+          end
+          s5 << r6
+          if r6
+            if index < input_length
+              r8 = instantiate_node(SyntaxNode,input, index...(index + 1))
+              @index += 1
+            else
+              terminal_parse_failure("any character")
+              r8 = nil
+            end
+            s5 << r8
+          end
+          if s5.last
+            r5 = instantiate_node(SyntaxNode,input, i5...index, s5)
+            r5.extend(Regexp0)
+          else
+            @index = i5
+            r5 = nil
+          end
+          if r5
+            r3 = r5
+          else
+            @index = i3
+            r3 = nil
+          end
+        end
+        if r3
+          s2 << r3
+        else
+          break
+        end
+      end
+      r2 = instantiate_node(SyntaxNode,input, i2...index, s2)
+      s0 << r2
+      if r2
+        if has_terminal?('/', false, index)
+          r9 = instantiate_node(SyntaxNode,input, index...(index + 1))
+          @index += 1
+        else
+          terminal_parse_failure('/')
+          r9 = nil
+        end
+        s0 << r9
+      end
+    end
+    if s0.last
+      r0 = instantiate_node(LogStash::Config::AST::RegExp,input, i0...index, s0)
+      r0.extend(Regexp1)
+    else
+      @index = i0
+      r0 = nil
+    end
+
+    node_cache[:regexp][start_index] = r0
+
+    r0
+  end
+
   module Number0
   end
 
@@ -2417,8 +2529,13 @@ module LogStashConfig
             if r5
               r0 = r5
             else
-              @index = i0
-              r0 = nil
+              r6 = _nt_regexp
+              if r6
+                r0 = r6
+              else
+                @index = i0
+                r0 = nil
+              end
             end
           end
         end
@@ -2639,44 +2756,44 @@ module LogStashConfig
         r0 = r2
         r0.extend(LogStash::Config::AST::ComparisonOperator)
       else
-        if has_terminal?("<", false, index)
-          r3 = instantiate_node(SyntaxNode,input, index...(index + 1))
-          @index += 1
+        if has_terminal?("<=", false, index)
+          r3 = instantiate_node(SyntaxNode,input, index...(index + 2))
+          @index += 2
         else
-          terminal_parse_failure("<")
+          terminal_parse_failure("<=")
           r3 = nil
         end
         if r3
           r0 = r3
           r0.extend(LogStash::Config::AST::ComparisonOperator)
         else
-          if has_terminal?(">", false, index)
-            r4 = instantiate_node(SyntaxNode,input, index...(index + 1))
-            @index += 1
+          if has_terminal?(">=", false, index)
+            r4 = instantiate_node(SyntaxNode,input, index...(index + 2))
+            @index += 2
           else
-            terminal_parse_failure(">")
+            terminal_parse_failure(">=")
             r4 = nil
           end
           if r4
             r0 = r4
             r0.extend(LogStash::Config::AST::ComparisonOperator)
           else
-            if has_terminal?("<=", false, index)
-              r5 = instantiate_node(SyntaxNode,input, index...(index + 2))
-              @index += 2
+            if has_terminal?("<", false, index)
+              r5 = instantiate_node(SyntaxNode,input, index...(index + 1))
+              @index += 1
             else
-              terminal_parse_failure("<=")
+              terminal_parse_failure("<")
               r5 = nil
             end
             if r5
               r0 = r5
               r0.extend(LogStash::Config::AST::ComparisonOperator)
             else
-              if has_terminal?(">=", false, index)
-                r6 = instantiate_node(SyntaxNode,input, index...(index + 2))
-                @index += 2
+              if has_terminal?(">", false, index)
+                r6 = instantiate_node(SyntaxNode,input, index...(index + 1))
+                @index += 1
               else
-                terminal_parse_failure(">=")
+                terminal_parse_failure(">")
                 r6 = nil
               end
               if r6

lib/logstash/config/grammar.treetop

@@ -78,6 +78,10 @@ grammar LogStashConfig
     )
   end
 
+  rule regexp
+    ( '/' ( '\/' / !'/' . )* '/'  <LogStash::Config::AST::RegExp>)
+  end
+
   rule number
     "-"? [0-9]+ ("." [0-9]*)? 
     <LogStash::Config::AST::Number>
@@ -149,7 +153,7 @@ grammar LogStashConfig
   end
 
   rule rvalue
-    string / number / selector / array / method_call
+    string / number / selector / array / method_call / regexp
   end
 
   rule method_call
@@ -166,7 +170,7 @@ grammar LogStashConfig
   end
 
   rule comparison
-    ("==" / "!=" / "<" / ">" / "<=" / ">=" / "=~" / "!~" / "in")
+    ("==" / "!=" / "<=" / ">=" / "<" / ">" / "=~" / "!~" / "in")
     <LogStash::Config::AST::ComparisonOperator>
   end
 

lib/logstash/filters/json.rb

@@ -20,7 +20,7 @@ class LogStash::Filters::Json < LogStash::Filters::Base
   #       }
   #     }
   #
-  # The above would parse the xml from the @message field
+  # The above would parse the json from the @message field
   config :source, :validate => :string, :required => true
 
   # Define target for placing the data. If this setting is omitted,

lib/logstash/filters/metaevent.rb

@@ -50,7 +50,7 @@ class LogStash::Filters::Metaevent < LogStash::Filters::Base
 
     event = LogStash::Event.new
     event["source"] = Socket.gethostname
-    event["tags"] = [@add_tag
+    event["tags"] = [@add_tag]
 
     @metaevents << event
     @start_event = nil

lib/logstash/inputs/base.rb

@@ -76,7 +76,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
       # charset is deprecated on inputs, but provide backwards compatibility
       # by copying the charset setting into the codec.
 
-      @logger.warn("Copying input's charset setting into codec", :input => self, :codec => @codec)
+      @logger.info("Copying input's charset setting into codec", :input => self, :codec => @codec)
       charset = @charset
       @codec.instance_eval { @charset = charset }
     end

lib/logstash/inputs/lumberjack.rb

@@ -10,6 +10,8 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   config_name "lumberjack"
   milestone 1
 
+  default :codec, "plain"
+
   # the address to listen on.
   config :host, :validate => :string, :default => "0.0.0.0"
 
@@ -40,15 +42,10 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
   public
   def run(output_queue)
     @lumberjack.run do |l|
-      line = l.delete("line")
-      #if file[0,1] == "/"
-        #source = "lumberjack://#{l.delete("host")}#{file}"
-      #else
-        #source = "lumberjack://#{l.delete("host")}/#{file}"
-      #end
-      event = LogStash::Event.new(l)
-      event["message"] = line
-      output_queue << event
+      @codec.decode(l.delete("line")) do |event|
+        l.each { |k,v| event[k] = v }
+        output_queue << event
+      end
     end
   end # def run
 end # class LogStash::Inputs::Lumberjack

lib/logstash/inputs/unix.rb

@@ -72,7 +72,7 @@ class LogStash::Inputs::Unix < LogStash::Inputs::Base
         # or socket dies
         # TODO(sissel): Why do we have a timeout here? What's the point?
         if @data_timeout == -1
-          buf = socket.readpartial(16384
+          buf = socket.readpartial(16384)
         else
           Timeout::timeout(@data_timeout) do
             buf = socket.readpartial(16384)

lib/logstash/pipeline.rb

@@ -130,6 +130,7 @@ class LogStash::Pipeline
   end
 
   def start_filters
+    @filters.each(&:register)
     @filter_threads = @settings["filter-workers"].times.collect do
       Thread.new { filterworker }
     end
@@ -170,11 +171,14 @@ class LogStash::Pipeline
 
   def filterworker
     LogStash::Util::set_thread_name("|worker")
-    @filters.each(&:register)
     begin
       while true
         event = @input_to_filter.pop
-        break if event == LogStash::ShutdownSignal
+        if event == LogStash::ShutdownSignal
+          @input_to_filter.push(event)
+          break
+        end
+
 
         # TODO(sissel): we can avoid the extra array creation here
         # if we don't guarantee ordering of origin vs created events.

spec/conditionals/test.rb

@@ -1,7 +1,29 @@
 require "test_utils"
 
+module ConditionalFancines
+  def description
+    return example.metadata[:example_group][:description_args][0]
+  end
+
+  def conditional(expression, &block)
+    describe(expression) do
+      config <<-CONFIG
+        filter {
+          if #{expression} {
+            mutate { add_tag => "success" }
+          } else {
+            mutate { add_tag => "failure" }
+          }
+        }
+      CONFIG
+      instance_eval(&block)
+    end
+  end
+end
+
 describe "conditionals" do
   extend LogStash::RSpec
+  extend ConditionalFancines
 
   describe "simple" do
     config <<-CONFIG
@@ -127,4 +149,50 @@ describe "conditionals" do
       reject { subject["tags"] }.include?("shouldnotexist")
     end
   end
+
+  describe "operators" do
+    conditional "[message] == 'sample'" do
+      sample("sample") { insist { subject["tags"] }.include?("success") }
+      sample("different") { insist { subject["tags"] }.include?("failure") }
+    end 
+
+    conditional "[message] != 'sample'" do
+      sample("sample") { insist { subject["tags"] }.include?("failure") }
+      sample("different") { insist { subject["tags"] }.include?("success") }
+    end 
+
+    conditional "[message] < 'sample'" do
+      sample("apple") { insist { subject["tags"] }.include?("success") }
+      sample("zebra") { insist { subject["tags"] }.include?("failure") }
+    end
+
+    conditional "[message] > 'sample'" do
+      sample("zebra") { insist { subject["tags"] }.include?("success") }
+      sample("apple") { insist { subject["tags"] }.include?("failure") }
+    end
+
+    conditional "[message] <= 'sample'" do
+      sample("apple") { insist { subject["tags"] }.include?("success") }
+      sample("zebra") { insist { subject["tags"] }.include?("failure") }
+      sample("sample") { insist { subject["tags"] }.include?("success") }
+    end
+
+    conditional "[message] >= 'sample'" do
+      sample("zebra") { insist { subject["tags"] }.include?("success") }
+      sample("sample") { insist { subject["tags"] }.include?("success") }
+      sample("apple") { insist { subject["tags"] }.include?("failure") }
+    end
+
+    conditional "[message] =~ /sample/" do
+      sample("apple") { insist { subject["tags"] }.include?("failure") }
+      sample("sample") { insist { subject["tags"] }.include?("success") }
+      sample("some sample") { insist { subject["tags"] }.include?("success") }
+    end
+
+    conditional "[message] !~ /sample/" do
+      sample("apple") { insist { subject["tags"] }.include?("success") }
+      sample("sample") { insist { subject["tags"] }.include?("failure") }
+      sample("some sample") { insist { subject["tags"] }.include?("failure") }
+    end
+  end
 end

spec/test_utils.rb

@@ -42,6 +42,7 @@ module LogStash
       name = name[0..50] + "..." if name.length > 50
 
       describe "\"#{name}\"" do
+        extend LogStash::RSpec
         let(:pipeline) { LogStash::Pipeline.new(config) }
         let(:event) do
           sample_event = [sample_event] unless sample_event.is_a?(Array)