Merge branch 'master' into codec-hacking

README.md

@@ -92,8 +92,9 @@ know.
 
 ## Project Principles
 
-* Software: Make it work, then make it right, then make it fast.
 * Community: If a newbie has a bad time, it's a bug.
+* Software: Make it work, then make it right, then make it fast.
+* Technology: If it doesn't do a thing today, we can make it do it tomorrow.
 
 ## Contributing
 

bin/logstash

@@ -2,7 +2,7 @@
 
 basedir=$(cd `dirname $0`/..; pwd)
 if [ -d "$basedir/.git" ] ; then
-  RUBY=${RUBY=jruby}
+  RUBY=${RUBY=$(ruby -e 'puts RUBY_ENGINE')}
   RUBYVER=${RUBYVER=1.9}
   export GEM_HOME="$basedir/vendor/bundle/${RUBY}/${RUBYVER}"
   export GEM_PATH=
@@ -12,10 +12,10 @@ export RUBYLIB="$basedir/lib"
 
 which ruby > /dev/null 2>&1
 if [ "$?" -eq 0 -a -z "$USE_JRUBY" ] ; then
-  ruby "$basedir/lib/logstash/runner.rb" "$@"
+  exec ruby "$basedir/lib/logstash/runner.rb" "$@"
 else
   # No ruby found, fetch JRuby and run.
   jruby="vendor/jar/jruby-complete-1.7.3.jar"
   [ ! -f "$jruby" ] && make build-jruby
-  java -jar "$jruby" "$basedir/lib/logstash/runner.rb" "$@"
+  exec java -jar "$jruby" "$basedir/lib/logstash/runner.rb" "$@"
 fi

lib/logstash/agent2.rb

@@ -43,13 +43,10 @@ class LogStash::Agent2 < Clamp::Command
   option ["-V", "--version"], :flag,
     I18n.t("logstash.agent.flag.version")
 
-  plugin_paths = []
   option ["-p", "--pluginpath"] , "PATH",
     I18n.t("logstash.agent.flag.pluginpath"),
-    :attribute_name => :plugin_paths do |value|
-    plugin_paths << value unless plugin_paths.include?(value)
-    next plugin_paths
-  end # -p / --pluginpath
+    :multivalued => true,
+    :attribute_name => :plugin_paths
 
   # Emit a warning message.
   def warn(message)
@@ -74,6 +71,16 @@ class LogStash::Agent2 < Clamp::Command
 
     configure
 
+    # You must specify a config_string or config_path
+    if config_string.nil? && config_path.nil?
+      puts help
+      fail(I18n.t("logstash.agent.missing-configuration"))
+    end
+
+    if @config_path
+      @config_string = load_config(@config_path)
+    end
+
     begin
       pipeline = LogStash::Pipeline.new(@config_string)
     rescue LoadError => e
@@ -93,10 +100,9 @@ class LogStash::Agent2 < Clamp::Command
     puts I18n.t("logstash.agent.error", :error => e)
     return 1
   rescue => e
-    puts I18n.t("unexpected-exception", :error => e)
+    puts I18n.t("oops", :error => e)
     puts e.backtrace if @logger.debug?
     return 1
-    #puts e.backtrace
   ensure
     Stud::untrap("INT", trap_id) unless trap_id.nil?
   end # def execute
@@ -207,4 +213,20 @@ class LogStash::Agent2 < Clamp::Command
       $LOAD_PATH.unshift(path)
     end
   end # def configure_plugin_path
+
+  def load_config(path)
+    path = File.join(path, "*") if File.directory?(path)
+
+    if Dir.glob(path).length == 0
+      fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
+    end
+
+    config = ""
+    Dir.glob(path).sort.each do |file|
+      next unless File.file?(file)
+      @logger.debug("Reading config file", :file => file)
+      config << File.read(file) + "\n"
+    end
+    return config
+  end # def load_config
 end # class LogStash::Agent2

lib/logstash/filters/kv.rb

@@ -18,7 +18,7 @@ require "logstash/namespace"
 # * error: REFUSED
 #
 # This is great for postfix, iptables, and other types of logs that
-# tend towards 'key=value' syntax. 
+# tend towards 'key=value' syntax.
 #
 # Further, this can often be used to parse query parameters like
 # 'foo=bar&baz=fizz' by setting the field_split to "&"
@@ -30,17 +30,23 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # values are wrapped in brackets or are terminated by comma (like postfix
   # logs)
   #
-  # Example, to strip '<' '>' and ',' characters from values:
-  # 
-  #     filter { 
-  #       kv { 
-  #         trim => "<>,"
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
+  # Example, to strip '<' '>' '[' ']' and ',' characters from values:
+  #
+  #     filter {
+  #       kv {
+  #         trim => "<>\[\],"
   #       }
   #     }
   config :trim, :validate => :string
 
   # A string of characters to use as delimiters for parsing out key-value pairs.
   #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
   # #### Example with URL Query Strings
   #
   # Example, to split out the args from a url query string such as
@@ -48,7 +54,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #
   #     filter {
   #       kv {
-  #         field_split => "&?" 
+  #         field_split => "&?"
   #       }
   #     }
   #
@@ -65,9 +71,12 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
   # A string of characters to use as delimiters for identifying key-value relations.
   #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
   # Example, to identify key-values such as
   # 'key1:value1 key2:value2':
-  # 
+  #
   #     filter { kv { value_split => ":" } }
   config :value_split, :validate => :string, :default => '='
 
@@ -85,7 +94,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     filter { kv { source => "message" } }
   config :source, :validate => :string, :default => "message"
 
-  # The name of the container to put all of the key-value pairs into 
+  # The name of the container to put all of the key-value pairs into
   #
   # If this setting is omitted, fields will be written to the root of the
   # event.
@@ -147,7 +156,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
       when nil; # Nothing to do
       when String; kv = parse(value, event, kv)
       when Array; value.each { |v| kv = parse(v, event, kv) }
-      else 
+      else
         @logger.warn("kv filter has no support for this type of data",
                      :type => value.class, :value => value)
     end # case value

lib/logstash/outputs/stdout.rb

@@ -17,7 +17,7 @@ class LogStash::Outputs::Stdout < LogStash::Outputs::Base
   config :debug_format, :default => "ruby", :validate => ["ruby", "json", "dots"]
 
   # The message to emit to stdout.
-  config :message, :validate => :string, :default => "%{@timestamp} %{@source}: %{@message}"
+  config :message, :validate => :string, :default => "%{+yyyy-MM-dd'T'HH:mm:ss.SSSZ} %{host}: %{message}"
 
   public
   def register

lib/logstash/pipeline.rb

@@ -152,8 +152,8 @@ class LogStash::Pipeline
         begin
           output.receive(event)
         rescue => e
-          @logger.error("Exception in plugin #{plugin.class}",
-                        "plugin" => plugin.inspect, "exception" => e)
+          @logger.error("Exception in plugin #{output.class}",
+                        "plugin" => output.inspect, "exception" => e)
         end
       end # @outputs.each
     end # while true

lib/logstash/runner.rb

@@ -78,7 +78,7 @@ class LogStash::Runner
     end
 
     # Avoid running test/unit's at_exit crap
-    if status.empty?
+    if status.empty? || status.first.nil?
       exit(0)
     else
       exit(status.first)

locales/en.yml

@@ -3,17 +3,17 @@
 #   > means 'scalar block' but it chomps all newlines. Useful 
 #     for unformatted text.
 en:
-  unexpected-exception: |-
-    +-------------------------------------------------------
-    | An unexpected error occurred. This is probably a bug.
-    | You can find help with this problem in a few places:
-    | 
-    | * chat: #logstash IRC channel on freenode irc. To get
-    |         to IRC on the web, go here: http://goo.gl/TI4Ro
-    | * email: logstash-users@googlegroups.com 
-    | * bug system: https://logstash.jira.com/
-    |
-    +-------------------------------------------------------
+  oops: |-
+    +---------------------------------------------------------+
+    | An unexpected error occurred. This is probably a bug.   |
+    | You can find help with this problem in a few places:    |
+    |                                                         |
+    | * chat: #logstash IRC channel on freenode irc.          |
+    |     IRC via the web: http://goo.gl/TI4Ro                |
+    | * email: logstash-users@googlegroups.com                |
+    | * bug system: https://logstash.jira.com/                |
+    |                                                         |
+    +---------------------------------------------------------+
     The error reported is: 
       %{error}
   logstash:
@@ -23,17 +23,23 @@ en:
           Plugin: %{plugin}
           Error: %{error}
     agent:
+      missing-configuration: |-
+        No configuration file was specified. Perhaps you forgot to provide
+        the '-f yourlogstash.conf' flag?
       error: |-
         Error: %{error}
       interrupted: |-
         Interrupt received. Shutting down the pipeline.
       configuration:
+        file-not-found: |-
+          No config files found: %{path}
+          Can you make sure this path is a logstash config file?
         setting_missing: |-
           Missing a required setting for the %{plugin} %{type} plugin:
 
             %{type} {
               %{plugin} {
-                %{setting} => # MISSING
+                %{setting} => # SETTING MISSING
                 ...
               }
             }
@@ -62,8 +68,8 @@ en:
           This is often a permissions issue, or the wrong 
           path was specified?
       flag:
-        # Wrap these at 45 chars so they display nicely when clamp emits them in
-        # an 80-character terminal
+        # Note: Wrap these at 45 chars so they display nicely when clamp emits
+        # them in an 80-character terminal
         config: |-
           Load the logstash config from a specific file
           or directory.  If a direcory is given, all
@@ -75,7 +81,7 @@ en:
         config-string: |-
           Use the given string as the configuration
           data. Same syntax as the config file. If not
-          input is specified, then 'stdin { type =|-
+          input is specified, then 'stdin { type =>
           stdin }' is the default input. If no output
           is specified, then 'stdout { debug => true
           }}' is default output.

spec/support/postwait1.rb

@@ -0,0 +1,26 @@
+require "test_utils"
+
+describe "some stuff" do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grep {
+          tags => web
+          drop => false
+          add_field => [ "application", "web" ]
+      }
+     
+      mutate {
+          tags => web
+          #replace => [ "message", "%{request}" ]
+          add_field => [ "message", "%{request}" ]
+      }
+    }
+  CONFIG
+
+  sample("tags" => [ "web" ], "request" => "hello") do
+    insist { subject["tags"] }.include?("web")
+    insist { subject["message"] } == "hello"
+  end
+end