diff --git a/README.md b/README.md index 02e7b38e1..312c756fa 100755 --- a/README.md +++ b/README.md @@ -92,8 +92,9 @@ know. ## Project Principles -* Software: Make it work, then make it right, then make it fast. * Community: If a newbie has a bad time, it's a bug. +* Software: Make it work, then make it right, then make it fast. +* Technology: If it doesn't do a thing today, we can make it do it tomorrow. ## Contributing diff --git a/bin/logstash b/bin/logstash index fb19ca867..696edd490 100755 --- a/bin/logstash +++ b/bin/logstash @@ -2,7 +2,7 @@ basedir=$(cd `dirname $0`/..; pwd) if [ -d "$basedir/.git" ] ; then - RUBY=${RUBY=jruby} + RUBY=${RUBY=$(ruby -e 'puts RUBY_ENGINE')} RUBYVER=${RUBYVER=1.9} export GEM_HOME="$basedir/vendor/bundle/${RUBY}/${RUBYVER}" export GEM_PATH= @@ -12,10 +12,10 @@ export RUBYLIB="$basedir/lib" which ruby > /dev/null 2>&1 if [ "$?" -eq 0 -a -z "$USE_JRUBY" ] ; then - ruby "$basedir/lib/logstash/runner.rb" "$@" + exec ruby "$basedir/lib/logstash/runner.rb" "$@" else # No ruby found, fetch JRuby and run. jruby="vendor/jar/jruby-complete-1.7.3.jar" [ ! -f "$jruby" ] && make build-jruby - java -jar "$jruby" "$basedir/lib/logstash/runner.rb" "$@" + exec java -jar "$jruby" "$basedir/lib/logstash/runner.rb" "$@" fi diff --git a/lib/logstash/agent2.rb b/lib/logstash/agent2.rb index 29aaf73b0..e524816cc 100644 --- a/lib/logstash/agent2.rb +++ b/lib/logstash/agent2.rb @@ -43,13 +43,10 @@ class LogStash::Agent2 < Clamp::Command option ["-V", "--version"], :flag, I18n.t("logstash.agent.flag.version") - plugin_paths = [] option ["-p", "--pluginpath"] , "PATH", I18n.t("logstash.agent.flag.pluginpath"), - :attribute_name => :plugin_paths do |value| - plugin_paths << value unless plugin_paths.include?(value) - next plugin_paths - end # -p / --pluginpath + :multivalued => true, + :attribute_name => :plugin_paths # Emit a warning message. def warn(message) @@ -74,6 +71,16 @@ class LogStash::Agent2 < Clamp::Command configure + # You must specify a config_string or config_path + if config_string.nil? && config_path.nil? + puts help + fail(I18n.t("logstash.agent.missing-configuration")) + end + + if @config_path + @config_string = load_config(@config_path) + end + begin pipeline = LogStash::Pipeline.new(@config_string) rescue LoadError => e @@ -93,10 +100,9 @@ class LogStash::Agent2 < Clamp::Command puts I18n.t("logstash.agent.error", :error => e) return 1 rescue => e - puts I18n.t("unexpected-exception", :error => e) + puts I18n.t("oops", :error => e) puts e.backtrace if @logger.debug? return 1 - #puts e.backtrace ensure Stud::untrap("INT", trap_id) unless trap_id.nil? end # def execute @@ -207,4 +213,20 @@ class LogStash::Agent2 < Clamp::Command $LOAD_PATH.unshift(path) end end # def configure_plugin_path + + def load_config(path) + path = File.join(path, "*") if File.directory?(path) + + if Dir.glob(path).length == 0 + fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path)) + end + + config = "" + Dir.glob(path).sort.each do |file| + next unless File.file?(file) + @logger.debug("Reading config file", :file => file) + config << File.read(file) + "\n" + end + return config + end # def load_config end # class LogStash::Agent2 diff --git a/lib/logstash/filters/kv.rb b/lib/logstash/filters/kv.rb index e85ee52a6..8d1a17a9f 100644 --- a/lib/logstash/filters/kv.rb +++ b/lib/logstash/filters/kv.rb @@ -18,7 +18,7 @@ require "logstash/namespace" # * error: REFUSED # # This is great for postfix, iptables, and other types of logs that -# tend towards 'key=value' syntax. +# tend towards 'key=value' syntax. # # Further, this can often be used to parse query parameters like # 'foo=bar&baz=fizz' by setting the field_split to "&" @@ -30,17 +30,23 @@ class LogStash::Filters::KV < LogStash::Filters::Base # values are wrapped in brackets or are terminated by comma (like postfix # logs) # - # Example, to strip '<' '>' and ',' characters from values: - # - # filter { - # kv { - # trim => "<>," + # These characters form a regex character class and thus you must escape special regex + # characters like [ or ] using \. + # + # Example, to strip '<' '>' '[' ']' and ',' characters from values: + # + # filter { + # kv { + # trim => "<>\[\]," # } # } config :trim, :validate => :string # A string of characters to use as delimiters for parsing out key-value pairs. # + # These characters form a regex character class and thus you must escape special regex + # characters like [ or ] using \. + # # #### Example with URL Query Strings # # Example, to split out the args from a url query string such as @@ -48,7 +54,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base # # filter { # kv { - # field_split => "&?" + # field_split => "&?" # } # } # @@ -65,9 +71,12 @@ class LogStash::Filters::KV < LogStash::Filters::Base # A string of characters to use as delimiters for identifying key-value relations. # + # These characters form a regex character class and thus you must escape special regex + # characters like [ or ] using \. + # # Example, to identify key-values such as # 'key1:value1 key2:value2': - # + # # filter { kv { value_split => ":" } } config :value_split, :validate => :string, :default => '=' @@ -85,7 +94,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base # filter { kv { source => "message" } } config :source, :validate => :string, :default => "message" - # The name of the container to put all of the key-value pairs into + # The name of the container to put all of the key-value pairs into # # If this setting is omitted, fields will be written to the root of the # event. @@ -147,7 +156,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base when nil; # Nothing to do when String; kv = parse(value, event, kv) when Array; value.each { |v| kv = parse(v, event, kv) } - else + else @logger.warn("kv filter has no support for this type of data", :type => value.class, :value => value) end # case value diff --git a/lib/logstash/outputs/stdout.rb b/lib/logstash/outputs/stdout.rb index 7378a3ad2..7f2a44e98 100644 --- a/lib/logstash/outputs/stdout.rb +++ b/lib/logstash/outputs/stdout.rb @@ -17,7 +17,7 @@ class LogStash::Outputs::Stdout < LogStash::Outputs::Base config :debug_format, :default => "ruby", :validate => ["ruby", "json", "dots"] # The message to emit to stdout. - config :message, :validate => :string, :default => "%{@timestamp} %{@source}: %{@message}" + config :message, :validate => :string, :default => "%{+yyyy-MM-dd'T'HH:mm:ss.SSSZ} %{host}: %{message}" public def register diff --git a/lib/logstash/pipeline.rb b/lib/logstash/pipeline.rb index 28be931c7..0b5bf1597 100644 --- a/lib/logstash/pipeline.rb +++ b/lib/logstash/pipeline.rb @@ -152,8 +152,8 @@ class LogStash::Pipeline begin output.receive(event) rescue => e - @logger.error("Exception in plugin #{plugin.class}", - "plugin" => plugin.inspect, "exception" => e) + @logger.error("Exception in plugin #{output.class}", + "plugin" => output.inspect, "exception" => e) end end # @outputs.each end # while true diff --git a/lib/logstash/runner.rb b/lib/logstash/runner.rb index 65eca63d3..e92c37189 100644 --- a/lib/logstash/runner.rb +++ b/lib/logstash/runner.rb @@ -78,7 +78,7 @@ class LogStash::Runner end # Avoid running test/unit's at_exit crap - if status.empty? + if status.empty? || status.first.nil? exit(0) else exit(status.first) diff --git a/locales/en.yml b/locales/en.yml index d45b09783..1bdef86bf 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -3,17 +3,17 @@ # > means 'scalar block' but it chomps all newlines. Useful # for unformatted text. en: - unexpected-exception: |- - +------------------------------------------------------- - | An unexpected error occurred. This is probably a bug. - | You can find help with this problem in a few places: - | - | * chat: #logstash IRC channel on freenode irc. To get - | to IRC on the web, go here: http://goo.gl/TI4Ro - | * email: logstash-users@googlegroups.com - | * bug system: https://logstash.jira.com/ - | - +------------------------------------------------------- + oops: |- + +---------------------------------------------------------+ + | An unexpected error occurred. This is probably a bug. | + | You can find help with this problem in a few places: | + | | + | * chat: #logstash IRC channel on freenode irc. | + | IRC via the web: http://goo.gl/TI4Ro | + | * email: logstash-users@googlegroups.com | + | * bug system: https://logstash.jira.com/ | + | | + +---------------------------------------------------------+ The error reported is: %{error} logstash: @@ -23,17 +23,23 @@ en: Plugin: %{plugin} Error: %{error} agent: + missing-configuration: |- + No configuration file was specified. Perhaps you forgot to provide + the '-f yourlogstash.conf' flag? error: |- Error: %{error} interrupted: |- Interrupt received. Shutting down the pipeline. configuration: + file-not-found: |- + No config files found: %{path} + Can you make sure this path is a logstash config file? setting_missing: |- Missing a required setting for the %{plugin} %{type} plugin: %{type} { %{plugin} { - %{setting} => # MISSING + %{setting} => # SETTING MISSING ... } } @@ -62,8 +68,8 @@ en: This is often a permissions issue, or the wrong path was specified? flag: - # Wrap these at 45 chars so they display nicely when clamp emits them in - # an 80-character terminal + # Note: Wrap these at 45 chars so they display nicely when clamp emits + # them in an 80-character terminal config: |- Load the logstash config from a specific file or directory. If a direcory is given, all @@ -75,7 +81,7 @@ en: config-string: |- Use the given string as the configuration data. Same syntax as the config file. If not - input is specified, then 'stdin { type =|- + input is specified, then 'stdin { type => stdin }' is the default input. If no output is specified, then 'stdout { debug => true }}' is default output. diff --git a/spec/support/postwait1.rb b/spec/support/postwait1.rb new file mode 100644 index 000000000..303a3fa8e --- /dev/null +++ b/spec/support/postwait1.rb @@ -0,0 +1,26 @@ +require "test_utils" + +describe "some stuff" do + extend LogStash::RSpec + + config <<-'CONFIG' + filter { + grep { + tags => web + drop => false + add_field => [ "application", "web" ] + } + + mutate { + tags => web + #replace => [ "message", "%{request}" ] + add_field => [ "message", "%{request}" ] + } + } + CONFIG + + sample("tags" => [ "web" ], "request" => "hello") do + insist { subject["tags"] }.include?("web") + insist { subject["message"] } == "hello" + end +end