diff --git a/bin/logstash b/bin/logstash index f14c247dc..5e6fc9192 100755 --- a/bin/logstash +++ b/bin/logstash @@ -5,7 +5,7 @@ $: << File.dirname($0) + "/../lib" require "rubygems" require "logstash/agent" require "optparse" -require "yaml" +#require "yaml" Settings = Struct.new(:config_file, :daemonize, :logfile, :verbose) @@ -52,12 +52,12 @@ if !File.exist?(settings.config_file) $stderr.puts "Config file '#{settings.config_file}' does not exist." end -begin - config = YAML::load_file(settings.config_file) -rescue => e - $stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}" - exit 1 -end +#begin + #config = YAML::load_file(settings.config_file) +#rescue => e + #$stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}" + ##exit 1 +#end if settings.daemonize if Process.fork == nil @@ -78,8 +78,5 @@ elsif settings.daemonize STDERR.reopen(devnull) end -agent = LogStash::Agent.new(config) -#if settings.logfile - #agent.log_to(settings.logfile) -#end +agent = LogStash::Agent.new(settings) agent.run diff --git a/lib/logstash/agent.rb b/lib/logstash/agent.rb index 6641d99a2..bde4fe8d1 100644 --- a/lib/logstash/agent.rb +++ b/lib/logstash/agent.rb @@ -4,6 +4,7 @@ require "logstash/logging" require "logstash/multiqueue" require "logstash/namespace" require "logstash/outputs" +require "logstash/config/file" require "java" require "uri" @@ -17,18 +18,14 @@ class LogStash::Agent attr_reader :filters public - def initialize(config) + def initialize(settings) log_to(STDERR) - @config = config + @settings = settings @threads = {} @outputs = [] @inputs = [] @filters = [] - # Config should have: - # - list of logs to monitor - # - log config - # - where to ship to Thread::abort_on_exception = true end # def initialize @@ -40,6 +37,14 @@ class LogStash::Agent public def run + # Load the config file + p @settings.config_file + config = LogStash::Config::File.new(@settings.config_file) + config.parse do |plugin| + ap plugin + end + exit + if @config["inputs"].length == 0 or @config["outputs"].length == 0 raise "Must have both inputs and outputs configured." end @@ -170,6 +175,7 @@ class LogStash::Agent end @sigchannel.subscribe do |msg| + # TODO(sissel): Make this a function. case msg when :USR1 counts = Hash.new { |h,k| h[k] = 0 } diff --git a/lib/logstash/config/grammar.rl b/lib/logstash/config/grammar.rl index 2c8f484ec..34304c1d4 100644 --- a/lib/logstash/config/grammar.rl +++ b/lib/logstash/config/grammar.rl @@ -81,7 +81,10 @@ require "logstash/namespace" @config[name] += @components } - ws = ([ \t\n])** ; + comment = "#" (any - "\n")* ; + ws = ([ \t\n] | comment)** ; + #ws = ([ \t\n])** ; + # TODO(sissel): Support floating point values? numeric = ( ("+" | "-")? [0-9] :>> [0-9]** ) >mark %stack_numeric; quoted_string = ( @@ -106,9 +109,13 @@ require "logstash/namespace" # } component_implementation = ( - naked_string ws "{" ws - parameters - ws "}" + ( + naked_string ws "{" ws + parameters + ws "}" + ) | ( + naked_string ws "{" ws "}" + ) ) %component_implementation ; component = ( diff --git a/lib/logstash/config/mixin.rb b/lib/logstash/config/mixin.rb index 9a47c7a8c..fb6e7cf38 100644 --- a/lib/logstash/config/mixin.rb +++ b/lib/logstash/config/mixin.rb @@ -49,6 +49,7 @@ module LogStash::Config::Mixin end # def config def dsl_gen + # TODO(sissel): Deprecated. puts "#{@dsl_parent.config_name} { #parent" if @dsl_parent config = [] config << "#{@config_name} { #node" diff --git a/lib/logstash/config/parser.rb b/lib/logstash/config/parser.rb deleted file mode 100644 index b630f196c..000000000 --- a/lib/logstash/config/parser.rb +++ /dev/null @@ -1,65 +0,0 @@ -require "logstash/namespace" -require "logstash/config/grammar" -require "logstash/config/registry" -require "logstash/agent" - -class LogStash::Config::File - def initialize(file) - @file = file - end - - def parse - grammar = LogStash::Config::Grammar.new - grammar.parse(File.new(@file).read) - @config = grammar.config - end - - def each(&block) - #ap @config - - # First level is the components - # Like: - # inputs { - # ... - # } - @config.each do |type, plugin_config_array| - # plugin_config_array has arrays of each component config: - # inputs { - # amqp { ... } - # file { ... } - # file { ... } - # } - plugin_config_array.each do |plugin_config| - yield({ - :type => type, - :plugin => plugin_config.keys.first, - :parameters => plugin_config.values.first - }) - end - end # @config.each - end # def each -end # class LogStash::Config::Parser - -#agent = LogStash::Agent.new({}) -cfg = LogStash::Config::File.new(ARGV[0]) -cfg.parse -registry = LogStash::Config::Registry::registry -cfg.each do |o| - ap ["Config", o] - - # Load the base class for the type given (like inputs/base, or filters/base) - # TODO(sissel): Error handling - require "logstash/#{o[:type]}s/base" - type = registry[o[:type]] - - # Load the plugin itself (inputs/file, outputs/amqp, etc) - # TODO(sissel): Error handling - require "logstash/#{o[:type]}s/#{o[:plugin].downcase}" - config_class = registry[o[:plugin]] - if type - ap :got => [type, config_class] - else - puts "Unknown config #{o.awesome_inspect}" - end -end -#agent.config(cfg) diff --git a/lib/logstash/inputs/beanstalk.rb b/lib/logstash/inputs/beanstalk.rb index eabeffef6..87ed957ba 100644 --- a/lib/logstash/inputs/beanstalk.rb +++ b/lib/logstash/inputs/beanstalk.rb @@ -3,6 +3,9 @@ require "logstash/inputs/base" require "logstash/namespace" class LogStash::Inputs::Beanstalk < LogStash::Inputs::Base + + config_name "beanstalk" + public def initialize(url, type, config={}, &block) super diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 94ccb2fde..6f5ecf103 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -6,7 +6,7 @@ require "socket" # for Socket.gethostname class LogStash::Inputs::File < LogStash::Inputs::Base config_name "file" - #dsl_parent LogStash::Inputs::Base + config :path => :string public def initialize(configs, output_queue) diff --git a/lib/logstash/inputs/internal.rb b/lib/logstash/inputs/internal.rb index 5e4596a01..44b5244be 100644 --- a/lib/logstash/inputs/internal.rb +++ b/lib/logstash/inputs/internal.rb @@ -6,6 +6,8 @@ require "socket" # for Socket.gethostname class LogStash::Inputs::Internal < LogStash::Inputs::Base attr_reader :channel + config_name "internal" + public def initialize(url, type, config={}, &block) super diff --git a/lib/logstash/inputs/stdin.rb b/lib/logstash/inputs/stdin.rb index f2b85844c..f3f58169a 100644 --- a/lib/logstash/inputs/stdin.rb +++ b/lib/logstash/inputs/stdin.rb @@ -4,6 +4,9 @@ require "logstash/namespace" require "socket" # for Socket.gethostname class LogStash::Inputs::Stdin < LogStash::Inputs::Base + + config_name "stdin" + public def register EventMachine::attach($stdin, InputHandler, self) diff --git a/lib/logstash/inputs/stomp.rb b/lib/logstash/inputs/stomp.rb index f48989301..e0403760e 100644 --- a/lib/logstash/inputs/stomp.rb +++ b/lib/logstash/inputs/stomp.rb @@ -3,6 +3,9 @@ require "logstash/namespace" require "logstash/stomp/handler" class LogStash::Inputs::Stomp < LogStash::Inputs::Base + + config_name "stomp" + public def initialize(url, config={}, &block) super diff --git a/lib/logstash/inputs/syslog.rb b/lib/logstash/inputs/syslog.rb index b2d852c97..105a24e04 100644 --- a/lib/logstash/inputs/syslog.rb +++ b/lib/logstash/inputs/syslog.rb @@ -6,6 +6,9 @@ require "logstash/time" # should really use the filters/date.rb bits require "socket" # for Socket.gethostname class LogStash::Inputs::Syslog < LogStash::Inputs::Base + + config_name "syslog" + public def register if !@url.host or !@url.port diff --git a/lib/logstash/inputs/tcp.rb b/lib/logstash/inputs/tcp.rb index 351ddea90..37d71c300 100644 --- a/lib/logstash/inputs/tcp.rb +++ b/lib/logstash/inputs/tcp.rb @@ -4,6 +4,9 @@ require "logstash/namespace" require "socket" # for Socket.gethostname class LogStash::Inputs::Tcp < LogStash::Inputs::Base + + config_name "tcp" + public def initialize(url, type, config={}, &block) super diff --git a/lib/logstash/inputs/twitter.rb b/lib/logstash/inputs/twitter.rb index 8acc5f801..8e6ed6b1a 100644 --- a/lib/logstash/inputs/twitter.rb +++ b/lib/logstash/inputs/twitter.rb @@ -5,6 +5,9 @@ require "logstash/namespace" # TODO(sissel): This could use some refactoring. class LogStash::Inputs::Twitter < LogStash::Inputs::Base + + config_name "twitter" + public def register api_url = "https://stream.twitter.com/1/statuses/filter.json" diff --git a/lib/logstash/outputs/amqp.rb b/lib/logstash/outputs/amqp.rb index f0033e2f2..4de6b0e7d 100644 --- a/lib/logstash/outputs/amqp.rb +++ b/lib/logstash/outputs/amqp.rb @@ -7,6 +7,7 @@ require "cgi" class LogStash::Outputs::Amqp < LogStash::Outputs::Base MQTYPES = [ "fanout", "queue", "topic" ] + config_name "amqp" config :host => :string config :queue_type => :string config :queue_name => :string diff --git a/lib/logstash/outputs/base.rb b/lib/logstash/outputs/base.rb index 0835b6266..5235b8f5e 100644 --- a/lib/logstash/outputs/base.rb +++ b/lib/logstash/outputs/base.rb @@ -11,7 +11,6 @@ class LogStash::Outputs::Base attr_accessor :logger config_name "outputs" - dsl_parent nil public def initialize(url)