Merge remote branch 'origin/issue/17' into experimental/petef-jruby

bin/logstash

@@ -5,7 +5,7 @@ $: << File.dirname($0) + "/../lib"
 require "rubygems"
 require "logstash/agent"
 require "optparse"
-require "yaml"
+#require "yaml"
 
 Settings = Struct.new(:config_file, :daemonize, :logfile, :verbose)
 
@@ -52,12 +52,12 @@ if !File.exist?(settings.config_file)
   $stderr.puts "Config file '#{settings.config_file}' does not exist."
 end
 
-begin
+#begin
-  config = YAML::load_file(settings.config_file)
+  #config = YAML::load_file(settings.config_file)
-rescue => e
+#rescue => e
-  $stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}"
+  #$stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}"
-  exit 1
+  ##exit 1
-end
+#end
 
 if settings.daemonize
   if Process.fork == nil
@@ -78,8 +78,5 @@ elsif settings.daemonize
   STDERR.reopen(devnull)
 end
 
-agent = LogStash::Agent.new(config)
+agent = LogStash::Agent.new(settings)
-#if settings.logfile
-  #agent.log_to(settings.logfile)
-#end
 agent.run 

lib/logstash/agent.rb

@@ -4,6 +4,7 @@ require "logstash/logging"
 require "logstash/multiqueue"
 require "logstash/namespace"
 require "logstash/outputs"
+require "logstash/config/file"
 require "java"
 require "uri"
 
@@ -17,18 +18,14 @@ class LogStash::Agent
   attr_reader :filters
 
   public
-  def initialize(config)
+  def initialize(settings)
     log_to(STDERR)
 
-    @config = config
+    @settings = settings
     @threads = {}
     @outputs = []
     @inputs = []
     @filters = []
-    # Config should have:
-    # - list of logs to monitor
-    #   - log config
-    # - where to ship to
 
     Thread::abort_on_exception = true
   end # def initialize
@@ -40,6 +37,14 @@ class LogStash::Agent
 
   public
   def run
+    # Load the config file
+    p @settings.config_file
+    config = LogStash::Config::File.new(@settings.config_file)
+    config.parse do |plugin|
+      ap plugin
+    end
+    exit
+    
     if @config["inputs"].length == 0 or @config["outputs"].length == 0
       raise "Must have both inputs and outputs configured."
     end
@@ -170,6 +175,7 @@ class LogStash::Agent
     end
 
     @sigchannel.subscribe do |msg|
+      # TODO(sissel): Make this a function.
       case msg
       when :USR1
         counts = Hash.new { |h,k| h[k] = 0 }

lib/logstash/config/grammar.rl

@@ -81,7 +81,10 @@ require "logstash/namespace"
     @config[name] += @components
   }
 
-  ws = ([ \t\n])** ;
+  comment = "#" (any - "\n")* ;
+  ws = ([ \t\n] | comment)** ;
+  #ws = ([ \t\n])** ;
+
   # TODO(sissel): Support floating point values?
   numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
   quoted_string = ( 
@@ -106,9 +109,13 @@ require "logstash/namespace"
   # }
 
   component_implementation = (
+    (
       naked_string ws "{" ws
         parameters
       ws "}"
+    ) | ( 
+      naked_string ws "{" ws "}" 
+    )
   ) %component_implementation ; 
 
   component = (

lib/logstash/config/mixin.rb

@@ -49,6 +49,7 @@ module LogStash::Config::Mixin
     end # def config
 
     def dsl_gen
+      # TODO(sissel): Deprecated.
       puts "#{@dsl_parent.config_name} { #parent" if @dsl_parent
       config = []
       config << "#{@config_name} { #node"

lib/logstash/config/parser.rb

@@ -1,65 +0,0 @@
-require "logstash/namespace"
-require "logstash/config/grammar"
-require "logstash/config/registry"
-require "logstash/agent"
-
-class LogStash::Config::File
-  def initialize(file)
-    @file = file
-  end
-
-  def parse
-    grammar = LogStash::Config::Grammar.new
-    grammar.parse(File.new(@file).read)
-    @config = grammar.config
-  end
-
-  def each(&block)
-    #ap @config
-
-    # First level is the components
-    # Like:
-    #   inputs {
-    #     ...
-    #   }
-    @config.each do |type, plugin_config_array|
-      # plugin_config_array has arrays of each component config:
-      # inputs {
-      #   amqp { ... }
-      #   file { ... }
-      #   file { ... }
-      # }
-      plugin_config_array.each do |plugin_config|
-        yield({
-          :type => type,
-          :plugin => plugin_config.keys.first,
-          :parameters => plugin_config.values.first
-        })
-      end
-    end # @config.each
-  end # def each
-end #  class LogStash::Config::Parser
-
-#agent = LogStash::Agent.new({})
-cfg = LogStash::Config::File.new(ARGV[0])
-cfg.parse
-registry = LogStash::Config::Registry::registry
-cfg.each do |o|
-  ap ["Config", o]
-
-  # Load the base class for the type given (like inputs/base, or filters/base)
-  # TODO(sissel): Error handling
-  require "logstash/#{o[:type]}s/base"
-  type = registry[o[:type]]
-
-  # Load the plugin itself (inputs/file, outputs/amqp, etc)
-  # TODO(sissel): Error handling
-  require "logstash/#{o[:type]}s/#{o[:plugin].downcase}"
-  config_class = registry[o[:plugin]]
-  if type
-    ap :got => [type, config_class]
-  else
-    puts "Unknown config #{o.awesome_inspect}"
-  end
-end
-#agent.config(cfg)

lib/logstash/inputs/beanstalk.rb

@@ -3,6 +3,9 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 
 class LogStash::Inputs::Beanstalk < LogStash::Inputs::Base
+
+  config_name "beanstalk"
+
   public
   def initialize(url, type, config={}, &block)
     super

lib/logstash/inputs/file.rb

@@ -6,7 +6,7 @@ require "socket" # for Socket.gethostname
 class LogStash::Inputs::File < LogStash::Inputs::Base
 
   config_name "file"
-  #dsl_parent LogStash::Inputs::Base
+  config :path => :string
 
   public
   def initialize(configs, output_queue)

lib/logstash/inputs/internal.rb

@@ -6,6 +6,8 @@ require "socket" # for Socket.gethostname
 class LogStash::Inputs::Internal < LogStash::Inputs::Base
   attr_reader :channel
 
+  config_name "internal"
+
   public
   def initialize(url, type, config={}, &block)
     super

lib/logstash/inputs/stdin.rb

@@ -4,6 +4,9 @@ require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Stdin < LogStash::Inputs::Base
+
+  config_name "stdin"
+
   public
   def register
     EventMachine::attach($stdin, InputHandler, self)

lib/logstash/inputs/stomp.rb

@@ -3,6 +3,9 @@ require "logstash/namespace"
 require "logstash/stomp/handler"
 
 class LogStash::Inputs::Stomp < LogStash::Inputs::Base
+
+  config_name "stomp"
+
   public
   def initialize(url, config={}, &block)
     super

lib/logstash/inputs/syslog.rb

@@ -6,6 +6,9 @@ require "logstash/time" # should really use the filters/date.rb bits
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Syslog < LogStash::Inputs::Base
+
+  config_name "syslog"
+
   public
   def register
     if !@url.host or !@url.port

lib/logstash/inputs/tcp.rb

@@ -4,6 +4,9 @@ require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::Tcp < LogStash::Inputs::Base
+
+  config_name "tcp"
+
   public
   def initialize(url, type, config={}, &block)
     super

lib/logstash/inputs/twitter.rb

@@ -5,6 +5,9 @@ require "logstash/namespace"
 
 # TODO(sissel): This could use some refactoring.
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
+
+  config_name "twitter"
+
   public
   def register
     api_url = "https://stream.twitter.com/1/statuses/filter.json"

lib/logstash/outputs/amqp.rb

@@ -7,6 +7,7 @@ require "cgi"
 class LogStash::Outputs::Amqp < LogStash::Outputs::Base
   MQTYPES = [ "fanout", "queue", "topic" ]
 
+  config_name "amqp"
   config :host => :string
   config :queue_type => :string
   config :queue_name => :string

lib/logstash/outputs/base.rb

@@ -11,7 +11,6 @@ class LogStash::Outputs::Base
   attr_accessor :logger
 
   config_name "outputs"
-  dsl_parent nil
 
   public
   def initialize(url)