merge issue/17 branch

.gitignore

@@ -1,4 +1,4 @@
 .*.swp
 *.gem
 local
-test/setup
+test/setup/elasticsearch/elasticsearch-*

etc/logstash-jruby-test.yaml

@@ -3,10 +3,13 @@ inputs:
   linux-syslog:
   - file:///var/log/messages
 filters:
+<<<<<<< HEAD
 - grok:
     linux-syslog:
       patterns:
       - %{SYSLOGLINE}
+=======
+>>>>>>> origin/issue/17
 - grep:
     linux-syslog:
       - match:
@@ -14,4 +17,8 @@ filters:
         add_fields:
           filter_test: hello world
 outputs:
+<<<<<<< HEAD
   - stdout:///debug
+=======
+  - stdout:///
+>>>>>>> origin/issue/17

lib/logstash/config/Makefile

@@ -0,0 +1,3 @@
+
+grammar.rb: grammar.rl
+	ragel -R grammar.rl

lib/logstash/config/grammar.rl

@@ -0,0 +1,182 @@
+require "rubygems"
+require "logstash/namespace"
+
+%%{
+  machine logstash_config;
+
+  action mark {
+    @tokenstack.push(p)
+    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
+  }
+
+  action stack_numeric {
+    startpos = @tokenstack.pop
+    endpos = p
+    token = string[startpos ... endpos]
+    #puts "numeric: #{token}"
+    #puts "numeric?: #{string[startpos,50]}"
+    #puts [startpos, endpos].join(",")
+    @stack << token.to_i
+  }
+
+  action stack_string {
+    startpos = @tokenstack.pop
+    endpos = p
+    token = string[startpos ... endpos]
+    #puts "string: #{token}"
+    @stack << token
+  }
+
+  action stack_quoted_string {
+    startpos = @tokenstack.pop
+    endpos = p
+    token = string[startpos + 1 ... endpos - 1] # Skip quotations
+
+    # Parse escapes.
+    token.gsub(/\\./) { |m| return m[1,1] }
+    #puts "quotedstring: #{token}"
+    @stack << token
+  }
+
+  action array_init {
+    @array = []
+    @stack << :array_init
+  }
+
+  action array_push {
+    while @stack.last != :array_init
+      @array.unshift @stack.pop
+    end
+    @stack.pop # pop :array_init
+
+    @stack << @array
+  }
+
+  action parameter_init {
+    @parameters = Hash.new { |h,k| h[k] = [] }
+  }
+
+  action parameter {
+    value = @stack.pop
+    name = @stack.pop
+    #puts "parameter: #{name} => #{value}"
+    @parameters[name] << value
+  }
+
+  action component_implementation {
+    @components ||= []
+    name = @stack.pop
+    #@components << { :name => name, :parameters => @parameters }
+    @components << { name => @parameters }
+  }
+
+  action component_init {
+    #puts "current component: " + @stack.last
+    @components = []
+  }
+
+  action component {
+    name = @stack.pop
+    @config ||= Hash.new { |h,k| h[k] = [] }
+    @config[name] += @components
+  }
+
+  ws = ([ \t\n])** ;
+  # TODO(sissel): Support floating point values?
+  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
+  quoted_string = ( 
+    ( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" ) |
+    ( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" ) 
+  ) >mark %stack_quoted_string ;
+  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
+  string = ( quoted_string | naked_string ) ;
+
+  array = ( "[" ws string ws ("," ws string ws)* "]" ) >array_init %array_push;
+  parameter_value = ( numeric | string | array );
+  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
+  parameters = ( parameter ( ws parameter )** ) >parameter_init ;
+
+  # Statement:
+  # component {
+  #   component_implementation_name {
+  #     bar => ...
+  #     baz => ...
+  #   }
+  #   ...
+  # }
+
+  component_implementation = (
+    naked_string ws "{" ws
+      parameters
+    ws "}"
+  ) %component_implementation ; 
+
+  component = (
+    naked_string ws "{"
+      >component_init
+      ( ws component_implementation )**
+    ws "}"
+  ) %component ;
+
+  config = (ws component)** ;
+
+  main := config 
+          $err { 
+            # Compute line and column of the cursor (p)
+            puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
+            # TODO(sissel): Note what we were expecting?
+          } ;
+}%%
+
+class LogStash::Config::Grammar
+  attr_accessor :eof
+  attr_accessor :config
+
+  def initialize
+    # BEGIN RAGEL DATA
+    %% write data;
+    # END RAGEL DATA
+
+    @tokenstack = Array.new
+    @stack = Array.new
+
+    @types = Hash.new { |h,k| h[k] = [] }
+    @edges = []
+  end
+
+  def parse(string)
+    data = string.unpack("c*")
+
+    # BEGIN RAGEL INIT
+    %% write init;
+    # END RAGEL INIT
+
+    begin 
+      # BEGIN RAGEL EXEC 
+      %% write exec;
+      # END RAGEL EXEC
+    rescue => e
+      # Compute line and column of the cursor (p)
+      raise e
+    end
+
+    return cs
+  end # def parse
+
+  def line(str, pos)
+    return str[0 .. pos].count("\n") + 1
+  end
+
+  def column(str, pos)
+    return str[0 .. pos].split("\n").last.length
+  end
+end # class LogStash::Config::Grammar
+
+#def parse(string)
+  #cfgparser = LogStash::Config::Grammar.new
+  #result = cfgparser.parse(string)
+  #puts "result %s" % result
+  #ap cfgparser.config
+#end
+
+#parse(File.open(ARGV[0]).read)

lib/logstash/config/mixin.rb

@@ -0,0 +1,82 @@
+
+require "logstash/namespace"
+require "logstash/config/registry"
+
+# This module is meant as a mixin to classes wishing to be configurable from
+# config files
+#
+# The idea is that you can do this:
+#
+# class Foo < LogStash::Config
+#   config "path" => ...
+#   config "tag" => ...
+# end
+#
+# And the config file should let you do:
+#
+# foo {
+#   "path" => ...
+#   "tag" => ...
+# }
+#
+# TODO(sissel): This is not yet fully designed.
+module LogStash::Config::Mixin
+  # This method is called when someone does 'include LogStash::Config'
+  def self.included(base)
+    puts "Configurable class #{base.name}"
+    #
+    # Add the DSL methods to the 'base' given.
+    base.extend(LogStash::Config::Mixin::DSL)
+  end
+
+  module DSL
+    # If name is given, set the name and return it.
+    # If no name given (nil), return the current name.
+    def config_name(name=nil)
+      @config_name = name if !name.nil?
+      LogStash::Config::Registry.registry[name] = self
+      return @config_name
+    end
+
+    # If config is given, add this config.
+    # If no config given (nil), return the current config hash
+    def config(cfg=nil)
+      # cfg should be hash with one entry of { "key" => "val" }
+      @config ||= Hash.new
+      key, value = cfg.to_a.first
+      @config[key] = value
+      return @config
+    end # def config
+
+    def dsl_gen
+      puts "#{@dsl_parent.config_name} { #parent" if @dsl_parent
+      config = []
+      config << "#{@config_name} { #node"
+      config << "  \"somename\":"
+      attrs = []
+      (@config || Hash.new).each do |key, value|
+        attrs << "    #{key} => #{value},"
+      end
+      config += attrs
+      config << "} #node"
+      config = config.collect { |p| "#{@dsl_parent.nil? ? "" : "  "}#{p}" }
+      puts config.join("\n")
+      puts "} #parent" if @dsl_parent
+    end
+
+    # This is called whenever someone subclasses a class that has this mixin.
+    def inherited(subclass)
+      # Copy our parent's config to a subclass.
+      # This method is invoked whenever someone subclasses us, like:
+      # class Foo < Bar ...
+      subconfig = Hash.new
+      if !@config.nil?
+        @config.each do |key, val|
+          puts "#{self}: Sharing config '#{key}' with subclass #{subclass}"
+          subconfig[key] = val
+        end
+      end
+      subclass.instance_variable_set("@config", subconfig)
+    end # def inherited
+  end # module LogStash::Config::DSL
+end # module LogStash::Config

lib/logstash/config/parser.rb

@@ -0,0 +1,65 @@
+require "logstash/namespace"
+require "logstash/config/grammar"
+require "logstash/config/registry"
+require "logstash/agent"
+
+class LogStash::Config::File
+  def initialize(file)
+    @file = file
+  end
+
+  def parse
+    grammar = LogStash::Config::Grammar.new
+    grammar.parse(File.new(@file).read)
+    @config = grammar.config
+  end
+
+  def each(&block)
+    #ap @config
+
+    # First level is the components
+    # Like:
+    #   inputs {
+    #     ...
+    #   }
+    @config.each do |type, plugin_config_array|
+      # plugin_config_array has arrays of each component config:
+      # inputs {
+      #   amqp { ... }
+      #   file { ... }
+      #   file { ... }
+      # }
+      plugin_config_array.each do |plugin_config|
+        yield({
+          :type => type,
+          :plugin => plugin_config.keys.first,
+          :parameters => plugin_config.values.first
+        })
+      end
+    end # @config.each
+  end # def each
+end #  class LogStash::Config::Parser
+
+#agent = LogStash::Agent.new({})
+cfg = LogStash::Config::File.new(ARGV[0])
+cfg.parse
+registry = LogStash::Config::Registry::registry
+cfg.each do |o|
+  ap ["Config", o]
+
+  # Load the base class for the type given (like inputs/base, or filters/base)
+  # TODO(sissel): Error handling
+  require "logstash/#{o[:type]}s/base"
+  type = registry[o[:type]]
+
+  # Load the plugin itself (inputs/file, outputs/amqp, etc)
+  # TODO(sissel): Error handling
+  require "logstash/#{o[:type]}s/#{o[:plugin].downcase}"
+  config_class = registry[o[:plugin]]
+  if type
+    ap :got => [type, config_class]
+  else
+    puts "Unknown config #{o.awesome_inspect}"
+  end
+end
+#agent.config(cfg)

lib/logstash/config/test.conf

@@ -0,0 +1,18 @@
+input {
+  amqp {
+    port => 12345 
+    tag => [ a, b, c ]
+  }
+
+  stomp {
+    port => 12345
+    tag => [ stomp ]
+  }
+}
+
+filter {
+  date { 
+    hello => world
+    hello => "Hello"
+  }
+}

lib/logstash/filters/base.rb

@@ -1,9 +1,14 @@
 require "logstash/namespace"
 require "logstash/logging"
+require "logstash/config/mixin"
 
 class LogStash::Filters::Base
+  include LogStash::Config::Mixin
+
   attr_accessor :logger
 
+  config_name "filter"
+
   public
   def initialize(config = {})
     @logger = LogStash::Logger.new(STDERR)

lib/logstash/filters/date.rb

@@ -3,6 +3,12 @@ require "logstash/namespace"
 require "logstash/time"
 
 class LogStash::Filters::Date < LogStash::Filters::Base
+
+  # Config for date is:
+  #   fieldname: dateformat
+  #   Allow arbitrary keys for this config.
+  config /[A-Za-z0-9_-]+/ => :string
+
   # The 'date' filter will take a value from your event and use it as the
   # event timestamp. This is useful for parsing logs generated on remote
   # servers or for importing old logs.

lib/logstash/filters/grok.rb

@@ -5,6 +5,11 @@ gem "jls-grok", ">=0.3.3209"
 require "grok" # rubygem 'jls-grok'
 
 class LogStash::Filters::Grok < LogStash::Filters::Base
+
+  config :pattern => :string
+  config :patterns_dir => :path
+  config :drop_if_match => :boolean  # googlecode/issue/26
+         
   public
   def initialize(config = {})
     super

lib/logstash/filters/multiline.rb

@@ -7,6 +7,11 @@ require "logstash/filters/base"
 require "logstash/namespace"
 
 class LogStash::Filters::Multiline < LogStash::Filters::Base
+
+  config :pattern => :string
+  config :negate => :boolean
+  config :what => ["previous", "next"]
+
   # The 'date' filter will take a value from your event and use it as the
   # event timestamp. This is useful for parsing logs generated on remote
   # servers or for importing old logs.

lib/logstash/inputs/amqp.rb

@@ -8,6 +8,9 @@ require "cgi"
 class LogStash::Inputs::Amqp < LogStash::Inputs::Base
   MQTYPES = [ "fanout", "queue", "topic" ]
 
+  config_name "amqp"
+  config "pantscon" => :string #LogStash::Config::Path
+
   public
   def initialize(url, type, config={}, &block)
     super

lib/logstash/inputs/base.rb

@@ -1,11 +1,19 @@
 require "logstash/namespace"
 require "logstash/event"
 require "logstash/logging"
+require "logstash/config/mixin"
 require "uri"
 
 class LogStash::Inputs::Base
+  include LogStash::Config::Mixin
   attr_accessor :logger
 
+  config_name "input"
+
+  # Define the basic config
+  config "path" => :string #LogStash::Config::Path
+  config "tag" => :string #LogStash::Config::Array
+
   public
   def initialize(configs, output_queue)
     @logger = LogStash::Logger.new(STDERR)

lib/logstash/inputs/file.rb

@@ -4,6 +4,10 @@ require "logstash/namespace"
 require "socket" # for Socket.gethostname
 
 class LogStash::Inputs::File < LogStash::Inputs::Base
+
+  config_name "file"
+  #dsl_parent LogStash::Inputs::Base
+
   public
   def initialize(configs, output_queue)
     super

lib/logstash/namespace.rb

@@ -3,4 +3,5 @@ module LogStash
   module Outputs; end
   module Filters; end
   module Search; end
+  module Config; end
 end # module LogStash

lib/logstash/outputs/amqp.rb

@@ -7,6 +7,10 @@ require "cgi"
 class LogStash::Outputs::Amqp < LogStash::Outputs::Base
   MQTYPES = [ "fanout", "queue", "topic" ]
 
+  config :host => :string
+  config :queue_type => :string
+  config :queue_name => :string
+
   public
   def initialize(url, config={}, &block)
     super

lib/logstash/outputs/base.rb

@@ -2,11 +2,17 @@ require "cgi"
 require "logstash/event"
 require "logstash/logging"
 require "logstash/namespace"
+require "logstash/config/mixin"
 require "uri"
 
 class LogStash::Outputs::Base
+  include LogStash::Config::Mixin
+
   attr_accessor :logger
 
+  config_name "outputs"
+  dsl_parent nil
+
   public
   def initialize(url)
     @url = url

lib/logstash/outputs/elasticsearch.rb

@@ -4,6 +4,13 @@ require "logstash/outputs/amqp"
 require "logstash/outputs/base"
 
 class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
+
+  # http://host/index/type
+  config :host => :string
+  config :index => :string
+  config :type => :string  
+  # TODO(sissel): Config for river?
+
   public
   def register
     @pending = []

lib/logstash/outputs/nagios.rb

@@ -5,6 +5,8 @@ class LogStash::Outputs::Nagios < LogStash::Outputs::Base
   NAGIOS_CRITICAL = 2
   NAGIOS_WARN = 1
 
+  config :commandfile => :string
+ 
   public
   def initialize(url, config={}, &block)
     super

lib/logstash/outputs/stdout.rb

@@ -2,6 +2,9 @@ require "logstash/outputs/base"
 require "logstash/namespace"
 
 class LogStash::Outputs::Stdout < LogStash::Outputs::Base
+
+  config :debug => :boolean
+
   public
   def initialize(*args)
     super

lib/logstash/outputs/websocket.rb

@@ -3,6 +3,8 @@ require "logstash/namespace"
 require "logstash/outputs/base"
 
 class LogStash::Outputs::Websocket < LogStash::Outputs::Base
+  config :address => :string
+
   public
   def register
     @channel = EventMachine::Channel.new