Merge pull request #252 from arsenerei/master

add experimental YAML configuration support

lib/logstash/agent.rb

@@ -1,4 +1,5 @@
 require "logstash/config/file"
+require "logstash/config/file/yaml"
 require "logstash/filterworker"
 require "logstash/logging"
 require "logstash/sized_queue"
@@ -34,6 +35,7 @@ class LogStash::Agent
     log_to(STDERR)
     @config_path = nil
     @config_string = nil
+    @is_yaml = false
     @logfile = nil
 
     # flag/config defaults
@@ -252,13 +254,25 @@ class LogStash::Agent
 
       concatconfig = []
       paths.each do |path|
-        concatconfig << File.new(path).read
+        file = File.new(path)
+        if File.extname(file) == '.yaml'
+          # assume always YAML if even one file is
+          @is_yaml = true
+        end
+        concatconfig << file.read
       end
-      config = LogStash::Config::File.new(nil, concatconfig.join("\n"))
+      config_data = concatconfig.join("\n")
     else # @config_string
       # Given a config string by the user (via the '-e' flag)
-      config = LogStash::Config::File.new(nil, @config_string)
+      config_data = @config_string
     end
+
+    if @is_yaml
+      config = LogStash::Config::File::Yaml.new(nil, config_data)
+    else
+      config = LogStash::Config::File.new(nil, config_data)
+    end
+
     config.logger = @logger
     config
   end

lib/logstash/config/file.rb

@@ -18,18 +18,24 @@ class LogStash::Config::File
     end
   end # def initialize
 
+  def _get_config_data
+    if @string.nil?
+      File.new(@path).read
+    else
+      @string
+    end
+  end
+
+  def _get_config(data)
+    grammar = LogStash::Config::Grammar.new
+    grammar.parse(data)
+    grammar.config
+  end
+
   public
   def parse
-    grammar = LogStash::Config::Grammar.new
+    @config = _get_config(_get_config_data);
 
-    if @string.nil?
-      grammar.parse(File.new(@path).read)
-    else
-      grammar.parse(@string)
-    end
-
-    @config = grammar.config
-    
     registry = LogStash::Config::Registry::registry
     each do |o|
       # Load the base class for the type given (like inputs/base, or filters/base)

lib/logstash/config/file/yaml.rb

@@ -0,0 +1,8 @@
+require "logstash/config/file"
+require "yaml"
+
+class LogStash::Config::File::Yaml < LogStash::Config::File
+  def _get_config(data)
+      return YAML.load(data)
+  end
+end

lib/logstash/config/mixin.rb

@@ -302,23 +302,29 @@ module LogStash::Config::Mixin
       elsif validator.is_a?(Symbol)
         # TODO(sissel): Factor this out into a coersion method?
         # TODO(sissel): Document this stuff.
+        value = hash_or_array(value)
+
         case validator
           when :hash
-            if value.size % 2 == 1
-              return false, "This field must contain an even number of items, got #{value.size}"
-            end
+            if value.is_a?(Hash)
+              result = value
+            else
+              if value.size % 2 == 1
+                return false, "This field must contain an even number of items, got #{value.size}"
+              end
 
-            # Convert the array the config parser produces into a hash.
-            result = {}
-            value.each_slice(2) do |key, value|
-              entry = result[key]
-              if entry.nil?
-                result[key] = value
-              else
-                if entry.is_a?(Array)
-                  entry << value
+              # Convert the array the config parser produces into a hash.
+              result = {}
+              value.each_slice(2) do |key, value|
+                entry = result[key]
+                if entry.nil?
+                  result[key] = value
                 else
-                  result[key] = [entry, value]
+                  if entry.is_a?(Array)
+                    entry << value
+                  else
+                    result[key] = [entry, value]
+                  end
                 end
               end
             end
@@ -342,11 +348,18 @@ module LogStash::Config::Mixin
               return false, "Expected boolean, got #{value.inspect}"
             end
 
-            if value.first !~ /^(true|false)$/
-              return false, "Expected boolean 'true' or 'false', got #{value.first.inspect}"
-            end
+            bool_value = value.first
+            if !!bool_value == bool_value
+              # is_a does not work for booleans
+              # we have Boolean and not a string
+              result = bool_value
+            else
+              if bool_value !~ /^(true|false)$/
+                return false, "Expected boolean 'true' or 'false', got #{bool_value.inspect}"
+              end
 
-            result = (value.first == "true")
+              result = (bool_value == "true")
+            end
           when :ipaddr
             if value.size > 1 # only one value wanted
               return false, "Expected IPaddr, got #{value.inspect}"
@@ -376,5 +389,12 @@ module LogStash::Config::Mixin
       # Return the validator for later use, like with type coercion.
       return true, result
     end # def validate_value
+
+    def hash_or_array(value)
+      if !value.is_a?(Hash)
+        value = [*value] # coerce scalar to array if necessary
+      end
+      return value
+    end
   end # module LogStash::Config::DSL
 end # module LogStash::Config

spec/examples/parse-apache-logs-yaml.rb

@@ -0,0 +1,63 @@
+require "test_utils"
+
+describe "apache common log format" do
+  extend LogStash::RSpec
+
+  # The logstash config goes here.
+  # At this time, only filters are supported.
+  config_yaml <<-CONFIG
+    filter:
+      - grok:
+          pattern: "%{COMBINEDAPACHELOG}"
+          singles: true
+      - date:
+          timestamp: "dd/MMM/yyyy:HH:mm:ss Z"
+  CONFIG
+
+  # Here we provide a sample log event for the testing suite.
+  #
+  # Any filters you define above will be applied the same way the logstash
+  # agent performs. Inside the 'sample ... ' block the 'subject' will be
+  # a LogStash::Event object for you to inspect and verify for correctness.
+  sample '198.151.8.4 - - [29/Aug/2012:20:17:38 -0400] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1"' do
+
+    # These 'insist' and 'reject' calls use my 'insist' rubygem.
+    # See http://rubydoc.info/gems/insist for more info.
+
+    # Require that grok does not fail to parse this event.
+    reject { subject["@tags"] }.include?("_grokparsefailure")
+
+    # Ensure that grok captures certain expected fields.
+    insist { subject }.include?("agent")
+    insist { subject }.include?("bytes")
+    insist { subject }.include?("clientip")
+    insist { subject }.include?("httpversion")
+    insist { subject }.include?("timestamp")
+    insist { subject }.include?("verb")
+    insist { subject }.include?("response")
+    insist { subject }.include?("request")
+
+    # Ensure that those fields match expected values from the event.
+    insist { subject["clientip"] } == "198.151.8.4"
+    insist { subject["timestamp"] } == "29/Aug/2012:20:17:38 -0400"
+    insist { subject["verb"] } == "GET"
+    insist { subject["request"] } == "/favicon.ico"
+    insist { subject["httpversion"] } == "1.1"
+    insist { subject["response"] } == "200"
+    insist { subject["bytes"] } == "3638"
+    insist { subject["referrer"] } == '"-"'
+    insist { subject["agent"] } == "\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1\""
+
+    # Verify date parsing
+    insist { subject.timestamp } == "2012-08-30T00:17:38.000Z"
+  end
+
+  sample '61.135.248.195 - - [26/Sep/2012:11:49:20 -0400] "GET /projects/keynav/ HTTP/1.1" 200 18985 "" "Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )"' do
+    reject { subject["@tags"] }.include?("_grokparsefailure")
+    insist { subject["clientip"] } == "61.135.248.195"
+  end
+
+  sample '72.14.164.185 - - [25/Sep/2012:12:05:02 -0400] "GET /robots.txt HTTP/1.1" 200 - "www.brandimensions.com" "BDFetch"' do
+    reject { subject["@tags"] }.include?("_grokparsefailure")
+  end
+end

spec/test_utils.rb

@@ -1,4 +1,5 @@
 require "insist"
+require "logstash/agent"
 require "logstash/event"
 require "insist"
 require "stud/try"
@@ -18,6 +19,11 @@ module LogStash
       @config_str = configstr
     end # def config
 
+    def config_yaml(configstr)
+      @config_str = configstr
+      @is_yaml = true
+    end
+
     def type(default_type)
       @default_type = default_type
     end
@@ -30,8 +36,7 @@ module LogStash
     def sample(event, &block)
       default_type = @default_type || "default"
       default_tags = @default_tags || []
-      require "logstash/config/file"
-      config = LogStash::Config::File.new(nil, @config_str)
+      config = get_config
       agent = LogStash::Agent.new
       @inputs, @filters, @outputs = agent.instance_eval { parse_config(config) }
       [@inputs, @filters, @outputs].flatten.each do |plugin|
@@ -95,8 +100,7 @@ module LogStash
     end # def sample
 
     def input(&block)
-      require "logstash/config/file"
-      config = LogStash::Config::File.new(nil, @config_str)
+      config = get_config
       agent = LogStash::Agent.new
       it "looks good" do
         inputs, filters, outputs = agent.instance_eval { parse_config(config) }
@@ -104,6 +108,16 @@ module LogStash
       end
     end # def input
 
+    def get_config
+      if @is_yaml
+        require "logstash/config/file/yaml"
+        config = LogStash::Config::File::Yaml.new(nil, @config_str)
+      else
+        require "logstash/config/file"
+        config = LogStash::Config::File.new(nil, @config_str)
+      end
+    end # def get_config
+
     def agent(&block)
       @agent_count ||= 0
       require "logstash/agent"