github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 22:57:16 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote branch 'origin/issue/17' into experimental/petef-jruby

Browse source 
Conflicts:
	lib/logstash/inputs/file.rb
This commit is contained in:
Pete Fritchman 2011-02-20 13:36:41 -08:00
commit f7f574340a
35 changed files with 997 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Rakefile
View file

@ -1,6 +1,7 @@
require 'tempfile'
task :package do
system("make -C lib/logstash/config grammar.rb")
system("gem build logstash.gemspec")
system("gem build logstash-lite.gemspec")
end

6
etc/agent.conf Normal file
View file

@ -0,0 +1,6 @@
input {
file {
path => [ "/var/log/messages", "/var/log/kern.log" ]
type => "linux-syslog"
}
}

59
etc/agent.lgtm.conf Normal file
View file

@ -0,0 +1,59 @@
input {
file {
path => [ "/var/log/messages", "/var/log/kern.log" ]
type => "linux-syslog"
}
file {
path => "/var/log/apache2/access.log"
type => "apache-access"
}
file {
path => "/var/log/apache2/error.log"
type => "apache-error"
}
}
output {
amqp {
host => "myamqpserver"
exchange_type => "fanout"
name => "rawlogs"
}
amqp {
host => "127.0.0.1"
exchange_type => "topic"
name => "logsniff"
}
stdout { }
}
# Filters are applied in the order the appear.
filter {
multiline {
type => "supervisorlogs"
pattern => "^\s"
what => previous
}
multiline {
type => "testing"
pattern => "^\s"
what => previous
}
grok {
type => "linux-syslog"
pattern => ["%{SYSLOG_SUDO}", "%{SYSLOG_KERNEL}", "%{SYSLOGLINE}" ]
}
grok {
type => "nagios"
pattern => "%{NAGIOSLOGLINE}"
}
#date {
#" testing" => fizzle
#}
}

29
lib/logstash/agent.rb
View file

@ -38,12 +38,33 @@ class LogStash::Agent
public
def run
# Load the config file
p @settings.config_file
config = LogStash::Config::File.new(@settings.config_file)
config.parse do |plugin|
ap plugin
end
exit
# 'plugin' is a has containing:
# :type => the base class of the plugin (LogStash::Inputs::Base, etc)
# :plugin => the class of the plugin (LogStash::Inputs::File, etc)
# :parameters => hash of key-value parameters from the config.
type = plugin[:type].config_name # "input" or "filter" etc...
klass = plugin[:plugin]
# Create a new instance of a plugin, called like:
# -> LogStash::Inputs::File.new( params )
instance = klass.new(plugin[:parameters])
case type
when "input"
@inputs << instance
when "filter"
@filters << instance
when "output"
@outputs << instance
else
@logger.error("Unknown config type '#{type}'")
exit 1
end # case type
end # config.parse
exit 0
if @config["inputs"].length == 0 or @config["outputs"].length == 0
raise "Must have both inputs and outputs configured."

84
lib/logstash/config/file.rb Normal file
View file

@ -0,0 +1,84 @@
require "logstash/namespace"
require "logstash/config/grammar"
require "logstash/config/registry"
require "logstash/agent"
class LogStash::Config::File
public
def initialize(file)
@file = file
end
public
def parse
grammar = LogStash::Config::Grammar.new
grammar.parse(File.new(@file).read)
@config = grammar.config
registry = LogStash::Config::Registry::registry
self.each do |o|
# Load the base class for the type given (like inputs/base, or filters/base)
# TODO(sissel): Error handling
tryload o[:type], :base
type = registry[o[:type]]
# Load the plugin itself (inputs/file, outputs/amqp, etc)
# TODO(sissel): Error handling
tryload o[:type], o[:plugin]
plugin = registry[o[:plugin]]
if type.nil?
puts "Unknown config #{o[:type]}/#{o[:plugin]}"
end
yield :type => type, :plugin => plugin, :parameters => o[:parameters]
end
end # def parse
public
def tryload(parent, child)
child = child.downcase if child.is_a? String
begin
loaded = (require "logstash/#{parent}s/#{child}")
#if loaded
#puts "Loading logstash/#{parent}s/#{child}"
#end
rescue => e
if child == :base
$stderr.puts "Failure loading base class '#{parent}': #{e.inspect}"
else
$stderr.puts "Failure loading plugin #{parent}s/#{child}: #{e.inspect}"
end
raise e
end
end # def tryload
public
def each(&block)
#ap @config
# First level is the components
# Like:
# inputs {
# ...
# }
@config.each do |type, plugin_config_array|
# plugin_config_array has arrays of each component config:
# inputs {
# amqp { ... }
# file { ... }
# file { ... }
# }
plugin_config_array.each do |plugin_config|
yield({
:type => type,
:plugin => plugin_config.keys.first,
:parameters => plugin_config.values.first
})
end
end # @config.each
end # def each
end # class LogStash::Config::Parser
#agent.config(cfg)

539
lib/logstash/config/grammar.rb Normal file
View file

@ -0,0 +1,539 @@
# line 1 "grammar.rl"
require "rubygems"
require "logstash/namespace"
# line 136 "grammar.rl"
class LogStash::Config::Grammar
attr_accessor :eof
attr_accessor :config
def initialize
# BEGIN RAGEL DATA
# line 18 "grammar.rb"
class << self
attr_accessor :_logstash_config_actions
private :_logstash_config_actions, :_logstash_config_actions=
end
self._logstash_config_actions = [
0, 1, 0, 1, 2, 1, 3, 1,
4, 1, 8, 1, 9, 1, 10, 1,
11, 2, 1, 7, 2, 2, 7, 2,
2, 9, 2, 3, 7, 2, 5, 7,
2, 6, 0, 2, 8, 0, 2, 10,
0, 3, 1, 7, 0, 3, 2, 7,
0, 3, 3, 7, 0, 3, 5, 7,
0
]
class << self
attr_accessor :_logstash_config_key_offsets
private :_logstash_config_key_offsets, :_logstash_config_key_offsets=
end
self._logstash_config_key_offsets = [
0, 0, 9, 10, 22, 27, 28, 38,
39, 51, 56, 57, 69, 72, 77, 82,
83, 84, 100, 103, 115, 127, 128, 131,
131, 143, 153, 153, 154, 157, 157, 159,
173, 187, 198, 201, 207, 213, 214, 226,
226, 227, 230, 230, 243, 243, 244, 253
]
class << self
attr_accessor :_logstash_config_trans_keys
private :_logstash_config_trans_keys, :_logstash_config_trans_keys=
end
self._logstash_config_trans_keys = [
32, 35, 95, 9, 10, 65, 90, 97,
122, 10, 32, 35, 95, 123, 9, 10,
48, 57, 65, 90, 97, 122, 32, 35,
123, 9, 10, 10, 32, 35, 95, 125,
9, 10, 65, 90, 97, 122, 10, 32,
35, 95, 123, 9, 10, 48, 57, 65,
90, 97, 122, 32, 35, 123, 9, 10,
10, 32, 34, 35, 39, 95, 125, 9,
10, 65, 90, 97, 122, 10, 34, 92,
32, 35, 61, 9, 10, 32, 35, 61,
9, 10, 10, 62, 32, 34, 35, 39,
43, 45, 91, 95, 9, 10, 48, 57,
65, 90, 97, 122, 10, 34, 92, 32,
34, 35, 39, 95, 125, 9, 10, 65,
90, 97, 122, 32, 34, 35, 39, 95,
125, 9, 10, 65, 90, 97, 122, 10,
10, 39, 92, 32, 35, 61, 95, 9,
10, 48, 57, 65, 90, 97, 122, 32,
35, 95, 125, 9, 10, 65, 90, 97,
122, 10, 10, 39, 92, 48, 57, 32,
34, 35, 39, 95, 125, 9, 10, 48,
57, 65, 90, 97, 122, 32, 34, 35,
39, 95, 125, 9, 10, 48, 57, 65,
90, 97, 122, 32, 34, 35, 39, 95,
9, 10, 65, 90, 97, 122, 10, 34,
92, 32, 35, 44, 93, 9, 10, 32,
35, 44, 93, 9, 10, 10, 32, 34,
35, 39, 95, 125, 9, 10, 65, 90,
97, 122, 10, 10, 39, 92, 32, 35,
44, 93, 95, 9, 10, 48, 57, 65,
90, 97, 122, 10, 32, 35, 95, 9,
10, 65, 90, 97, 122, 32, 35, 95,
9, 10, 65, 90, 97, 122, 0
]
class << self
attr_accessor :_logstash_config_single_lengths
private :_logstash_config_single_lengths, :_logstash_config_single_lengths=
end
self._logstash_config_single_lengths = [
0, 3, 1, 4, 3, 1, 4, 1,
4, 3, 1, 6, 3, 3, 3, 1,
1, 8, 3, 6, 6, 1, 3, 0,
4, 4, 0, 1, 3, 0, 0, 6,
6, 5, 3, 4, 4, 1, 6, 0,
1, 3, 0, 5, 0, 1, 3, 3
]
class << self
attr_accessor :_logstash_config_range_lengths
private :_logstash_config_range_lengths, :_logstash_config_range_lengths=
end
self._logstash_config_range_lengths = [
0, 3, 0, 4, 1, 0, 3, 0,
4, 1, 0, 3, 0, 1, 1, 0,
0, 4, 0, 3, 3, 0, 0, 0,
4, 3, 0, 0, 0, 0, 1, 4,
4, 3, 0, 1, 1, 0, 3, 0,
0, 0, 0, 4, 0, 0, 3, 3
]
class << self
attr_accessor :_logstash_config_index_offsets
private :_logstash_config_index_offsets, :_logstash_config_index_offsets=
end
self._logstash_config_index_offsets = [
0, 0, 7, 9, 18, 23, 25, 33,
35, 44, 49, 51, 61, 65, 70, 75,
77, 79, 92, 96, 106, 116, 118, 122,
123, 132, 140, 141, 143, 147, 148, 150,
161, 172, 181, 185, 191, 197, 199, 209,
210, 212, 216, 217, 227, 228, 230, 237
]
class << self
attr_accessor :_logstash_config_indicies
private :_logstash_config_indicies, :_logstash_config_indicies=
end
self._logstash_config_indicies = [
1, 2, 3, 1, 3, 3, 0, 1,
2, 4, 5, 6, 7, 4, 6, 6,
6, 0, 8, 9, 10, 8, 0, 8,
9, 11, 12, 13, 14, 11, 13, 13,
0, 11, 12, 15, 16, 17, 18, 15,
17, 17, 17, 0, 19, 20, 21, 19,
0, 19, 20, 21, 22, 23, 24, 25,
26, 21, 25, 25, 0, 0, 28, 29,
27, 30, 31, 32, 30, 0, 33, 34,
35, 33, 0, 33, 34, 36, 0, 36,
37, 38, 39, 40, 40, 43, 42, 36,
41, 42, 42, 0, 0, 45, 46, 44,
47, 48, 49, 50, 51, 52, 47, 51,
51, 0, 53, 54, 55, 56, 57, 26,
53, 57, 57, 0, 53, 55, 0, 28,
59, 58, 58, 60, 61, 63, 62, 60,
62, 62, 62, 0, 64, 65, 66, 67,
64, 66, 66, 0, 44, 36, 38, 0,
45, 69, 68, 68, 70, 0, 71, 72,
73, 74, 75, 76, 71, 70, 75, 75,
0, 77, 78, 79, 80, 81, 82, 77,
81, 81, 81, 0, 83, 84, 85, 86,
87, 83, 87, 87, 0, 0, 89, 90,
88, 91, 92, 93, 94, 91, 0, 95,
96, 83, 97, 95, 0, 95, 96, 98,
99, 100, 101, 102, 103, 98, 102, 102,
0, 88, 83, 85, 0, 89, 105, 104,
104, 106, 107, 108, 110, 109, 106, 109,
109, 109, 0, 27, 21, 23, 1, 2,
3, 1, 3, 3, 0, 111, 112, 113,
111, 113, 113, 0, 0
]
class << self
attr_accessor :_logstash_config_trans_targs
private :_logstash_config_trans_targs, :_logstash_config_trans_targs=
end
self._logstash_config_trans_targs = [
0, 1, 2, 3, 4, 5, 3, 6,
4, 5, 6, 6, 7, 8, 47, 9,
10, 8, 11, 9, 10, 11, 12, 45,
22, 24, 25, 12, 13, 44, 14, 15,
16, 14, 15, 16, 17, 18, 27, 28,
30, 31, 32, 33, 18, 19, 26, 20,
12, 21, 22, 24, 25, 20, 12, 21,
22, 24, 22, 23, 14, 15, 24, 16,
6, 7, 8, 47, 28, 29, 31, 20,
12, 21, 22, 24, 25, 20, 12, 21,
22, 32, 25, 33, 34, 40, 41, 43,
34, 35, 39, 36, 37, 33, 38, 36,
37, 38, 20, 12, 21, 22, 24, 25,
41, 42, 36, 37, 33, 43, 38, 1,
2, 3
]
class << self
attr_accessor :_logstash_config_trans_actions
private :_logstash_config_trans_actions, :_logstash_config_trans_actions=
end
self._logstash_config_trans_actions = [
15, 0, 0, 1, 3, 3, 0, 23,
0, 0, 11, 0, 0, 1, 0, 3,
3, 0, 3, 0, 0, 0, 32, 0,
32, 32, 0, 0, 0, 0, 5, 5,
5, 0, 0, 0, 0, 1, 0, 1,
1, 1, 1, 7, 0, 0, 0, 26,
49, 26, 49, 49, 26, 0, 1, 0,
1, 1, 0, 0, 3, 3, 0, 3,
9, 9, 35, 9, 0, 0, 0, 17,
41, 17, 41, 41, 17, 20, 45, 20,
45, 0, 20, 0, 1, 0, 1, 1,
0, 0, 0, 5, 5, 5, 5, 0,
0, 0, 29, 53, 29, 53, 53, 29,
0, 0, 3, 3, 3, 0, 3, 13,
13, 38
]
class << self
attr_accessor :_logstash_config_eof_actions
private :_logstash_config_eof_actions, :_logstash_config_eof_actions=
end
self._logstash_config_eof_actions = [
0, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 0, 13
]
class << self
attr_accessor :logstash_config_start
end
self.logstash_config_start = 46;
class << self
attr_accessor :logstash_config_first_final
end
self.logstash_config_first_final = 46;
class << self
attr_accessor :logstash_config_error
end
self.logstash_config_error = 0;
class << self
attr_accessor :logstash_config_en_main
end
self.logstash_config_en_main = 46;
# line 145 "grammar.rl"
# END RAGEL DATA
@tokenstack = Array.new
@stack = Array.new
@types = Hash.new { |h,k| h[k] = [] }
@edges = []
end
def parse(string)
data = string.unpack("c*")
# BEGIN RAGEL INIT
# line 255 "grammar.rb"
begin
p ||= 0
pe ||= data.length
cs = logstash_config_start
end
# line 159 "grammar.rl"
# END RAGEL INIT
begin
# BEGIN RAGEL EXEC
# line 268 "grammar.rb"
begin
_klen, _trans, _keys, _acts, _nacts = nil
_goto_level = 0
_resume = 10
_eof_trans = 15
_again = 20
_test_eof = 30
_out = 40
while true
_trigger_goto = false
if _goto_level <= 0
if p == pe
_goto_level = _test_eof
next
end
if cs == 0
_goto_level = _out
next
end
end
if _goto_level <= _resume
_keys = _logstash_config_key_offsets[cs]
_trans = _logstash_config_index_offsets[cs]
_klen = _logstash_config_single_lengths[cs]
_break_match = false
begin
if _klen > 0
_lower = _keys
_upper = _keys + _klen - 1
loop do
break if _upper < _lower
_mid = _lower + ( (_upper - _lower) >> 1 )
if data[p] < _logstash_config_trans_keys[_mid]
_upper = _mid - 1
elsif data[p] > _logstash_config_trans_keys[_mid]
_lower = _mid + 1
else
_trans += (_mid - _keys)
_break_match = true
break
end
end # loop
break if _break_match
_keys += _klen
_trans += _klen
end
_klen = _logstash_config_range_lengths[cs]
if _klen > 0
_lower = _keys
_upper = _keys + (_klen << 1) - 2
loop do
break if _upper < _lower
_mid = _lower + (((_upper-_lower) >> 1) & ~1)
if data[p] < _logstash_config_trans_keys[_mid]
_upper = _mid - 2
elsif data[p] > _logstash_config_trans_keys[_mid+1]
_lower = _mid + 2
else
_trans += ((_mid - _keys) >> 1)
_break_match = true
break
end
end # loop
break if _break_match
_trans += _klen
end
end while false
_trans = _logstash_config_indicies[_trans]
cs = _logstash_config_trans_targs[_trans]
if _logstash_config_trans_actions[_trans] != 0
_acts = _logstash_config_trans_actions[_trans]
_nacts = _logstash_config_actions[_acts]
_acts += 1
while _nacts > 0
_nacts -= 1
_acts += 1
case _logstash_config_actions[_acts - 1]
when 0 then
# line 7 "grammar.rl"
begin
@tokenstack.push(p)
#puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
end
when 1 then
# line 12 "grammar.rl"
begin
startpos = @tokenstack.pop
endpos = p
token = string[startpos ... endpos]
#puts "numeric: #{token}"
#puts "numeric?: #{string[startpos,50]}"
#puts [startpos, endpos].join(",")
@stack << token.to_i
end
when 2 then
# line 22 "grammar.rl"
begin
startpos = @tokenstack.pop
endpos = p
token = string[startpos ... endpos]
#puts "string: #{token}"
@stack << token
end
when 3 then
# line 30 "grammar.rl"
begin
startpos = @tokenstack.pop
endpos = p
token = string[startpos + 1 ... endpos - 1] # Skip quotations
# Parse escapes.
token.gsub(/\\./) { |m| return m[1,1] }
#puts "quotedstring: #{token}"
@stack << token
end
when 4 then
# line 41 "grammar.rl"
begin
@array = []
@stack << :array_init
end
when 5 then
# line 46 "grammar.rl"
begin
while @stack.last != :array_init
@array.unshift @stack.pop
end
@stack.pop # pop :array_init
@stack << @array
end
when 6 then
# line 55 "grammar.rl"
begin
@parameters = Hash.new { |h,k| h[k] = [] }
end
when 7 then
# line 59 "grammar.rl"
begin
value = @stack.pop
name = @stack.pop
#puts "parameter: #{name} => #{value}"
@parameters[name] << value
end
when 8 then
# line 66 "grammar.rl"
begin
@components ||= []
name = @stack.pop
#@components << { :name => name, :parameters => @parameters }
@components << { name => @parameters }
end
when 9 then
# line 73 "grammar.rl"
begin
#puts "current component: " + @stack.last
@components = []
end
when 10 then
# line 78 "grammar.rl"
begin
name = @stack.pop
@config ||= Hash.new { |h,k| h[k] = [] }
@config[name] += @components
end
when 11 then
# line 131 "grammar.rl"
begin
# Compute line and column of the cursor (p)
puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
# TODO(sissel): Note what we were expecting?
end
# line 456 "grammar.rb"
end # action switch
end
end
if _trigger_goto
next
end
end
if _goto_level <= _again
if cs == 0
_goto_level = _out
next
end
p += 1
if p != pe
_goto_level = _resume
next
end
end
if _goto_level <= _test_eof
if p == eof
__acts = _logstash_config_eof_actions[cs]
__nacts = _logstash_config_actions[__acts]
__acts += 1
while __nacts > 0
__nacts -= 1
__acts += 1
case _logstash_config_actions[__acts - 1]
when 10 then
# line 78 "grammar.rl"
begin
name = @stack.pop
@config ||= Hash.new { |h,k| h[k] = [] }
@config[name] += @components
end
when 11 then
# line 131 "grammar.rl"
begin
# Compute line and column of the cursor (p)
puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
# TODO(sissel): Note what we were expecting?
end
# line 500 "grammar.rb"
end # eof action switch
end
if _trigger_goto
next
end
end
end
if _goto_level <= _out
break
end
end
end
# line 164 "grammar.rl"
# END RAGEL EXEC
rescue => e
# Compute line and column of the cursor (p)
raise e
end
return cs
end # def parse
def line(str, pos)
return str[0 .. pos].count("\n") + 1
end
def column(str, pos)
return str[0 .. pos].split("\n").last.length
end
end # class LogStash::Config::Grammar
#def parse(string)
#cfgparser = LogStash::Config::Grammar.new
#result = cfgparser.parse(string)
#puts "result %s" % result
#ap cfgparser.config
#end
#parse(File.open(ARGV[0]).read)

39
lib/logstash/config/grammar.rl
View file

@ -33,7 +33,7 @@ require "logstash/namespace"
token = string[startpos + 1 ... endpos - 1] # Skip quotations
# Parse escapes.
token.gsub(/\\./) { |m| return m[1,1] }
token.gsub(/\\./) { |m| m[1,1] }
#puts "quotedstring: #{token}"
@stack << token
}
@ -60,14 +60,19 @@ require "logstash/namespace"
value = @stack.pop
name = @stack.pop
#puts "parameter: #{name} => #{value}"
@parameters[name] << value
if value.is_a?(Array)
@parameters[name] += value
else
@parameters[name] << value
end
}
action component_implementation {
action plugin {
@components ||= []
name = @stack.pop
#@components << { :name => name, :parameters => @parameters }
@components << { name => @parameters }
@parameters = {}
}
action component_init {
@ -79,21 +84,25 @@ require "logstash/namespace"
name = @stack.pop
@config ||= Hash.new { |h,k| h[k] = [] }
@config[name] += @components
#puts "Config component: #{name}"
}
comment = "#" (any - "\n")* ;
comment = "#" (any - [\n])* >mark %{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
ws = ([ \t\n] | comment)** ;
#ws = ([ \t\n])** ;
# TODO(sissel): Support floating point values?
numeric = ( ("+" | "-")? [0-9] :>> [0-9]** ) >mark %stack_numeric;
quoted_string = (
( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" ) |
( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" )
( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" )
| ( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" )
) >mark %stack_quoted_string ;
naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
string = ( quoted_string | naked_string ) ;
# TODO(sissel): allow use of this.
regexp_literal = ( "/" ( ( (any - [\\'\n]) | "\\" any )* ) "/" ) ;
array = ( "[" ws string ws ("," ws string ws)* "]" ) >array_init %array_push;
parameter_value = ( numeric | string | array );
parameter = ( string ws "=>" ws parameter_value ) %parameter ;
@ -101,14 +110,14 @@ require "logstash/namespace"
# Statement:
# component {
# component_implementation_name {
# plugin_name {
# bar => ...
# baz => ...
# }
# ...
# }
component_implementation = (
plugin = (
(
naked_string ws "{" ws
parameters
@ -116,21 +125,21 @@ require "logstash/namespace"
) | (
naked_string ws "{" ws "}"
)
) %component_implementation ;
) %plugin ;
component = (
naked_string ws "{"
>component_init
( ws component_implementation )**
( ws plugin )**
ws "}"
) %component ;
config = (ws component)** ;
config = (ws component? )** ;
main := config
main := config %{ puts "END" }
$err {
# Compute line and column of the cursor (p)
puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
$stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
# TODO(sissel): Note what we were expecting?
} ;
}%%
@ -167,6 +176,10 @@ class LogStash::Config::Grammar
raise e
end
if cs < self.logstash_config_first_final
$stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
raise "Invalid Configuration. Check syntax of config file."
end
return cs
end # def parse

168
lib/logstash/config/mixin.rb
View file

@ -1,6 +1,7 @@
require "logstash/namespace"
require "logstash/config/registry"
require "logstash/logging"
# This module is meant as a mixin to classes wishing to be configurable from
# config files
@ -21,15 +22,38 @@ require "logstash/config/registry"
#
# TODO(sissel): This is not yet fully designed.
module LogStash::Config::Mixin
attr_accessor :config
# This method is called when someone does 'include LogStash::Config'
def self.included(base)
puts "Configurable class #{base.name}"
#puts "Configurable class #{base.name}"
#
# Add the DSL methods to the 'base' given.
base.extend(LogStash::Config::Mixin::DSL)
end
def config_init(params)
# Validation will modify the values inside params if necessary.
# For example: converting a string to a number, etc.
if !self.class.validate(params)
@logger.error "Config validation failed."
exit 1
end
params.each do |key, value|
validator = self.class.validator_find(key)
@logger.info("config #{self.class.name}/@#{key} = #{value.inspect}")
# set @foo
#ivar = "@#{key}"
self.instance_variable_set("@#{key}", value)
end
@config = params
end # def config_init
module DSL
# If name is given, set the name and return it.
# If no name given (nil), return the current name.
def config_name(name=nil)
@ -44,27 +68,11 @@ module LogStash::Config::Mixin
# cfg should be hash with one entry of { "key" => "val" }
@config ||= Hash.new
key, value = cfg.to_a.first
key = key.to_s if key.is_a?(Symbol)
@config[key] = value
return @config
end # def config
def dsl_gen
# TODO(sissel): Deprecated.
puts "#{@dsl_parent.config_name} { #parent" if @dsl_parent
config = []
config << "#{@config_name} { #node"
config << " \"somename\":"
attrs = []
(@config || Hash.new).each do |key, value|
attrs << " #{key} => #{value},"
end
config += attrs
config << "} #node"
config = config.collect { |p| "#{@dsl_parent.nil? ? "" : " "}#{p}" }
puts config.join("\n")
puts "} #parent" if @dsl_parent
end
# This is called whenever someone subclasses a class that has this mixin.
def inherited(subclass)
# Copy our parent's config to a subclass.
@ -79,5 +87,129 @@ module LogStash::Config::Mixin
end
subclass.instance_variable_set("@config", subconfig)
end # def inherited
def validate(params)
@plugin_name = [ancestors[1].config_name, config_name].join("/")
@logger = LogStash::Logger.new(STDERR)
is_valid = true
is_valid &&= validate_check_invalid_parameter_names(params)
is_valid &&= validate_check_parameter_values(params)
return is_valid
end # def validate
def validate_check_invalid_parameter_names(params)
invalid_params = params.keys
# Filter out parametrs that match regexp keys.
# These are defined in plugins like this:
# config /foo.*/ => ...
@config.each_key do |config_key|
if config_key.is_a?(Regexp)
invalid_params.reject! { |k| k =~ config_key }
elsif config_key.is_a?(String)
invalid_params.reject! { |k| k == config_key }
end
end
if invalid_params.size > 0
invalid_params.each do |name|
@logger.error("Unknown setting '#{name}' for #{@plugin_name}")
end
return false
end # if invalid_params.size > 0
return true
end # def validate_check_invalid_parameter_names
def validate_check_parameter_values(params)
# Filter out parametrs that match regexp keys.
# These are defined in plugins like this:
# config /foo.*/ => ...
is_valid = true
params.each do |key, value|
@config.find do |config_key, config_val|
if (config_key.is_a?(Regexp) && key =~ config_key) \
|| (config_key.is_a?(String) && key == config_key)
success, result = validate_value(value, config_val)
if success
params[key] = result if !result.nil?
else
@logger.error("Failed config #{@plugin_name}/#{key}: #{result} (#{value.inspect}")
end
#puts "Result: #{key} / #{result.inspect} / #{success}"
is_valid &&= success
end
end # config.each
end # params.each
return is_valid
end # def validate_check_parameter_values
def validator_find(key)
@config.each do |config_key, config_val|
if (config_key.is_a?(Regexp) && key =~ config_key) \
|| (config_key.is_a?(String) && key == config_key)
return config_val
end
end # @config.each
return nil
end
def validate_value(value, validator)
# Validator comes from the 'config' pieces of plugins.
# They look like this
# config :mykey => lambda do |value| ... end
# (see LogStash::Inputs::File for example)
result = nil
if validator.nil?
return true
elsif validator.is_a?(Proc)
return validator.call(value)
elsif validator.is_a?(Array)
if value.size > 1
return false, "Expected one of #{validator.inspect}, got #{value.inspect}"
end
if !validator.include?(value.first)
return false, "Expected one of #{validator.inspect}, got #{value.inspect}"
end
result = value.first
elsif validator.is_a?(Symbol)
# TODO(sissel): Factor this out into a coersion method?
case validator
when :string
if value.size > 1 # only one value wanted
return false, "Expected string, got #{value.inspect}"
end
result = value.first
when :number
if value.size > 1 # only one value wanted
return false, "Expected number, got #{value.inspect}"
end
if value != value.to_i.to_s # Try convert to number
return false, "Expected number, got #{value.inspect}"
end
result = value.first.to_i
when :boolean
if value.size > 1 # only one value wanted
return false, "Expected boolean, got #{value.inspect}"
end
if value.first !~ /^(true|false)$/
return false, "Expected boolean 'true' or 'false', got #{value.inspect}"
end
result = (value == "true")
end # case validator
else
return false, "Unknown validator #{validator.class}"
end
# Return the validator for later use, like with type coercion.
return true, result
end # def validate_value
end # module LogStash::Config::DSL
end # module LogStash::Config

12
lib/logstash/config/registry.rb Normal file
View file

@ -0,0 +1,12 @@
require "logstash/namespace"
# Global config registry.
module LogStash::Config::Registry
@registry = Hash.new
class << self
attr_accessor :registry
# TODO(sissel): Add some helper methods here.
end
end # module LogStash::Config::Registry

5
lib/logstash/filters/base.rb
View file

@ -8,11 +8,12 @@ class LogStash::Filters::Base
attr_accessor :logger
config_name "filter"
config :type => :string
public
def initialize(config = {})
def initialize(params)
@logger = LogStash::Logger.new(STDERR)
@config = config
config_init(params)
end # def initialize
public

2
lib/logstash/filters/date.rb
View file

@ -4,6 +4,8 @@ require "logstash/time"
class LogStash::Filters::Date < LogStash::Filters::Base
config_name "date"
# Config for date is:
# fieldname: dateformat
# Allow arbitrary keys for this config.

4
lib/logstash/filters/field.rb
View file

@ -3,6 +3,10 @@ require "logstash/namespace"
require "ostruct"
class LogStash::Filters::Field < LogStash::Filters::Base
# TODO(sissel): Finish this.
config_name "field"
class EvalSpace < OpenStruct
def get_binding
return binding

3
lib/logstash/filters/grep.rb
View file

@ -24,6 +24,9 @@ require "logstash/namespace"
# - tag2
#
class LogStash::Filters::Grep < LogStash::Filters::Base
config_name "grep"
public
def initialize(config = {})
super

7
lib/logstash/filters/grok.rb
View file

@ -6,12 +6,13 @@ require "grok" # rubygem 'jls-grok'
class LogStash::Filters::Grok < LogStash::Filters::Base
config :pattern => :string
config :patterns_dir => :path
config_name "grok"
config :pattern => nil
config :patterns_dir => nil
config :drop_if_match => :boolean # googlecode/issue/26
public
def initialize(config = {})
def initialize(params)
super
@grokpiles = {}

3
lib/logstash/filters/grokdiscovery.rb
View file

@ -4,6 +4,9 @@ gem "jls-grok", ">=0.3.3209"
require "grok" # rubygem 'jls-grok'
class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
config_name "grokdiscovery"
public
def initialize(config = {})
super

1
lib/logstash/filters/multiline.rb
View file

@ -8,6 +8,7 @@ require "logstash/namespace"
class LogStash::Filters::Multiline < LogStash::Filters::Base
config_name "multiline"
config :pattern => :string
config :negate => :boolean
config :what => ["previous", "next"]

14
lib/logstash/inputs/amqp.rb
View file

@ -4,15 +4,25 @@ require "logstash/namespace"
require "mq" # rubygem 'amqp'
require "uuidtools" # rubygem 'uuidtools'
require "cgi"
require "uri"
class LogStash::Inputs::Amqp < LogStash::Inputs::Base
MQTYPES = [ "fanout", "queue", "topic" ]
config_name "amqp"
config "pantscon" => :string #LogStash::Config::Path
config "host" => (lambda do |value|
# Use URI to validate.
u = URI.parse("dummy:///")
begin
u.host = value
rescue => e
return false, "Invalid hostname #{value.inspect}"
end
return true
) # config "host"
public
def initialize(url, type, config={}, &block)
def initialize(params)
super
@mq = nil

33
lib/logstash/inputs/base.rb
View file

@ -9,30 +9,23 @@ class LogStash::Inputs::Base
attr_accessor :logger
config_name "input"
config :type => :string
config :tag => (lambda do |value|
re = /^[A-Za-z0-9_]+$/
value.each do |v|
if v !~ re
return [false, "Tag '#{v}' does not match #{re}"]
end # check 'v'
end # value.each
return true
end) # config :tag
# Define the basic config
config "path" => :string #LogStash::Config::Path
config "tag" => :string #LogStash::Config::Array
public
def initialize(configs, output_queue)
def initialize(params)
@logger = LogStash::Logger.new(STDERR)
@configs = configs
@output_queue = output_queue
#@url = url
#@url = URI.parse(url) if url.is_a? String
#@config = config
#@callback = block
#@type = type
#@tags = []
#@urlopts = {}
#if @url.query
# @urlopts = CGI.parse(@url.query)
# @urlopts.each do |k, v|
# @urlopts[k] = v.last if v.is_a?(Array)
# end
#end
config_init(params)
end # def initialize
public

4
lib/logstash/inputs/beanstalk.rb
View file

@ -5,10 +5,12 @@ require "logstash/namespace"
class LogStash::Inputs::Beanstalk < LogStash::Inputs::Base
config_name "beanstalk"
config :tube => nil # TODO(sissel): needs validation?
public
def initialize(url, type, config={}, &block)
def initialize(params)
super
raise "issue/17: needs refactor to support configfile"
if @url.path == "" or @url.path == "/"
raise "must specify a tube for beanstalk output"

5
lib/logstash/inputs/file.rb
View file

@ -6,12 +6,13 @@ require "socket" # for Socket.gethostname
class LogStash::Inputs::File < LogStash::Inputs::Base
config_name "file"
config :path => :string
config :path => nil # no validation on path, it can be anything.
public
def initialize(configs, output_queue)
def initialize(params)
super
#@output_queue = output_queue
@file_threads = {}
end # def initialize

3
lib/logstash/inputs/internal.rb
View file

@ -9,8 +9,9 @@ class LogStash::Inputs::Internal < LogStash::Inputs::Base
config_name "internal"
public
def initialize(url, type, config={}, &block)
def initialize(params)
super
raise "issue/17: needs refactor to support configfile"
# Default host to the machine's hostname if it's not set
@url.host ||= Socket.gethostname

3
lib/logstash/inputs/stomp.rb
View file

@ -7,8 +7,9 @@ class LogStash::Inputs::Stomp < LogStash::Inputs::Base
config_name "stomp"
public
def initialize(url, config={}, &block)
def initialize(params)
super
raise "issue/17: needs refactor to support configfile"
@logger.debug(["Connecting", { :url => @url }])
end # def initialize

4
lib/logstash/inputs/syslog.rb
View file

@ -9,6 +9,10 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
config_name "syslog"
# The address to listen on
config :address => nil # TODO(sissel): needs validation
config :port => nil # TODO(sissel): needs validation
public
def register
if !@url.host or !@url.port

3
lib/logstash/inputs/tcp.rb
View file

@ -8,8 +8,9 @@ class LogStash::Inputs::Tcp < LogStash::Inputs::Base
config_name "tcp"
public
def initialize(url, type, config={}, &block)
def initialize(params)
super
raise "issue/17: needs refactor to support configfile"
end # def initialize
public

1
lib/logstash/inputs/twitter.rb
View file

@ -7,6 +7,7 @@ require "logstash/namespace"
class LogStash::Inputs::Twitter < LogStash::Inputs::Base
config_name "twitter"
config :query => nil # TODO(sissel): Validation?
public
def register

26
lib/logstash/outputs/amqp.rb
View file

@ -9,23 +9,17 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
config_name "amqp"
config :host => :string
config :queue_type => :string
config :queue_name => :string
config :exchange_type => :string
config :name => :string
config :vhost => :string
public
def initialize(url, config={}, &block)
def initialize(params)
super
# Handle path /<vhost>/<type>/<name> or /<type>/<name>
# vhost allowed to contain slashes
if @url.path =~ %r{^/((.*)/)?([^/]+)/([^/]+)}
unused, @vhost, @mqtype, @name = $~.captures
else
raise "amqp urls must have a path of /<type>/name or /vhost/<type>/name where <type> is #{MQTYPES.join(", ")}"
end
if !MQTYPES.include?(@mqtype)
raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.join(", ")}"
p @exchange_type => MQTYPES
if !MQTYPES.include?(@exchange_type)
raise "Invalid exchange_type, #{@exchange_type.inspect}, must be one of #{MQTYPES.join(", ")}"
end
end # def initialize
@ -41,19 +35,19 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
amqpsettings[:user] = @url.user if @url.user
amqpsettings[:pass] = @url.password if @url.password
amqpsettings[:logging] = query_args.include? "debug"
@logger.debug("Connecting with AMQP settings #{amqpsettings.inspect} to set up #{@mqtype.inspect} queue #{@name.inspect}")
@logger.debug("Connecting with AMQP settings #{amqpsettings.inspect} to set up #{@exchange_type.inspect} queue #{@name.inspect}")
@amqp = AMQP.connect(amqpsettings)
@mq = MQ.new(@amqp)
@target = nil
case @mqtype
case @exchange_type
when "fanout"
@target = @mq.fanout(@name)
when "queue"
@target = @mq.queue(@name, :durable => @urlopts["durable"] ? true : false)
when "topic"
@target = @mq.topic(@name)
end # case @mqtype
end # case @exchange_type
end # def register
public

14
lib/logstash/outputs/base.rb
View file

@ -10,20 +10,12 @@ class LogStash::Outputs::Base
attr_accessor :logger
config_name "outputs"
config_name "output"
public
def initialize(url)
@url = url
@url = URI.parse(url) if url.is_a? String
def initialize(params)
@logger = LogStash::Logger.new(STDOUT)
@urlopts = {}
if @url.query
@urlopts = CGI.parse(@url.query)
@urlopts.each do |k, v|
@urlopts[k] = v.last if v.is_a?(Array)
end
end
config_init(params)
end
public

6
lib/logstash/outputs/beanstalk.rb
View file

@ -3,8 +3,12 @@ require "logstash/namespace"
require "em-jack"
class LogStash::Outputs::Beanstalk < LogStash::Outputs::Base
config_name "beanstalk"
config :ttr => :number
public
def initialize(url, config={}, &block)
def initialize(params)
super
@ttr = @urlopts["ttr"] || 300;

1
lib/logstash/outputs/elasticsearch.rb
View file

@ -6,6 +6,7 @@ require "logstash/outputs/base"
class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
# http://host/index/type
config_name "elasticsearch"
config :host => :string
config :index => :string
config :type => :string

3
lib/logstash/outputs/gelf.rb
View file

@ -9,6 +9,9 @@ require "logstash/namespace"
require "logstash/outputs/base"
class LogStash::Outputs::Gelf < LogStash::Outputs::Base
config_name "gelf"
public
def register
# nothing to do

3
lib/logstash/outputs/internal.rb
View file

@ -2,6 +2,9 @@ require "logstash/namespace"
require "logstash/outputs/base"
class LogStash::Outputs::Internal < LogStash::Outputs::Base
config_name "internal"
public
def initialize(url, config={}, &block)
super

3
lib/logstash/outputs/mongodb.rb
View file

@ -3,6 +3,9 @@ require "logstash/namespace"
require "em-mongo"
class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
config_name "mongodb"
public
def register
# TODO(sissel): Port?

1
lib/logstash/outputs/nagios.rb
View file

@ -5,6 +5,7 @@ class LogStash::Outputs::Nagios < LogStash::Outputs::Base
NAGIOS_CRITICAL = 2
NAGIOS_WARN = 1
config_name "nagios"
config :commandfile => :string
public

7
lib/logstash/outputs/stdout.rb
View file

@ -3,16 +3,13 @@ require "logstash/namespace"
class LogStash::Outputs::Stdout < LogStash::Outputs::Base
config_name "stdout"
config :debug => :boolean
public
def initialize(*args)
super
@opts = {}
if @url.path != "/"
@opts = @url.path[1..-1].split(",")
end
end # def register
public
@ -26,6 +23,6 @@ class LogStash::Outputs::Stdout < LogStash::Outputs::Base
public
def debug?
return @opts.member?("debug")
return @debug
end
end # class LogStash::Outputs::Stdout

2
lib/logstash/outputs/stomp.rb
View file

@ -5,6 +5,8 @@ require "logstash/stomp/handler"
class LogStash::Outputs::Stomp < LogStash::Outputs::Base
attr_reader :url
config_name "stomp"
public
def initialize(url, config={}, &block)
super