github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 06:37:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' of https://github.com/logstash/logstash

Browse source
This commit is contained in:
Thomas De Smedt 2013-10-18 00:36:27 +02:00
commit 5696f1da64
8 changed files with 81 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Makefile
View file

@ -2,14 +2,14 @@
# rsync
# wget or curl
#
JRUBY_VERSION=1.7.4
JRUBY_VERSION=1.7.5
ELASTICSEARCH_VERSION=0.90.3
#VERSION=$(shell ruby -r./lib/logstash/version -e 'puts LOGSTASH_VERSION')
VERSION=$(shell awk -F\" '/LOGSTASH_VERSION/ {print $$2}' lib/logstash/version.rb)
WITH_JRUBY=java -jar $(shell pwd)/$(JRUBY) -S
JRUBY=vendor/jar/jruby-complete-$(JRUBY_VERSION).jar
JRUBY_URL=http://repository.codehaus.org/org/jruby/jruby-complete/$(JRUBY_VERSION)
JRUBY_URL=http://jruby.org.s3.amazonaws.com/downloads/$(JRUBY_VERSION)/jruby-complete-$(JRUBY_VERSION).jar
JRUBY_CMD=java -jar $(JRUBY)
JRUBYC=$(WITH_JRUBY) jrubyc
ELASTICSEARCH_URL=http://download.elasticsearch.org/elasticsearch/elasticsearch
@ -96,7 +96,7 @@ build-jruby: $(JRUBY)
$(JRUBY): | vendor/jar
$(QUIET)echo " ==> Downloading jruby $(JRUBY_VERSION)"
$(QUIET)$(DOWNLOAD_COMMAND) $@ http://repository.codehaus.org/org/jruby/jruby-complete/$(JRUBY_VERSION)/jruby-complete-$(JRUBY_VERSION).jar
$(QUIET)$(DOWNLOAD_COMMAND) $@ $(JRUBY_URL)
vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION).tar.gz: | wget-or-curl vendor/jar
@echo "=> Fetching elasticsearch"
@ -136,14 +136,6 @@ vendor/bundle: | vendor $(JRUBY)
@echo "=> Installing gems to $@..."
@#$(QUIET)GEM_HOME=$(GEM_HOME) $(JRUBY_CMD) --1.9 $(GEM_HOME)/bin/bundle install --deployment
$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 ./gembag.rb logstash.gemspec
@# Purge old version of json
#$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem uninstall json -v 1.6.5
@# Purge old versions of gems installed because gembag doesn't do
@# dependency resolution correctly.
$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem uninstall addressable -v 2.2.8
@# uninstall the newer ffi (1.1.5 vs 1.3.1) because that's what makes
@#dependencies happy (launchy wants ffi 1.1.x)
#$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem uninstall ffi -v 1.3.1
@# Purge any junk that fattens our jar without need!
@# The riak gem includes previous gems in the 'pkg' dir. :(
-rm -rf $@/jruby/1.9/gems/riak-client-1.0.3/pkg

2
bin/logstash
View file

@ -46,7 +46,7 @@ if [ "$?" -eq 0 -a -z "$USE_JRUBY" ] ; then
setup
else
# No ruby found, fetch JRuby and run.
jruby="$basedir/vendor/jar/jruby-complete-1.7.4.jar"
jruby="$basedir/vendor/jar/jruby-complete-1.7.5.jar"
[ ! -f "$jruby" ] && make -C $basedir build-jruby
set -- java -jar "$jruby" "$program" "$@"
setup jruby

30
lib/logstash/codecs/netflow.rb
View file

@ -1,6 +1,7 @@
require "logstash/filters/base"
require "logstash/namespace"
# The "netflow" codec is for decoding Netflow v5/v9 flows.
class LogStash::Codecs::Netflow < LogStash::Codecs::Base
config_name "netflow"
milestone 1
@ -8,6 +9,12 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
# Netflow v9 template cache TTL (minutes)
config :cache_ttl, :validate => :number, :default => 4000
# Specify into what field you want the Netflow data.
config :target, :validate => :string, :default => "netflow"
# Specify which Netflow versions you will accept.
config :versions, :validate => :array, :default => [5, 9]
public
def initialize(params={})
super(params)
@ -24,6 +31,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
def decode(payload, &block)
header = Header.read(payload)
unless @versions.include?(header.version)
@logger.warn("Ignoring Netflow version v#{header.version}")
return
end
if header.version == 5
flowset = Netflow5PDU.read(payload)
elsif header.version == 9
@ -42,11 +54,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
# The flowset header gives us the UTC epoch seconds along with
# residual nanoseconds so we can set @timestamp to that easily
event["@timestamp"] = Time.at(flowset.unix_sec, flowset.unix_nsec / 1000).utc
event['netflow'] = {}
event[@target] = {}
# Copy some of the pertinent fields in the header to the event
['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
event['netflow'][f] = flowset[f]
event[@target][f] = flowset[f]
end
# Create fields in the event from each field in the flow record
@ -65,9 +77,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
micros += 1000000
end
# FIXME Again, probably doing this wrong WRT JRuby?
event['netflow'][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
else
event['netflow'][k.to_s] = v
event[@target][k.to_s] = v
end
end
@ -142,15 +154,15 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
records.each do |r|
event = LogStash::Event.new(
"@timestamp" => Time.at(flowset.unix_sec).utc,
"netflow" => {}
@target => {}
)
# Fewer fields in the v9 header
['version', 'flow_seq_num'].each do |f|
event['netflow'][f] = flowset[f]
event[@target][f] = flowset[f]
end
event['netflow']['flowset_id'] = record.flowset_id
event[@target]['flowset_id'] = record.flowset_id
r.each_pair do |k,v|
case k.to_s
@ -159,9 +171,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
seconds = flowset.unix_sec - (millis / 1000)
# v9 did away with the nanosecs field
micros = 1000000 - (millis % 1000)
event['netflow'][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
else
event['netflow'][k.to_s] = v
event[@target][k.to_s] = v
end
end

2
lib/logstash/config/config_ast.rb
View file

@ -299,7 +299,7 @@ module LogStash; module Config; module AST
module NotInExpression
def compile
item, list = recursive_select(LogStash::Config::AST::RValue)
return "(x = #{list.compile}; x.respond_to?(:include?) && !x.include?(#{item.compile}))"
return "(x = #{list.compile}; !x.respond_to?(:include?) || !x.include?(#{item.compile}))"
end
end

7
lib/logstash/filters/csv.rb
View file

@ -24,6 +24,11 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
# Optional.
config :separator, :validate => :string, :default => ","
# Define the character used to quote CSV fields. If this is not specified
# the default is a double quote '"'
# Optional.
config :quote_char, :validate => :string, :default => '"'
# Define target for placing the data
# Defaults to writing to the root of the event.
config :target, :validate => :string
@ -57,7 +62,7 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
raw = event[@source].first
begin
values = CSV.parse_line(raw, :col_sep => @separator)
values = CSV.parse_line(raw, :col_sep => @separator, :quote_char => @quote_char)
if @target.nil?
# Default is to write to the root of the event.

2
lib/logstash/inputs/unix.rb
View file

@ -84,7 +84,7 @@ class LogStash::Inputs::Unix < LogStash::Inputs::Base
decorate(event)
event["host"] = hostname
event["path"] = @path
output_queue << e
output_queue << event
end
end # loop do
rescue => e

2
spec/conditionals/test.rb
View file

@ -162,6 +162,7 @@ describe "conditionals" do
if !("foo" not in "foo") { mutate { add_tag => "notfoo" } }
if "foo" not in [somelist] { mutate { add_tag => "notsomelist" } }
if "one" not in [somelist] { mutate { add_tag => "somelist" } }
if "foo" not in [alsomissing] { mutate { add_tag => "no string in missing field" } }
}
CONFIG
@ -174,6 +175,7 @@ describe "conditionals" do
insist { subject["tags"] }.include?("notfoo")
insist { subject["tags"] }.include?("notsomelist")
reject { subject["tags"] }.include?("somelist")
insist { subject["tags"] }.include?("no string in missing field")
end
end

46
spec/filters/csv.rb
View file

@ -35,6 +35,52 @@ describe LogStash::Filters::CSV do
end
end
describe "custom quote char" do
config <<-CONFIG
filter {
csv {
quote_char => "'"
}
}
CONFIG
sample "big,bird,'sesame street'" do
insist { subject["column1"] } == "big"
insist { subject["column2"] } == "bird"
insist { subject["column3"] } == "sesame street"
end
end
describe "default quote char" do
config <<-CONFIG
filter {
csv {
}
}
CONFIG
sample 'big,bird,"sesame, street"' do
insist { subject["column1"] } == "big"
insist { subject["column2"] } == "bird"
insist { subject["column3"] } == "sesame, street"
end
end
describe "null quote char" do
config <<-CONFIG
filter {
csv {
quote_char => "\x00"
}
}
CONFIG
sample 'big,bird,"sesame" street' do
insist { subject["column1"] } == 'big'
insist { subject["column2"] } == 'bird'
insist { subject["column3"] } == '"sesame" street'
end
end
describe "given columns" do
# The logstash config goes here.
# At this time, only filters are supported.